Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 jschoubben
|
dd920ff854 |
@@ -1,335 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// An address is read from the node's settings where it is used, never recorded with a port
|
||||
// (novox/hq 04-ISSUES/102). Three readers did not follow the setting; each is held to it here.
|
||||
|
||||
var aDigest = "sha256:" + strings.Repeat("e", 64)
|
||||
|
||||
// **A build is recorded by digest and path**, whatever address the builder pushed to — and only
|
||||
// what the build made is rewritten: an image the module runs from elsewhere is left where it says.
|
||||
func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
||||
manifest, _ := json.Marshal(map[string]any{
|
||||
"module": "gitea", "version": "1",
|
||||
"resources": []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-gitea",
|
||||
"image": "anchor.internal:5100/gitea/server@" + aDigest},
|
||||
{"id": "config", "type": "archive", "path": "/etc/gitea", "digest": aDigest,
|
||||
"source": "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
||||
{"id": "cache", "type": "container", "name": "mesh-gitea-cache",
|
||||
"image": "valkey/valkey@" + aDigest},
|
||||
},
|
||||
})
|
||||
kept := buildFrom(link.BuildResult{
|
||||
ID: "b1", Repository: "https://forge.example/gitea.git", Commit: "abc", On: "laptop",
|
||||
Manifest: manifest,
|
||||
Made: []link.MadeArtifact{
|
||||
{Name: "server", Kind: "image", Reference: "anchor.internal:5100/gitea/server@" + aDigest},
|
||||
{Name: "config", Kind: "archive", Reference: "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
||||
},
|
||||
Against: []string{"anchor.internal:5100/mesh-tools/runtime@" + aDigest},
|
||||
})
|
||||
if kept.Module != "gitea" {
|
||||
t.Fatalf("the module was not read from the recorded manifest: %q", kept.Module)
|
||||
}
|
||||
if kept.Made[0].Reference != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
||||
t.Errorf("the image is recorded as %q, address and all", kept.Made[0].Reference)
|
||||
}
|
||||
if kept.Made[1].Reference != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
||||
t.Errorf("the archive is recorded as %q, address and all", kept.Made[1].Reference)
|
||||
}
|
||||
recorded, err := catalogue.ParseManifest(kept.Manifest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recorded.Resources[0]["image"]; got != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
||||
t.Errorf("the recorded manifest's image is %v", got)
|
||||
}
|
||||
if got := recorded.Resources[1]["source"]; got != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
||||
t.Errorf("the recorded manifest's archive is %v", got)
|
||||
}
|
||||
if got := recorded.Resources[2]["image"]; got != "valkey/valkey@"+aDigest {
|
||||
t.Errorf("an image the build did not make was rewritten: %v", got)
|
||||
}
|
||||
if strings.Contains(string(kept.Manifest), "anchor.internal:5100") {
|
||||
t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest)
|
||||
}
|
||||
if kept.Against[0] != "anchor.internal:5100/mesh-tools/runtime@"+aDigest {
|
||||
t.Errorf("what the build stood on was rewritten: %v", kept.Against)
|
||||
}
|
||||
}
|
||||
|
||||
// aStore is a module offering the artifact store on 5000, published the long way as the
|
||||
// distribution module does, so a node may be given another number for it.
|
||||
func aStore() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "distribution", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: catalogue.ArtifactStoreProvision, Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": float64(5000)}},
|
||||
Listens: []catalogue.Listening{{Port: 5000, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "store", "type": "container", "name": "mesh-registry",
|
||||
"ports": []any{"5000:5000"}, "image": "registry@" + aDigest}}}
|
||||
}
|
||||
|
||||
// **The trust a machine writes for the store, and the address every built image is fetched
|
||||
// through, say the port the node gave the store** — not the catalogue's number.
|
||||
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A module the mesh built, recorded by digest and path, running on the other machine.
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "app", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
|
||||
"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
on := map[string]bool{"anchor": true, "laptop": true}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||
if err != nil || !found || node != "anchor" || port != "5101" {
|
||||
t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
|
||||
}
|
||||
|
||||
var trust string
|
||||
for _, r := range composed(t, open, "laptop").Resources {
|
||||
if r["path"] == "/etc/docker/daemon.json" {
|
||||
trust, _ = r["content"].(string)
|
||||
}
|
||||
if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Errorf("the image the mesh built is fetched as %v", r["image"])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
|
||||
t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
|
||||
}
|
||||
|
||||
// And a replay to the catalogue says where the store is now.
|
||||
announced, err := following{open}.Announceable(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Fatalf("the replay announces %+v", announced)
|
||||
}
|
||||
}
|
||||
|
||||
// **The control plane's own connections say the port the node gave the store and the broker.**
|
||||
//
|
||||
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
||||
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
||||
// container is told so beside the sealed connection genesis wrote.
|
||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "server", Kind: catalogue.ArtifactImage,
|
||||
Reference: "registry.example/control@" + aDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, control)
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
|
||||
register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
|
||||
{Port: 5672, From: catalogue.FromMesh}},
|
||||
Guards: []int{15672},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The store's module is registered and given its port, and NOT assigned: the state genesis
|
||||
// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var env map[string]any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "mesh-controller.server" {
|
||||
env, _ = r["env"].(map[string]any)
|
||||
}
|
||||
}
|
||||
if env == nil {
|
||||
t.Fatal("the control plane's container is not in its own node's declaration")
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
"MESH_STORE_IDENTITY_PORT": "6852",
|
||||
"MESH_STORE_LICENCES_PORT": "6852",
|
||||
"MESH_BROKER_AMQP_PORT": "5679",
|
||||
// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
|
||||
// because the sealed value beside it carries the port genesis wrote (finding F3).
|
||||
"MESH_BROKER_ADDRESS_PORT": "",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "",
|
||||
} {
|
||||
if env[key] != want {
|
||||
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// withSeatPorts is the control plane's manifest with the seat placeholders in its environment —
|
||||
// added here until module.json carries them (the manifest lands one commit after the code that
|
||||
// fills it, so a control plane one build behind never sees a placeholder it cannot fill).
|
||||
func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
||||
seatPorts := map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}",
|
||||
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}",
|
||||
}
|
||||
out := m
|
||||
out.Resources = nil
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "container" {
|
||||
out.Resources = append(out.Resources, r)
|
||||
continue
|
||||
}
|
||||
copied := map[string]any{}
|
||||
for k, v := range r {
|
||||
copied[k] = v
|
||||
}
|
||||
env := map[string]any{}
|
||||
if had, ok := r["env"].(map[string]any); ok {
|
||||
for k, v := range had {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range seatPorts {
|
||||
if _, said := env[k]; !said {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
copied["env"] = env
|
||||
out.Resources = append(out.Resources, copied)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// aLoneNode is one capable machine with nothing placed on any network — the control-node during
|
||||
// genesis, before the "network" step, which is after the store, the broker, the vault and the
|
||||
// catalogue have each been built and pushed (finding F2).
|
||||
func aLoneNode(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
inventory.ForTest(t)
|
||||
licences.ForTest(t)
|
||||
open, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record, err := open.inventory.AddNode(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true}}})
|
||||
var profile map[string]any
|
||||
_ = json.Unmarshal(reported, &profile)
|
||||
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
// **Before the network exists, the store's own node reaches it by loopback** — never refused,
|
||||
// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to
|
||||
// a node on no network, and builds the catalogue on a base it must be able to pull.
|
||||
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
|
||||
open := aLoneNode(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
|
||||
Requires: []string{catalogue.ArtifactStoreProvision},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
|
||||
"image": "registry.example/mesh-builder@" + aDigest}}})
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
|
||||
"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
|
||||
for _, module := range []string{"distribution", "builder", "postgres"} {
|
||||
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var image any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "postgres.runtime" {
|
||||
image = r["image"]
|
||||
}
|
||||
}
|
||||
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
||||
}
|
||||
held := heldBy(ctx)
|
||||
if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("a builder beside the store is handed the base %q", got)
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// A node is adopted or converged (novox/hq ADR 0100), and it is said to be adopted wherever the
|
||||
@@ -46,9 +45,6 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node
|
||||
return nil
|
||||
}
|
||||
fmt.Printf(" firewall found %s\n", orNone(said.Firewall))
|
||||
if err := showTunnel(ctx, inv, node.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(said.Held) == 0 {
|
||||
fmt.Printf(" holding nothing found\n")
|
||||
}
|
||||
@@ -67,44 +63,6 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node
|
||||
return nil
|
||||
}
|
||||
|
||||
// showTunnel is the node show lines about the tunnel an adopted node found and carried (novox/hq
|
||||
// ADR 0105): what it presented at enrolment, and what it last said about taking it over.
|
||||
func showTunnel(ctx context.Context, inv *inventory.Inventory, name string) error {
|
||||
tunnel, err := inv.TunnelOf(ctx, name)
|
||||
if errors.Is(err, inventory.ErrNoTunnel) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf(" tunnel found %s on port %d, %s in %s, %d peer(s)\n",
|
||||
tunnel.Interface, tunnel.Port, tunnel.Address, tunnel.Range, len(tunnel.Peers))
|
||||
carried, said, err := inv.CarriedTunnelOf(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case !said:
|
||||
fmt.Printf(" %-17s not yet taken over — the node has not said so\n", "")
|
||||
case carried.State == inventory.CarriedTaken:
|
||||
fmt.Printf(" %-17s taken over: %s is down and disabled, never flushed; the mesh's interface "+
|
||||
"runs with its key, port and %d peer(s)\n", "", carried.Interface, carried.Peers)
|
||||
case carried.State == inventory.CarriedDown:
|
||||
fmt.Printf(" %-17s TUNNEL DOWN: %s is stopped and the mesh's interface is not up — the peers "+
|
||||
"reach nothing. On the machine: systemctl start %s\n", "", carried.Interface,
|
||||
"wg-quick@"+carried.Interface)
|
||||
default:
|
||||
fmt.Printf(" %-17s NOT taken over: %s is still the interface the peers reach\n", "", carried.Interface)
|
||||
}
|
||||
if said && carried.Note != "" {
|
||||
fmt.Printf(" %-17s %s\n", "", carried.Note)
|
||||
}
|
||||
if said && carried.Kept != "" {
|
||||
fmt.Printf(" %-17s its configuration's original kept at %s\n", "", carried.Kept)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func orNone(s string) string {
|
||||
if s == "" {
|
||||
return "none reported"
|
||||
@@ -255,28 +213,6 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
return "", fmt.Errorf("%s still holds what it found, and a service is taken on its own, "+
|
||||
"never by the flip:\n%s", node, strings.Join(holding, "\n"))
|
||||
}
|
||||
// And refused while a peer of the tunnel this hub took over has not enrolled (novox/hq ADR
|
||||
// 0105): the flip loads the derived filter and retires the found firewall, and a machine the
|
||||
// mesh has no record of is not one the filter admits — it would go dark.
|
||||
if _, hubName, adopted, err := inv.AdoptedTunnel(ctx); err != nil {
|
||||
return "", err
|
||||
} else if adopted && hubName == node {
|
||||
carried, err := inv.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var waiting []string
|
||||
for _, c := range carried {
|
||||
if c.EnrolledAs == "" {
|
||||
waiting = append(waiting, fmt.Sprintf(" %s at %s", overlay.CarriedName(c.PublicKey), c.Address))
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
return "", fmt.Errorf("%s carries peers of the tunnel it took over that have not enrolled, and "+
|
||||
"converging would cut them off — enrol each first (`overlay show` says which are enrolled):\n%s",
|
||||
node, strings.Join(waiting, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -68,11 +68,6 @@ func buildCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
// buildFrom turns what a builder said into what the mesh keeps.
|
||||
//
|
||||
// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder
|
||||
// says `<registry>:<port>/<module>/<artifact>@sha256:…`; the mesh records the artifact-store
|
||||
// reference and composes the store's address back in where a reference is used. `against` is kept
|
||||
// as announced: it is what the build stood on as the builder saw it, and the catalogue's edge.
|
||||
func buildFrom(result link.BuildResult) inventory.Build {
|
||||
kept := inventory.Build{
|
||||
ID: result.ID, Repository: result.Repository, Ref: result.Ref,
|
||||
@@ -82,21 +77,16 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
// edges, and it is not always listening when a build happens — on a fresh mesh it cannot
|
||||
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
|
||||
// rebuild the graph rather than a list of names.
|
||||
Path: result.Path, Against: result.Against,
|
||||
Path: result.Path, Manifest: result.Manifest, Against: result.Against,
|
||||
}
|
||||
var announced []inventory.Artifact
|
||||
for _, made := range result.Made {
|
||||
announced = append(announced, inventory.Artifact{
|
||||
kept.Made = append(kept.Made, inventory.Artifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||
})
|
||||
kept.Made = append(kept.Made, inventory.Artifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: catalogue.Recorded(made.Reference),
|
||||
})
|
||||
}
|
||||
kept.Manifest = recordedManifest(result.Manifest, announced)
|
||||
// The module name comes from the manifest, which only exists when the build got that far.
|
||||
if len(kept.Manifest) > 0 {
|
||||
if m, err := catalogue.ParseManifest(kept.Manifest); err == nil {
|
||||
if len(result.Manifest) > 0 {
|
||||
if m, err := catalogue.ParseManifest(result.Manifest); err == nil {
|
||||
kept.Module = m.Module
|
||||
}
|
||||
}
|
||||
@@ -388,8 +378,7 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
kept := buildFrom(result)
|
||||
if err := inv.RecordBuild(ctx, kept); err != nil {
|
||||
if err := inv.RecordBuild(ctx, buildFrom(result)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -399,15 +388,13 @@ func buildOne(ctx context.Context, repository, path, ref string, wait time.Durat
|
||||
return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
|
||||
}
|
||||
|
||||
// Said as recorded: what each artifact is, not where this builder happened to push it.
|
||||
for _, made := range kept.Made {
|
||||
for _, made := range result.Made {
|
||||
fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference)
|
||||
}
|
||||
|
||||
// Parsed with the same parser a hand-written manifest goes through. A second path would be a
|
||||
// second thing to disagree about what a manifest is. The manifest as recorded, so the catalogue
|
||||
// holds references by digest and path and every declaration composes the store's address in.
|
||||
manifest, err := catalogue.ParseManifest(kept.Manifest)
|
||||
// second thing to disagree about what a manifest is.
|
||||
manifest, err := catalogue.ParseManifest(result.Manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||
result.On, result.Repository, err)
|
||||
@@ -494,9 +481,6 @@ type answers struct {
|
||||
// all, and one that does gets a refusal naming exactly what is missing — which is a better sentence
|
||||
// than a build command refusing to start because a query did not run. So the store not opening is
|
||||
// reported and the build goes ahead without it.
|
||||
//
|
||||
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
|
||||
// base is recorded by digest and path, and a build machine needs something it can pull.
|
||||
func heldBy(ctx context.Context) map[string]string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -510,18 +494,5 @@ func heldBy(ctx context.Context) map[string]string {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
|
||||
return nil
|
||||
}
|
||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not find the artifact store on this mesh's network, so a "+
|
||||
"module naming a base will be handed a reference nothing can fetch: %v\n", err)
|
||||
return held
|
||||
}
|
||||
if address == "" {
|
||||
return held
|
||||
}
|
||||
routed := make(map[string]string, len(held))
|
||||
for repository, reference := range held {
|
||||
routed[repository] = catalogue.Rerouted(reference, address)
|
||||
}
|
||||
return routed
|
||||
return held
|
||||
}
|
||||
|
||||
+20
-232
@@ -7,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -22,28 +21,11 @@ import (
|
||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||
|
||||
// DefaultOverlayCIDR is the range the mesh allocates from when nothing says another.
|
||||
const DefaultOverlayCIDR = "10.42.0.0/16"
|
||||
|
||||
// overlayRange is the range the mesh allocates node addresses from.
|
||||
//
|
||||
// **The adopted tunnel's range first** (novox/hq ADR 0105): a hub that took over the tunnel it
|
||||
// found is at that tunnel's address, its peers are at theirs, and every node's address is
|
||||
// composed from the same range — the hub's, and every binding, hosts entry and endpoint derived
|
||||
// from it. Those are readers of this; none of them stores the range. Without an adopted tunnel,
|
||||
// the range genesis was told, or the default.
|
||||
func overlayRange(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
tunnel, _, adopted, err := inv.AdoptedTunnel(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if adopted {
|
||||
return tunnel.Range, nil
|
||||
}
|
||||
func overlayCIDR() string {
|
||||
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
|
||||
return v, nil
|
||||
return v
|
||||
}
|
||||
return DefaultOverlayCIDR, nil
|
||||
return "10.42.0.0/16"
|
||||
}
|
||||
|
||||
func overlayCommand(ctx context.Context, args []string) error {
|
||||
@@ -128,46 +110,16 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string)
|
||||
}
|
||||
}
|
||||
|
||||
// A hub that took over a tunnel listens on that tunnel's port — it is what the peers dial, and
|
||||
// the reason the port is worth having (novox/hq ADR 0105). An endpoint on another port would
|
||||
// have the mesh's interface up where no peer is listening for it.
|
||||
found, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var tunnel inventory.Tunnel
|
||||
adoptsTunnel := false
|
||||
if *hub && found.Adopted {
|
||||
if t, err := inv.TunnelOf(ctx, node); err == nil {
|
||||
tunnel = t
|
||||
placed, _ := inv.Overlays(ctx)
|
||||
for _, o := range placed {
|
||||
if o.Name == node && o.Key == t.PublicKey {
|
||||
adoptsTunnel = true
|
||||
}
|
||||
}
|
||||
} else if !errors.Is(err, inventory.ErrNoTunnel) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if adoptsTunnel {
|
||||
if port := portOfEndpoint(*endpoint); port != strconv.Itoa(tunnel.Port) {
|
||||
return fmt.Errorf("%s takes over the tunnel it found on %s, which listens on port %d, and "+
|
||||
"its endpoint %q names another port: the peers dial the tunnel's port, so the hub's "+
|
||||
"endpoint must be on it", node, tunnel.Interface, tunnel.Port, *endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
// Declared, all three. The address is evidence of reachability and is not the fact, and hub
|
||||
// election by address prefix fails silently (novox/hq ADR 0007).
|
||||
if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
found, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
address, err := inv.AssignAddress(ctx, found.ID, cidr)
|
||||
address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -176,10 +128,6 @@ func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string)
|
||||
fmt.Println(" credentials issued for it before this placement keep their old broker address —" +
|
||||
" `module issue` them again and push (novox/hq issue 059)")
|
||||
switch {
|
||||
case adoptsTunnel:
|
||||
fmt.Printf(" the hub — it takes over the tunnel it found on %s: range %s, port %d, "+
|
||||
"%d peer(s) carried until they enrol\n", tunnel.Interface, tunnel.Range, tunnel.Port,
|
||||
len(tunnel.Peers))
|
||||
case *hub:
|
||||
fmt.Println(" the hub — every node not sharing a site routes through it")
|
||||
case *endpoint == "":
|
||||
@@ -206,47 +154,15 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The tunnels adopted nodes take over, and the peers the hub's carries (novox/hq ADR 0105).
|
||||
tunnels, err := inv.Tunnels(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
carried, err := inv.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := make([]overlay.Node, 0, len(places))
|
||||
for _, p := range places {
|
||||
if !on[p.Name] {
|
||||
continue
|
||||
}
|
||||
n := overlay.Node{
|
||||
nodes = append(nodes, overlay.Node{
|
||||
Name: p.Name, Key: p.Key, Endpoint: p.Endpoint,
|
||||
Site: p.Site, Hub: p.Hub, Address: p.Address,
|
||||
}
|
||||
if t, takes := tunnels[p.Name]; takes && t.NodeAdopted {
|
||||
// Only an adopted node is told to take the found unit over: on a converged one there
|
||||
// is nothing found to keep, and the host refuses the field. The range and the carried
|
||||
// peers do not depend on the mode; the takeover does.
|
||||
//
|
||||
// **Refused, not composed, when the hub's record disagrees with the tunnel.** A
|
||||
// declaration that stopped the found unit and raised the mesh's interface on another
|
||||
// port or address would leave every peer dark while reporting the tunnel taken — so a
|
||||
// hub placed before it took the tunnel over (or at the wrong port) is named here, and
|
||||
// nothing is sent until it is re-placed.
|
||||
if wrong := disagrees(p, t.Tunnel); wrong != "" {
|
||||
return nil, fmt.Errorf("%s takes over the tunnel on %s and its placement disagrees with it: %s. "+
|
||||
"Re-place it — `overlay place %s --hub --endpoint <host>:%d …` — and push again; "+
|
||||
"nothing was composed", p.Name, t.Interface, wrong, p.Name, t.Port)
|
||||
}
|
||||
n.TakesOver = &overlay.TakeOver{Interface: t.Interface, Unit: t.Unit, Config: t.Config}
|
||||
}
|
||||
if p.Hub {
|
||||
for _, c := range carried {
|
||||
n.Carried = append(n.Carried, overlay.Carried{Key: c.PublicKey, Address: c.Address})
|
||||
}
|
||||
}
|
||||
nodes = append(nodes, n)
|
||||
})
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
// Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this
|
||||
@@ -254,11 +170,7 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
// "no hub" to somebody who never asked for a network would be a lie about the cause.
|
||||
return overlay.Empty(), nil
|
||||
}
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g, err := overlay.From(nodes, cidr, "")
|
||||
g, err := overlay.From(nodes, overlayCIDR(), "")
|
||||
if g != nil {
|
||||
// The artifact store, as this network reaches it. Found rather than configured: the
|
||||
// provider is whichever module offers it, on whichever machine holds that module — and if
|
||||
@@ -380,17 +292,6 @@ func overlayShow(ctx context.Context, open *stores) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// The tunnel the hub took over, if any, and the peers carried from it (novox/hq ADR 0105):
|
||||
// listed apart from the nodes, because they are peers of the tunnel and not nodes of the
|
||||
// mesh until they enrol — and once one has, it is listed as the node it became.
|
||||
tunnel, hubName, adopted, err := open.inventory.AdoptedTunnel(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
carried, err := open.inventory.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, n := range nodes {
|
||||
place := n.Address
|
||||
if place == "" {
|
||||
@@ -400,74 +301,22 @@ func overlayShow(ctx context.Context, open *stores) error {
|
||||
}
|
||||
fmt.Printf("%-16s %-14s", n.Name, place)
|
||||
switch {
|
||||
case n.Hub && adopted:
|
||||
fmt.Printf(" hub — over the tunnel it took over on %s (range %s, port %d)",
|
||||
tunnel.Interface, tunnel.Range, tunnel.Port)
|
||||
case n.Hub && hubName == n.Name && tunnel.Interface != "":
|
||||
fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's; "+
|
||||
"`mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
|
||||
case n.Hub:
|
||||
fmt.Print(" hub — found no tunnel; if the machine runs the predecessor's, " +
|
||||
"`mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
|
||||
fmt.Print(" hub")
|
||||
case !n.Reachable():
|
||||
fmt.Print(" not dialable")
|
||||
}
|
||||
if n.Site != "" {
|
||||
fmt.Printf(" at %s", n.Site)
|
||||
}
|
||||
if n.TakesOver != nil && !n.Hub {
|
||||
fmt.Printf(" takes over %s", n.TakesOver.Interface)
|
||||
}
|
||||
fmt.Println()
|
||||
for _, p := range computed[n.Name] {
|
||||
fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why)
|
||||
}
|
||||
}
|
||||
if len(carried) > 0 {
|
||||
fmt.Printf("\npeers of the tunnel %s took over — not nodes of the mesh until they enrol:\n", hubName)
|
||||
for _, c := range carried {
|
||||
state := "not yet enrolled"
|
||||
if c.EnrolledAs != "" {
|
||||
state = "enrolled as " + c.EnrolledAs + ", which keeps this address"
|
||||
}
|
||||
fmt.Printf(" %-16s %-14s %s\n", overlay.CarriedName(c.PublicKey), c.Address, state)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// disagrees says how a node's placement differs from the tunnel it takes over — its address not
|
||||
// the tunnel's, its endpoint not on the tunnel's port — or nothing when both agree.
|
||||
func disagrees(p inventory.Overlay, t inventory.Tunnel) string {
|
||||
var wrong []string
|
||||
want := t.Address
|
||||
if i := strings.Index(want, "/"); i >= 0 {
|
||||
want = want[:i]
|
||||
}
|
||||
if p.Address != want {
|
||||
wrong = append(wrong, fmt.Sprintf("its address is %s and the tunnel's is %s", orNothing(p.Address), want))
|
||||
}
|
||||
if p.Reachable() && portOfEndpoint(p.Endpoint) != strconv.Itoa(t.Port) {
|
||||
wrong = append(wrong, fmt.Sprintf("its endpoint %s is not on the tunnel's port %d", p.Endpoint, t.Port))
|
||||
}
|
||||
return strings.Join(wrong, "; ")
|
||||
}
|
||||
|
||||
func orNothing(s string) string {
|
||||
if s == "" {
|
||||
return "unset"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// portOfEndpoint is the port in host:port, or empty.
|
||||
func portOfEndpoint(endpoint string) string {
|
||||
if i := strings.LastIndex(endpoint, ":"); i >= 0 {
|
||||
return endpoint[i+1:]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// SilentFor is how long a node may be quiet before the mesh says so.
|
||||
//
|
||||
// A node speaks every minute, so three of them missed is a gap rather than a slow one. The number
|
||||
@@ -591,11 +440,8 @@ func namesInTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// artifactStoreOnNetwork is the machine on this network that offers the artifact store, and the
|
||||
// port THAT MACHINE put it on — the node's setting when it was given one (novox/hq ADR 0100,
|
||||
// 04-ISSUES/102), the mesh's assignment when it made one, and the manifest's own number only when
|
||||
// neither says anything. Read exactly as a consumer's binding is, because the trust a machine
|
||||
// writes for the store and the address it pulls from are the same fact as what a consumer is told.
|
||||
// artifactStoreOnNetwork is the machine and port the mesh's artifact store answers on, when a
|
||||
// module providing it is assigned to a machine that is on the private network.
|
||||
//
|
||||
// A lookup failure is an error, never "not found": collapsing the two composed a declaration
|
||||
// without the trust whenever the inventory hiccuped, delivered by a push that reported success —
|
||||
@@ -608,87 +454,29 @@ func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading the catalogue: %w", err)
|
||||
}
|
||||
providers := map[string]catalogue.Manifest{}
|
||||
providers := map[string]string{} // module -> served port
|
||||
for name, m := range shelf {
|
||||
if _, offers := m.Serves[catalogue.ArtifactStoreProvision]; offers {
|
||||
providers[name] = m
|
||||
served, offers := m.Serves[catalogue.ArtifactStoreProvision]
|
||||
if !offers {
|
||||
continue
|
||||
}
|
||||
if p, ok := served["port"]; ok {
|
||||
providers[name] = fmt.Sprintf("%v", p)
|
||||
}
|
||||
}
|
||||
if len(providers) == 0 {
|
||||
return "", "", false, nil
|
||||
}
|
||||
// In a stated order, so two machines offering it would always answer the same one.
|
||||
machines := make([]string, 0, len(on))
|
||||
for machine := range on {
|
||||
machines = append(machines, machine)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
for _, machine := range machines {
|
||||
assigned, err := inv.Assigned(ctx, machine)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading what %s is assigned: %w", machine, err)
|
||||
}
|
||||
for _, a := range assigned {
|
||||
m, offers := providers[a]
|
||||
if !offers {
|
||||
continue
|
||||
}
|
||||
serves, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading where %s puts the artifact store: %w", machine, err)
|
||||
}
|
||||
if p, ok := serves["port"]; ok {
|
||||
return machine, fmt.Sprintf("%v", p), true, nil
|
||||
if p, ok := providers[a]; ok {
|
||||
return machine, p, true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", "", false, nil
|
||||
}
|
||||
|
||||
// artifactStoreAddress is the artifact store as `forNode` reaches it: `<node>.internal:<port>`
|
||||
// over the private network, or — when nothing is on the network yet — `127.0.0.1:<port>` for the
|
||||
// node that holds the store itself, and "" for any other. The address composed into every
|
||||
// reference the mesh built, at the moment it is used and never before (novox/hq 04-ISSUES/102).
|
||||
//
|
||||
// **Genesis places the network after the store, the broker, the vault and the catalogue.** Each
|
||||
// of those is built and pushed to a node that is on no network, and the store is on that same
|
||||
// node; an answer of "no store" there would refuse every one of those pushes and hand every one
|
||||
// of those builds a base nothing can pull. Loopback is the truth on that machine, and it is the
|
||||
// address genesis itself reaches the store by.
|
||||
func artifactStoreAddress(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest, forNode string) (string, error) {
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
on := map[string]bool{}
|
||||
for name := range onNetwork {
|
||||
on[name] = true
|
||||
}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, inv, on)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if found {
|
||||
return overlay.InternalName(node) + ":" + port, nil
|
||||
}
|
||||
holder, port, found, err := artifactStoreHolder(ctx, inv)
|
||||
if err != nil || !found || holder != forNode {
|
||||
return "", err
|
||||
}
|
||||
return "127.0.0.1:" + port, nil
|
||||
}
|
||||
|
||||
// artifactStoreHolder is whichever node is assigned a module offering the artifact store, on or
|
||||
// off the network, and the port that node put it on.
|
||||
func artifactStoreHolder(ctx context.Context, inv *inventory.Inventory) (node, port string, found bool, err error) {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
all := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
all[n.Name] = true
|
||||
}
|
||||
return artifactStoreOnNetwork(ctx, inv, all)
|
||||
}
|
||||
|
||||
@@ -111,133 +111,6 @@ func TestOnlyAMachineOnThePrivateNetworkIsNamed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0105: the range every address is composed from is the adopted tunnel's, read from
|
||||
// the tunnel the hub holds — never stored anywhere else.
|
||||
func TestTheOverlaysRangeIsTheAdoptedTunnels(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
t.Setenv(OverlayCIDRVar, "10.99.0.0/16")
|
||||
|
||||
before, err := overlayRange(ctx, inv)
|
||||
if err != nil || before != "10.99.0.0/16" {
|
||||
t.Fatalf("without an adopted tunnel the range is not what genesis said: %q %v", before, err)
|
||||
}
|
||||
|
||||
// The hub becomes what genesis makes of a machine in use: adopted, enrolled with the found
|
||||
// tunnel's key, and presenting the tunnel.
|
||||
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const key = "THE-TUNNELS-KEY========================="
|
||||
if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{
|
||||
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
|
||||
Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key,
|
||||
Peers: []inventory.TunnelPeer{{PublicKey: "PEER-TWO", Address: "192.0.2.2"}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
after, err := overlayRange(ctx, inv)
|
||||
if err != nil || after != "192.0.2.0/24" {
|
||||
t.Fatalf("with an adopted tunnel the range is %q (%v), not the tunnel's", after, err)
|
||||
}
|
||||
|
||||
// A placement whose endpoint is on another port than the tunnel's is refused: the peers dial
|
||||
// the tunnel's port.
|
||||
err = overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting", "--hub"})
|
||||
if err == nil || !strings.Contains(err.Error(), "51900") {
|
||||
t.Fatalf("an endpoint off the tunnel's port was accepted: %v", err)
|
||||
}
|
||||
// On the tunnel's port, the hub is placed at the tunnel's address — whatever it had before.
|
||||
if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "hosting", "--hub"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if placed := placementOf(t, ctx, inv, "anchor"); placed.Address != "192.0.2.1" {
|
||||
t.Fatalf("the hub was placed at %s, not the tunnel's own address", placed.Address)
|
||||
}
|
||||
|
||||
// And the hub's declaration carries the peer and the takeover.
|
||||
nodes, computed, err := graph(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var hubNode overlay.Node
|
||||
for _, n := range nodes {
|
||||
if n.Name == "anchor" {
|
||||
hubNode = n
|
||||
}
|
||||
}
|
||||
if hubNode.TakesOver == nil || hubNode.TakesOver.Unit != "wg-quick@wg0" {
|
||||
t.Errorf("the hub is not told to take over the found tunnel: %+v", hubNode)
|
||||
}
|
||||
carried := false
|
||||
for _, p := range computed["anchor"] {
|
||||
if p.Key == "PEER-TWO" && p.Allowed == "192.0.2.2/32" {
|
||||
carried = true
|
||||
}
|
||||
}
|
||||
if !carried {
|
||||
t.Errorf("the hub's peer list does not carry the tunnel's peer: %+v", computed["anchor"])
|
||||
}
|
||||
}
|
||||
|
||||
// A takeover is composed only for a hub whose placement agrees with the tunnel: an address or an
|
||||
// endpoint port that differs would have the host stop the found interface and raise the mesh's
|
||||
// where no peer is listening.
|
||||
func TestATakeoverIsNotComposedForAHubPlacedOffItsTunnel(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const key = "THE-TUNNELS-KEY========================="
|
||||
if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{
|
||||
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
|
||||
Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// aMesh placed anchor at 10.77.0.1 on :51820 — the record of a hub placed before it took the
|
||||
// tunnel over.
|
||||
_, _, err = graph(ctx, open)
|
||||
if err == nil {
|
||||
t.Fatal("a takeover was composed for a hub whose address and port are not the tunnel's")
|
||||
}
|
||||
for _, want := range []string{"10.77.0.1", "192.0.2.1", "51820", "51900", "overlay place anchor"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
// Re-placed on the tunnel, it composes.
|
||||
if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "here", "--hub"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := graph(ctx, open); err != nil {
|
||||
t.Fatalf("re-placed on the tunnel, the graph still refuses: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// theResolver is the catalogue's dnsmasq module as it is, or the test is skipped where the
|
||||
// catalogue is not beside this checkout.
|
||||
func theResolver(t *testing.T) catalogue.Manifest {
|
||||
|
||||
+20
-138
@@ -229,10 +229,28 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
// What that module says a consumer needs to know, with that node's settings on
|
||||
// it: a port somebody moved on the provider is a port its consumers must be told
|
||||
// about, and the two coming from different places is how they come to disagree.
|
||||
serves, err := servedOnNode(ctx, inv, o.node.Name, m, name)
|
||||
assigned, err := portsOn(ctx, inv, o.node.Name, m.Module)
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
}
|
||||
layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module)
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
}
|
||||
// A port that node was given is where its consumers reach it (novox/hq ADR
|
||||
// 0100). Unreadable given ports are that node's refusal to report, not this one's.
|
||||
if given, err := catalogue.GivenPorts(m, layers); err == nil {
|
||||
for wanted, at := range given {
|
||||
assigned[wanted] = at
|
||||
}
|
||||
}
|
||||
serves := catalogue.ServedOn(m, name, assigned)
|
||||
if len(serves) > 0 {
|
||||
serves, err = catalogue.Settle(serves, layers)
|
||||
if err != nil {
|
||||
return catalogue.World{}, err
|
||||
}
|
||||
}
|
||||
offered[name] = append(offered[name], catalogue.Provider{
|
||||
Node: o.node.Name, At: o.node.At, Serves: serves})
|
||||
}
|
||||
@@ -481,22 +499,6 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
|
||||
// The artifact store as this node reaches it now — the address every image and archive the
|
||||
// mesh built is fetched through, composed here and recorded nowhere — with what the mesh has
|
||||
// built, so a reference recorded with an address before that is re-routed too.
|
||||
artifactStore, err := artifactStoreAddress(ctx, inv, shelf, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
held, err := inv.Held(ctx)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
built := make(map[string]bool, len(held))
|
||||
for repository := range held {
|
||||
built[repository] = true
|
||||
}
|
||||
|
||||
// And every machine's name, so a container can reach one. The same set that writes the
|
||||
// machine's own hosts file — one reading, so a container and its machine cannot disagree
|
||||
// about where another machine is.
|
||||
@@ -564,18 +566,11 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
taken[m] = true
|
||||
}
|
||||
}
|
||||
// Where this node put the foundation's servers, for the control plane's own connections
|
||||
// (novox/hq 04-ISSUES/102): read from the node's settings for whatever claims each seat,
|
||||
// exactly as a consumer's binding is, never from what genesis wrote into a secret.
|
||||
seats, err := seatsOn(ctx, inv, shelf, node, plan.Modules, record.Adopted, taken)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
||||
Given: given, Taken: taken,
|
||||
}, record, nil
|
||||
}
|
||||
|
||||
@@ -972,119 +967,6 @@ func managerPublicKeyFor(
|
||||
return inv.SealingKeyOf(ctx, node)
|
||||
}
|
||||
|
||||
// servedOnNode is what a module on a node tells a consumer of one of its provisions, with THAT
|
||||
// node's ports on it: the port the node was given (novox/hq ADR 0100) over the one the mesh
|
||||
// assigned over the manifest's own, settled with the node's settings layers.
|
||||
//
|
||||
// **The one derivation** for every reader of a provider's address — a consumer's binding, the
|
||||
// artifact store's trust and the references composed through it (04-ISSUES/102). Unreadable
|
||||
// given ports are that node's refusal to report, not this reader's.
|
||||
func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string,
|
||||
m catalogue.Manifest, provision string) (map[string]any, error) {
|
||||
ports, layers, err := portsGivenOn(ctx, inv, node, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
serves := catalogue.ServedOn(m, provision, ports)
|
||||
if len(serves) > 0 {
|
||||
serves, err = catalogue.Settle(serves, layers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return serves, nil
|
||||
}
|
||||
|
||||
// portsGivenOn is where a node puts a module's ports — assigned, then given over them — and the
|
||||
// node's settings layers for the module, read once for both.
|
||||
func portsGivenOn(ctx context.Context, inv *inventory.Inventory, node string,
|
||||
m catalogue.Manifest) (map[int]int, []catalogue.Layer, error) {
|
||||
ports, err := portsOn(ctx, inv, node, m.Module)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
layers, err := inv.SettingsFor(ctx, node, m.Module)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if given, err := catalogue.GivenPorts(m, layers); err == nil {
|
||||
for wanted, at := range given {
|
||||
ports[wanted] = at
|
||||
}
|
||||
}
|
||||
return ports, layers, nil
|
||||
}
|
||||
|
||||
// seatsOn is where a node put the holder of each mesh-scoped seat, by seat and by the port the
|
||||
// holder's software uses — what ${seat:…} answers with (novox/hq 04-ISSUES/102).
|
||||
//
|
||||
// Read for every module in the catalogue that claims a seat, in this node's set or not: the store
|
||||
// and the broker are given their ports at genesis, as settings on a module that may be registered
|
||||
// and not yet assigned (04-ISSUES/085), and the control plane must follow that setting from the
|
||||
// first declaration it composes for itself. A holder in this node's set wins over one that is not.
|
||||
func seatsOn(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest,
|
||||
node string, inSet []catalogue.Manifest, adopted bool, taken map[string]bool) (map[string]map[int]int, error) {
|
||||
assigned := map[string]bool{}
|
||||
for _, m := range inSet {
|
||||
assigned[m.Module] = true
|
||||
}
|
||||
names := make([]string, 0, len(shelf))
|
||||
for name := range shelf {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
seats := map[string]map[int]int{}
|
||||
for _, name := range names {
|
||||
m := shelf[name]
|
||||
var claims []string
|
||||
for _, c := range m.Claims {
|
||||
if c.At() == catalogue.ScopeMesh {
|
||||
claims = append(claims, c.Name)
|
||||
}
|
||||
}
|
||||
if len(claims) == 0 {
|
||||
continue
|
||||
}
|
||||
// **Only a port the node was given or the mesh assigned — never the manifest's own
|
||||
// number.** The sealed value the answer sits beside carries the port genesis wrote, which
|
||||
// on a given-port node is the predecessor's; a manifest's long-form mapping is the
|
||||
// catalogue's default, and answering with it would override the right number with one
|
||||
// the mesh never checked (the contract in seat_into.go). And on an adopted node a holder
|
||||
// assigned but not yet taken is the found container, on the ports it was found with, not
|
||||
// the declaration's — so its mesh-assigned ports do not count there either; a given port
|
||||
// does, because a given port is the found one by construction (ADR 0100).
|
||||
ports, _, err := portsGivenOn(ctx, inv, node, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if adopted && !taken[name] {
|
||||
layers, err := inv.SettingsFor(ctx, node, m.Module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ports = map[int]int{}
|
||||
if given, err := catalogue.GivenPorts(m, layers); err == nil {
|
||||
ports = given
|
||||
}
|
||||
}
|
||||
if len(ports) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, seat := range claims {
|
||||
if seats[seat] == nil {
|
||||
seats[seat] = map[int]int{}
|
||||
}
|
||||
for wanted, at := range ports {
|
||||
if _, said := seats[seat][wanted]; said && !assigned[name] {
|
||||
continue
|
||||
}
|
||||
seats[seat][wanted] = at
|
||||
}
|
||||
}
|
||||
}
|
||||
return seats, nil
|
||||
}
|
||||
|
||||
// portsOn is one module's assignments on one machine, by the port the software uses.
|
||||
func portsOn(
|
||||
ctx context.Context, inv *inventory.Inventory, node, module string,
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What a build is recorded as, and what it is announced and handed on as.
|
||||
//
|
||||
// **Recorded by what it is; routed where it is used** (novox/hq 04-ISSUES/102). The builder
|
||||
// announces each artifact by the reference it pushed — `<registry>:<port>/<module>/<artifact>@sha256:…`
|
||||
// — and the manifest with those references in it. The mesh records the digest and the path
|
||||
// (catalogue.Recorded) and composes the store's address back in wherever a machine or a builder
|
||||
// needs a reference it can fetch: a declaration, a replay to the catalogue, the bases a build is
|
||||
// given. Nothing recorded carries a port, so moving the store is a settings change and not a
|
||||
// rebuild of everything the mesh has ever built.
|
||||
|
||||
// recordedManifest is a build's manifest with the store's address taken off every reference the
|
||||
// build itself made. Other references — an image a module runs from a public registry — are what
|
||||
// they were, which is why this rewrites only what `made` names rather than everything that looks
|
||||
// like an address.
|
||||
func recordedManifest(raw json.RawMessage, made []inventory.Artifact) json.RawMessage {
|
||||
announced := map[string]bool{}
|
||||
for _, a := range made {
|
||||
announced[a.Reference] = true
|
||||
}
|
||||
return withReferences(raw, func(ref string) (string, bool) {
|
||||
if !announced[ref] {
|
||||
return ref, false
|
||||
}
|
||||
return catalogue.Recorded(ref), true
|
||||
})
|
||||
}
|
||||
|
||||
// routedManifest is a recorded manifest with the store's address, as this network reaches it now,
|
||||
// composed into every reference the build made — recorded either way, before or after references
|
||||
// were kept without their address.
|
||||
func routedManifest(raw json.RawMessage, made []inventory.Artifact, address string) json.RawMessage {
|
||||
recorded := map[string]bool{}
|
||||
for _, a := range made {
|
||||
recorded[catalogue.Recorded(a.Reference)] = true
|
||||
}
|
||||
return withReferences(raw, func(ref string) (string, bool) {
|
||||
if !recorded[catalogue.Recorded(ref)] {
|
||||
return ref, false
|
||||
}
|
||||
return catalogue.Rerouted(ref, address), true
|
||||
})
|
||||
}
|
||||
|
||||
// withReferences applies `rewrite` to each resource's `image` and `source`, and hands the manifest
|
||||
// back untouched — byte for byte — when nothing changed or it could not be read: what a manifest
|
||||
// is, is the parser's to say, and it says so with a better sentence than anything here would.
|
||||
func withReferences(raw json.RawMessage, rewrite func(string) (string, bool)) json.RawMessage {
|
||||
if len(raw) == 0 {
|
||||
return raw
|
||||
}
|
||||
var manifest map[string]any
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return raw
|
||||
}
|
||||
resources, _ := manifest["resources"].([]any)
|
||||
changed := false
|
||||
for _, r := range resources {
|
||||
resource, ok := r.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, key := range []string{"image", "source"} {
|
||||
if written, ok := resource[key].(string); ok {
|
||||
if rewritten, did := rewrite(written); did && rewritten != written {
|
||||
resource[key] = rewritten
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return raw
|
||||
}
|
||||
out, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// routedArtifacts is a build's artifacts as something can fetch them now.
|
||||
func routedArtifacts(made []inventory.Artifact, address string) []inventory.Artifact {
|
||||
if address == "" {
|
||||
return made
|
||||
}
|
||||
out := make([]inventory.Artifact, 0, len(made))
|
||||
for _, a := range made {
|
||||
out = append(out, inventory.Artifact{Name: a.Name, Kind: a.Kind,
|
||||
Reference: catalogue.Rerouted(a.Reference, address)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// whereTheStoreIs is the artifact store's address as something on `forNode` reaches it, or "" —
|
||||
// read for a caller that has the inventory open and nothing else in hand. With no node named, the
|
||||
// store's own node: loopback when nothing is on the network yet.
|
||||
func whereTheStoreIs(ctx context.Context, inv *inventory.Inventory, forNode string) (string, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if forNode == "" {
|
||||
if holder, _, found, err := artifactStoreHolder(ctx, inv); err != nil {
|
||||
return "", err
|
||||
} else if found {
|
||||
forNode = holder
|
||||
}
|
||||
}
|
||||
return artifactStoreAddress(ctx, inv, shelf, forNode)
|
||||
}
|
||||
|
||||
// whereABuilderReachesTheStore is the store's address for the machine that builds: the network's
|
||||
// when there is one, else loopback on the store's own node — when that node also holds a module
|
||||
// requiring the store, which is what a builder is (genesis: one node holds both).
|
||||
func whereABuilderReachesTheStore(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
holder, _, found, err := artifactStoreHolder(ctx, inv)
|
||||
if err != nil || !found {
|
||||
return "", err
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, holder)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
besideIt := false
|
||||
for _, a := range assigned {
|
||||
for _, r := range shelf[a].Requires {
|
||||
if r == catalogue.ArtifactStoreProvision {
|
||||
besideIt = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !besideIt {
|
||||
holder = ""
|
||||
}
|
||||
return artifactStoreAddress(ctx, inv, shelf, holder)
|
||||
}
|
||||
@@ -173,21 +173,11 @@ func sayUpgrade(module string, u inventory.Upgrade) string {
|
||||
// catalogue misses nothing — but the modules built before it first ran were announced to a queue
|
||||
// that did not exist, and on a fresh mesh those are always the same three: the shared base, the
|
||||
// store the catalogue runs on, and the catalogue itself.
|
||||
//
|
||||
// **Announced as fetchable, recorded as what it is** (novox/hq 04-ISSUES/102). A build is
|
||||
// recorded by digest and path; the catalogue hears the builder's own announcements, which name
|
||||
// the store's address, so a replay composes the address back in — the store's address as the
|
||||
// network reaches it NOW, which is the whole point of not having recorded the old one. With no
|
||||
// store on the network yet, the recorded form goes as it is.
|
||||
func (f following) Announceable(ctx context.Context) ([]link.Announcement, error) {
|
||||
builds, err := f.open.inventory.Announceable(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := whereTheStoreIs(ctx, f.open.inventory, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]link.Announcement, 0, len(builds))
|
||||
for _, b := range builds {
|
||||
a := link.Announcement{
|
||||
@@ -196,11 +186,8 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
|
||||
}
|
||||
if len(b.Manifest) > 0 {
|
||||
a.Manifest = b.Manifest
|
||||
if address != "" {
|
||||
a.Manifest = routedManifest(b.Manifest, b.Made, address)
|
||||
}
|
||||
}
|
||||
for _, made := range routedArtifacts(b.Made, address) {
|
||||
for _, made := range b.Made {
|
||||
a.Made = append(a.Made, link.MadeArtifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||
})
|
||||
|
||||
@@ -40,12 +40,8 @@ type Broker struct {
|
||||
var ErrNotConfigured = errors.New("this control plane has not been told about its broker")
|
||||
|
||||
// FromEnvironment reads the two settings, if they are there.
|
||||
//
|
||||
// The address's port follows MESH_BROKER_ADDRESS_PORT when the node's settings moved the bus
|
||||
// (novox/hq 04-ISSUES/102): the address genesis wrote is a public name and the port genesis
|
||||
// chose, and only the port is the node's to move.
|
||||
func FromEnvironment() (Broker, error) {
|
||||
address, err := envfile.Placed(AddressVar)
|
||||
address, err := envfile.Value(AddressVar)
|
||||
if err != nil {
|
||||
return Broker{}, err
|
||||
}
|
||||
|
||||
@@ -178,32 +178,3 @@ func TestBothTogetherGiveABroker(t *testing.T) {
|
||||
t.Errorf("got %+v", known)
|
||||
}
|
||||
}
|
||||
|
||||
// The node moved the bus, and the address a token carries follows (novox/hq 04-ISSUES/102).
|
||||
func TestTheAddressPortFollowsThePortTwin(t *testing.T) {
|
||||
t.Setenv(AddressVar, "broker.example:5671")
|
||||
t.Setenv(AddressVar+"_FILE", "")
|
||||
path, _ := writeCertificate(t)
|
||||
t.Setenv(CertificateVar, path)
|
||||
t.Setenv(AddressVar+"_PORT", "5679")
|
||||
b, err := FromEnvironment()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b.Address != "broker.example:5679" {
|
||||
t.Fatalf("the address is %q; the node put the bus on 5679", b.Address)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheManagementPortFollowsThePortTwin(t *testing.T) {
|
||||
t.Setenv(ManagementVar, "http://guest:guest@127.0.0.1:15672")
|
||||
t.Setenv(ManagementVar+"_FILE", "")
|
||||
t.Setenv(ManagementVar+"_PORT", "15673")
|
||||
m, err := ManagementFromEnvironment()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.base.Host != "127.0.0.1:15673" {
|
||||
t.Fatalf("the management API is at %q; the node put it on 15673", m.base.Host)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,11 +41,8 @@ type Management struct {
|
||||
}
|
||||
|
||||
// ManagementFromEnvironment reads where the management API is, if it is configured.
|
||||
//
|
||||
// On the port MESH_BROKER_MANAGEMENT_PORT names when the node moved it (novox/hq 04-ISSUES/102);
|
||||
// the URL's own port otherwise.
|
||||
func ManagementFromEnvironment() (*Management, error) {
|
||||
raw, err := envfile.Placed(ManagementVar)
|
||||
raw, err := envfile.Value(ManagementVar)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The artifact store's seat is one per mesh, read from the catalogue beside this checkout.
|
||||
//
|
||||
// **A second store anywhere is refused by name, not discovered as a consumer failure.** The seat
|
||||
// was node-scoped, so a second `distribution` on another machine resolved cleanly there — and a
|
||||
// node-scoped requirement with one candidate installs that candidate on the node, so anything that
|
||||
// required the store's presence beside it would have raised a fresh, empty store on the wrong
|
||||
// machine. Only afterwards did the mesh notice: `artifact-store` offered by two nodes, and every
|
||||
// consumer elsewhere refusing to choose. The claim says it first, where the second store is
|
||||
// assigned.
|
||||
func TestASecondArtifactStoreAnywhereIsRefusedByName(t *testing.T) {
|
||||
store := catalogueManifest(t, "distribution")
|
||||
|
||||
// The first store resolves as it always has.
|
||||
if _, err := Resolve(shelf(store), []string{"distribution"}, workstation(), World{}); err != nil {
|
||||
t.Fatalf("the store alone does not resolve: %v", err)
|
||||
}
|
||||
|
||||
// A second one, on any other machine, is refused — and the refusal names the seat.
|
||||
elsewhere := World{Held: []Held{{Claim: "the-artifact-store", Scope: ScopeMesh,
|
||||
Node: "anchor", Module: "distribution"}}}
|
||||
other := workstation()
|
||||
other.Name = "laptop"
|
||||
_, err := Resolve(shelf(store), []string{"distribution"}, other, elsewhere)
|
||||
if err == nil {
|
||||
t.Fatal("a second store was accepted on another machine; it would offer artifact-store a " +
|
||||
"second time and every consumer elsewhere would refuse to choose")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "the-artifact-store") || !strings.Contains(err.Error(), "one per mesh") {
|
||||
t.Fatalf("refused without naming the seat: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What the mesh built, named by what it is rather than by where it was pushed.
|
||||
//
|
||||
// **An image is recorded by its digest and its path; the registry's address is a route to it**
|
||||
// (novox/hq 04-ISSUES/102). A build used to be recorded as `<registry>:<port>/<module>/<artifact>@sha256:…`
|
||||
// — the reference the builder pushed to, kept whole — and every declaration carried that literal.
|
||||
// Move the registry's port, or the registry, and every fresh pull of a mesh image fails: a new
|
||||
// node, a recreate after eviction. The digest is the identity; the address is the node's setting
|
||||
// for the module that serves the artifact store, and it is read from there when a reference is
|
||||
// composed, never written into a record.
|
||||
//
|
||||
// So a kept reference has a scheme of the mesh's own, named after the provision that answers it:
|
||||
//
|
||||
// artifact-store://<module>/<artifact>@sha256:<hex> an image
|
||||
// artifact-store://<module>/<artifact>/blobs/sha256:<hex> an archive
|
||||
//
|
||||
// No runtime knows the scheme. That is the point of it being one: a reference that leaks to a
|
||||
// machine uncomposed is refused by the runtime as malformed, in front of whoever sent it, rather
|
||||
// than pulled from a public registry that happens to have a repository by that name — which is
|
||||
// what an address-less `<module>/<artifact>@sha256:…` would be.
|
||||
|
||||
// ArtifactStoreScheme marks a reference to something in the mesh's artifact store, kept without
|
||||
// the store's address.
|
||||
const ArtifactStoreScheme = ArtifactStoreProvision + "://"
|
||||
|
||||
// Recorded is a reference as the mesh records it: the artifact store's address, if the builder wrote
|
||||
// one, taken off.
|
||||
//
|
||||
// For a reference the builder announced — one it pushed to the store — which is the only kind
|
||||
// this is called on. An image the builder named `<host>/<path>@sha256:…` is kept as its path; an
|
||||
// archive it named `http://<host>/v2/<path>/blobs/<digest>` likewise. A reference with no address
|
||||
// in it — an image id from a genesis build that had nowhere to publish, a package version — is
|
||||
// what it was.
|
||||
func Recorded(reference string) string {
|
||||
if strings.HasPrefix(reference, ArtifactStoreScheme) {
|
||||
return reference
|
||||
}
|
||||
if rest, isURL := strings.CutPrefix(reference, "http://"); isURL {
|
||||
if _, path, ok := strings.Cut(rest, "/v2/"); ok && strings.Contains(path, "/blobs/") {
|
||||
return ArtifactStoreScheme + path
|
||||
}
|
||||
return reference
|
||||
}
|
||||
host, path, ok := strings.Cut(reference, "/")
|
||||
if !ok || !isRegistryHost(host) || !strings.Contains(path, "@sha256:") {
|
||||
return reference
|
||||
}
|
||||
return ArtifactStoreScheme + path
|
||||
}
|
||||
|
||||
// isRegistryHost is the runtime's own rule for reading the first component of a reference as a
|
||||
// registry rather than as a namespace: it has a dot or a port in it, or it is localhost.
|
||||
func isRegistryHost(component string) bool {
|
||||
return component == "localhost" || strings.ContainsAny(component, ".:")
|
||||
}
|
||||
|
||||
// InArtifactStore reports whether a reference is a kept one, and what it names there.
|
||||
func InArtifactStore(reference string) (path string, kept bool) {
|
||||
return strings.CutPrefix(reference, ArtifactStoreScheme)
|
||||
}
|
||||
|
||||
// Routed is a kept reference as a machine fetches it, through the artifact store at `address`
|
||||
// (host:port). A reference that is not a kept one is what it was.
|
||||
func Routed(reference, address string) string {
|
||||
path, kept := InArtifactStore(reference)
|
||||
if !kept {
|
||||
return reference
|
||||
}
|
||||
if strings.Contains(path, "/blobs/") {
|
||||
return "http://" + address + "/v2/" + path
|
||||
}
|
||||
return address + "/" + path
|
||||
}
|
||||
|
||||
// Rerouted is a reference the mesh recorded, whichever way it was recorded, as a machine fetches
|
||||
// it now: a kept one composed with the store's address, and one recorded before references were
|
||||
// kept without their address — the builder's own `<host>/<path>@sha256:…` — re-routed to where
|
||||
// the store is now. Only for references that are the mesh's own: everything a build record
|
||||
// holds is, by construction.
|
||||
func Rerouted(reference, address string) string {
|
||||
return Routed(Recorded(reference), address)
|
||||
}
|
||||
|
||||
// artifactsInto composes the artifact store's address into a resource's `image` and `source`.
|
||||
//
|
||||
// **Composed here, at the last moment before a machine, and stored nowhere.** A kept reference is
|
||||
// routed through the store as this network reaches it now. A reference recorded with an address
|
||||
// before references were kept without one is re-routed the same way — but only when the mesh
|
||||
// built it (`with.Built` names every `<module>/<artifact>` it has), because a module may run an
|
||||
// image from a public registry under its own name and that one is exactly where it says.
|
||||
//
|
||||
// A kept reference with no store to route it through is refused: sent as it is, the runtime would
|
||||
// refuse the scheme on the machine, one push away from the reason.
|
||||
//
|
||||
// **What this does not reach: the images genesis pinned.** The installer builds the control plane
|
||||
// and the builder before the mesh exists, pushes them itself and pins their manifests to
|
||||
// `<registry>:<port>/mesh-controller@…` and `<registry>:<port>/mesh-builder@…` — single-segment
|
||||
// repositories with no build record, so `with.Built` does not name them and they are left as
|
||||
// written until each is rebuilt through the mesh, which records it by digest and path. Until then
|
||||
// a registry that moves strands exactly those two on a recreate, and the control plane's is the
|
||||
// one that cannot be repaired through the mesh. Rebuild both through `build` before moving the
|
||||
// store (novox/hq 04-ISSUES/102, finding F4).
|
||||
func artifactsInto(resource map[string]any, module string, with Rendering) error {
|
||||
for _, key := range []string{"image", "source"} {
|
||||
written, ok := resource[key].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, kept := InArtifactStore(written); kept {
|
||||
if with.ArtifactStore == "" {
|
||||
return fmt.Errorf(
|
||||
"%s's %v names %s, which is in the mesh's artifact store, and this mesh has no "+
|
||||
"artifact store on its network to fetch it from — nothing assigned offers "+
|
||||
"%s, or the machine offering it is not on the private network",
|
||||
module, resource["id"], written, ArtifactStoreProvision)
|
||||
}
|
||||
resource[key] = Routed(written, with.ArtifactStore)
|
||||
continue
|
||||
}
|
||||
if with.ArtifactStore == "" {
|
||||
continue
|
||||
}
|
||||
recorded := Recorded(written)
|
||||
if recorded == written {
|
||||
continue
|
||||
}
|
||||
path, _ := InArtifactStore(recorded)
|
||||
repository := path
|
||||
if at := strings.IndexAny(path, "@"); at >= 0 {
|
||||
repository = path[:at]
|
||||
} else if blobs := strings.Index(path, "/blobs/"); blobs >= 0 {
|
||||
repository = path[:blobs]
|
||||
}
|
||||
if with.Built[repository] {
|
||||
resource[key] = Routed(recorded, with.ArtifactStore)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A build is recorded by what it is; where it is pushed is composed where it is used (novox/hq
|
||||
// 04-ISSUES/102).
|
||||
|
||||
var digest = "sha256:" + strings.Repeat("d", 64)
|
||||
|
||||
func TestAReferenceIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"anchor.internal:5100/gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest,
|
||||
"localhost:5000/gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest,
|
||||
"http://anchor.internal:5100/v2/gitea/config/blobs/" + digest: ArtifactStoreScheme + "gitea/config/blobs/" + digest,
|
||||
ArtifactStoreScheme + "gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest,
|
||||
digest: digest,
|
||||
"@novox/sdk@1.2.3": "@novox/sdk@1.2.3",
|
||||
"gitea/gitea@" + digest: "gitea/gitea@" + digest,
|
||||
"https://registry.example/v2/gitea/config/blobs/" + digest: "https://registry.example/v2/gitea/config/blobs/" + digest,
|
||||
}
|
||||
for announced, want := range cases {
|
||||
if got := Recorded(announced); got != want {
|
||||
t.Errorf("Recorded(%q) = %q, want %q", announced, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestARecordedReferenceIsRoutedThroughTheStoreAsItIsNow(t *testing.T) {
|
||||
if got := Routed(ArtifactStoreScheme+"gitea/server@"+digest, "anchor.internal:5101"); got != "anchor.internal:5101/gitea/server@"+digest {
|
||||
t.Errorf("an image is fetched as %q", got)
|
||||
}
|
||||
if got := Routed(ArtifactStoreScheme+"gitea/config/blobs/"+digest, "anchor.internal:5101"); got != "http://anchor.internal:5101/v2/gitea/config/blobs/"+digest {
|
||||
t.Errorf("an archive is fetched as %q", got)
|
||||
}
|
||||
if got := Routed("gitea/gitea@"+digest, "anchor.internal:5101"); got != "gitea/gitea@"+digest {
|
||||
t.Errorf("a reference that is not the store's was routed: %q", got)
|
||||
}
|
||||
// One recorded before references were kept without their address follows the store too.
|
||||
if got := Rerouted("anchor.internal:5100/gitea/server@"+digest, "anchor.internal:5101"); got != "anchor.internal:5101/gitea/server@"+digest {
|
||||
t.Errorf("a reference recorded with the old address stays there: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The address is composed into a declaration, and the record never carries it.**
|
||||
func TestAnImageTheMeshBuiltIsRoutedThroughTheStoreWhenDeclared(t *testing.T) {
|
||||
m := Manifest{Module: "gitea", Version: "1", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-gitea", "artifact": "server"},
|
||||
{"id": "config", "type": "archive", "path": "/etc/gitea", "artifact": "config"},
|
||||
{"id": "cache", "type": "container", "name": "mesh-gitea-cache", "image": "valkey/valkey@" + digest},
|
||||
}, Build: &Build{Artifacts: []Artifact{
|
||||
{Name: "server", Kind: ArtifactImage, From: "Dockerfile"},
|
||||
{Name: "config", Kind: ArtifactArchive, From: "config"},
|
||||
}}}
|
||||
resolved, err := m.Resolve([]Built{
|
||||
{Name: "server", Kind: ArtifactImage, Reference: ArtifactStoreScheme + "gitea/server@" + digest},
|
||||
{Name: "config", Kind: ArtifactArchive, Reference: ArtifactStoreScheme + "gitea/config/blobs/" + digest, Digest: digest},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{resolved}}
|
||||
|
||||
out, err := r.Declaration(Rendering{ArtifactStore: "anchor.internal:5101"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "gitea.server")["image"]; got != "anchor.internal:5101/gitea/server@"+digest {
|
||||
t.Errorf("the image the mesh built is fetched as %v", got)
|
||||
}
|
||||
if got := fileNamed(out, "gitea.config")["source"]; got != "http://anchor.internal:5101/v2/gitea/config/blobs/"+digest {
|
||||
t.Errorf("the archive the mesh built is fetched from %v", got)
|
||||
}
|
||||
if got := fileNamed(out, "gitea.cache")["image"]; got != "valkey/valkey@"+digest {
|
||||
t.Errorf("an image from a public registry was routed through the store: %v", got)
|
||||
}
|
||||
// The manifest the mesh holds still says what it is, not where it was fetched from.
|
||||
if got := resolved.Resources[0]["image"]; got != ArtifactStoreScheme+"gitea/server@"+digest {
|
||||
t.Errorf("composing wrote the address into the catalogue's copy: %v", got)
|
||||
}
|
||||
|
||||
// And the store moves: the same record, another address, without a rebuild.
|
||||
out, err = r.Declaration(Rendering{ArtifactStore: "laptop.internal:5000"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "gitea.server")["image"]; got != "laptop.internal:5000/gitea/server@"+digest {
|
||||
t.Errorf("after the store moved, the image is still fetched as %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnImageInTheStoreWithNoStoreToFetchItFromIsRefused(t *testing.T) {
|
||||
m := Manifest{Module: "gitea", Version: "1", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-gitea",
|
||||
"image": ArtifactStoreScheme + "gitea/server@" + digest},
|
||||
}}
|
||||
_, err := Resolution{Node: "anchor", Modules: []Manifest{m}}.Declaration(Rendering{})
|
||||
if err == nil || !strings.Contains(err.Error(), "no artifact store") {
|
||||
t.Fatalf("a reference nothing can fetch was sent to a machine: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A reference recorded with an address before this follows the store too** — when the mesh
|
||||
// built it. A module running an image straight from a public registry under its own name is left
|
||||
// exactly where it says: `quay.io/keycloak/keycloak` is not the mesh's, whatever it is called.
|
||||
func TestAReferenceRecordedWithAnAddressFollowsTheStoreWhenTheMeshBuiltIt(t *testing.T) {
|
||||
m := Manifest{Module: "keycloak", Version: "1", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-keycloak",
|
||||
"image": "anchor.internal:5100/keycloak/server@" + digest},
|
||||
{"id": "upstream", "type": "container", "name": "mesh-keycloak-upstream",
|
||||
"image": "quay.io/keycloak/keycloak@" + digest},
|
||||
}}
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{m}}
|
||||
out, err := r.Declaration(Rendering{
|
||||
ArtifactStore: "anchor.internal:5101",
|
||||
Built: map[string]bool{"keycloak/server": true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "keycloak.server")["image"]; got != "anchor.internal:5101/keycloak/server@"+digest {
|
||||
t.Errorf("an image the mesh built, recorded with the old address, is fetched as %v", got)
|
||||
}
|
||||
if got := fileNamed(out, "keycloak.upstream")["image"]; got != "quay.io/keycloak/keycloak@"+digest {
|
||||
t.Errorf("a public image was re-routed through the store: %v", got)
|
||||
}
|
||||
// Nothing known to be built: nothing re-routed, nothing refused.
|
||||
out, err = r.Declaration(Rendering{ArtifactStore: "anchor.internal:5101"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "keycloak.server")["image"]; got != "anchor.internal:5100/keycloak/server@"+digest {
|
||||
t.Errorf("with no build record, a reference was rewritten: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -143,23 +143,6 @@ type Rendering struct {
|
||||
// from these only (ADR 0103): a port of a module assigned but not taken may still be the
|
||||
// predecessor's.
|
||||
Taken map[string]bool
|
||||
|
||||
// Seats is where this machine put each mesh-scoped seat's holder, by seat and by the port the
|
||||
// holder's software uses (novox/hq 04-ISSUES/102) — read from the node's settings and
|
||||
// assignments for whichever module claims the seat, whether or not it is in this node's set.
|
||||
// What ${seat:…} answers with; see seat_into.go for why the answer may be absent.
|
||||
Seats map[string]map[int]int
|
||||
|
||||
// ArtifactStore is the mesh's artifact store as this network reaches it (host:port) — the
|
||||
// node holding it and the port that node put it on — or empty when the mesh has none on its
|
||||
// network yet. Composed into every image and archive the mesh built, at this moment and never
|
||||
// stored (novox/hq 04-ISSUES/102).
|
||||
ArtifactStore string
|
||||
|
||||
// Built is every `<module>/<artifact>` the mesh has built. What tells a reference recorded
|
||||
// with an address — before references were kept without one — from an image a module runs
|
||||
// straight from a public registry.
|
||||
Built map[string]bool
|
||||
}
|
||||
|
||||
// machinePort is where a module's port lives on this machine, or the port itself when the mesh has
|
||||
@@ -556,11 +539,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
|
||||
if err := portInto(copied, m.Module, m.Listens, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And where this machine put the foundation's servers, for the one module that
|
||||
// reaches them by seat rather than by binding (novox/hq 04-ISSUES/102).
|
||||
if err := seatInto(copied, m.Module, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := machineInto(copied, thisMachine, m.Module); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -572,11 +550,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
|
||||
if err := built(copied, m.Module); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// What the mesh built is kept by digest and path; the store's address is this
|
||||
// network's now, composed here and never recorded (novox/hq 04-ISSUES/102).
|
||||
if err := artifactsInto(copied, m.Module, with); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
publishedOn(copied, m.Module, with)
|
||||
copied["id"] = m.Module + "." + fmt.Sprint(resource["id"])
|
||||
// A service saying what it reflects names resources within its own module, so those
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Telling a module where this machine put the holder of a seat.
|
||||
//
|
||||
// **The foundation's ports are the node's** (novox/hq ADR 0100): the port a foundation server was
|
||||
// given at genesis becomes that node's setting for the module that serves it, and every reader
|
||||
// follows the setting. Every consumer's binding did. The control plane's own connections did not
|
||||
// (04-ISSUES/102): they are written at genesis, before any module exists to bind to — full
|
||||
// connection strings, sealed, with the port inside — so when the node moved the store, the
|
||||
// control plane went on dialling where genesis had written and the mesh was headless.
|
||||
//
|
||||
// The control plane cannot open its own sealed connection to move the port, and it cannot bind
|
||||
// the store as a consumer would: a binding mints a credential, and what the control plane holds
|
||||
// is the foundation's superuser, made before the mesh. What it can do is read the node's settings
|
||||
// when it composes its own declaration — it is the thing that composes every other module's — and
|
||||
// say in its own environment which port this machine put the store at.
|
||||
//
|
||||
// So a module may ask about a **seat** (ADR 0079: a foundation seat is named after the server it
|
||||
// guards — `mesh-store`, `mesh-broker`). `${seat:mesh-store:5432}` is "the port this machine put
|
||||
// the holder of the mesh-store seat's 5432 at". Not a provision: nothing is required, nothing is
|
||||
// granted, no credential is minted. A seat is the mesh's own vocabulary for the store and the
|
||||
// broker, which is what makes this the control plane's way of naming them and not a way for a
|
||||
// module to reach a server it was not granted — the answer is a port number the mesh holds in the
|
||||
// clear, and the credential to use it is still the module's own to have.
|
||||
//
|
||||
// **The answer may be empty, and that is the one place a placeholder answers with nothing.** The
|
||||
// store and the broker are raised at genesis, before the mesh knows them as modules; a mesh raised
|
||||
// on the catalogue's own ports never gives them a setting at all. In both, the port genesis wrote
|
||||
// into the connection string is the right one, and the mesh has nothing to add. An empty answer
|
||||
// says exactly that, and what reads it — the control plane's `_PORT` twin — treats an empty value
|
||||
// as no value. Answering with the software's own port instead would override what genesis wrote
|
||||
// with a number the mesh never checked, on the one machine where that is a headless mesh.
|
||||
|
||||
// ofSeat is where a module asks about a seat: ${seat:<seat>:<the port its holder's software uses>}.
|
||||
var ofSeat = regexp.MustCompile(`\$\{seat:([a-z0-9][a-z0-9-]*):([0-9]+)\}`)
|
||||
|
||||
// seatInto replaces a resource's ${seat:…} placeholders with where this machine put each seat's
|
||||
// holder — in a file's content, and in a value of a container's environment. The same two places
|
||||
// portInto fills, for the same reason: they are where a process reads a number from.
|
||||
func seatInto(resource map[string]any, module string, with Rendering) error {
|
||||
switch fmt.Sprint(resource["type"]) {
|
||||
case "file":
|
||||
content, ok := resource["content"].(string)
|
||||
if !ok || !ofSeat.MatchString(content) {
|
||||
return nil
|
||||
}
|
||||
filled, err := seatsFilledInto(content, fmt.Sprintf("%s has a file that", module), with)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resource["content"] = filled
|
||||
|
||||
case "container":
|
||||
env, ok := resource["env"].(map[string]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
named := make([]string, 0, len(env))
|
||||
for key := range env {
|
||||
named = append(named, key)
|
||||
}
|
||||
sort.Strings(named)
|
||||
|
||||
// A fresh map, and only when something changes — this map is the catalogue's, shared by
|
||||
// every node running the module (see portInto).
|
||||
var filled map[string]any
|
||||
for _, key := range named {
|
||||
written, ok := env[key].(string)
|
||||
if !ok || !ofSeat.MatchString(written) {
|
||||
continue
|
||||
}
|
||||
value, err := seatsFilledInto(written,
|
||||
fmt.Sprintf("%s's container %s sets %s to something that",
|
||||
module, resource["name"], key), with)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if filled == nil {
|
||||
filled = map[string]any{}
|
||||
for k, v := range env {
|
||||
filled[k] = v
|
||||
}
|
||||
}
|
||||
filled[key] = value
|
||||
}
|
||||
if filled != nil {
|
||||
resource["env"] = filled
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// seatsFilledInto answers every ${seat:…} in one written value.
|
||||
//
|
||||
// A port the seat's holder does not publish on this machine — or a seat nothing on it holds —
|
||||
// answers with nothing, for the reason the package comment gives. A port that is not one is
|
||||
// refused: it was written by a person and it is wrong.
|
||||
func seatsFilledInto(written, where string, with Rendering) (string, error) {
|
||||
for _, m := range ofSeat.FindAllStringSubmatch(written, -1) {
|
||||
seat, port := m[1], m[2]
|
||||
wanted, err := strconv.Atoi(port)
|
||||
if err != nil || wanted < 1 || wanted > 65535 {
|
||||
return "", fmt.Errorf("%s says ${seat:%s:%s}, and %s is not a port", where, seat, port, port)
|
||||
}
|
||||
answer := ""
|
||||
if at, known := with.Seats[seat][wanted]; known {
|
||||
answer = strconv.Itoa(at)
|
||||
}
|
||||
written = strings.ReplaceAll(written, m[0], answer)
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
@@ -1,216 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The control plane's own addresses follow the node's ports (novox/hq 04-ISSUES/102).
|
||||
|
||||
func TestASeatPlaceholderAnswersWhereThisMachinePutTheHolder(t *testing.T) {
|
||||
control := map[string]any{
|
||||
"type": "container", "id": "server", "name": "mesh-controller",
|
||||
"env": map[string]any{
|
||||
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
|
||||
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}",
|
||||
"MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory",
|
||||
},
|
||||
}
|
||||
with := Rendering{Seats: map[string]map[int]int{
|
||||
"mesh-store": {5432: 6852}, "mesh-broker": {5672: 5679, 5671: 5671},
|
||||
}}
|
||||
if err := seatInto(control, "mesh-controller", with); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := control["env"].(map[string]any)
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
"MESH_BROKER_AMQP_PORT": "5679",
|
||||
"MESH_BROKER_ADDRESS_PORT": "5671",
|
||||
"MESH_STORE_INVENTORY_FILE": "/run/secrets/inventory",
|
||||
} {
|
||||
if env[key] != want {
|
||||
t.Errorf("%s = %v, want %q", key, env[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A seat nothing on this machine holds — or one whose holder the mesh has given no port — answers
|
||||
// with nothing, so the port genesis wrote into the connection string stands. Not the software's
|
||||
// own port: on a node given a port at genesis before the store's module exists, that would
|
||||
// override the right number with the catalogue's.
|
||||
func TestASeatTheMeshCannotPlaceAnswersWithNothing(t *testing.T) {
|
||||
control := map[string]any{
|
||||
"type": "container", "id": "server", "name": "mesh-controller",
|
||||
"env": map[string]any{"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}"},
|
||||
}
|
||||
if err := seatInto(control, "mesh-controller", Rendering{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := control["env"].(map[string]any)["MESH_STORE_INVENTORY_PORT"]; got != "" {
|
||||
t.Fatalf("with nothing known, the seat answered %q", got)
|
||||
}
|
||||
file := map[string]any{"type": "file", "content": "port=${seat:mesh-store:5432}\n"}
|
||||
if err := seatInto(file, "x", Rendering{Seats: map[string]map[int]int{"mesh-store": {5433: 1}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := file["content"]; got != "port=\n" {
|
||||
t.Fatalf("a port the holder does not publish answered %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASeatPlaceholderNamingNoPortIsRefused(t *testing.T) {
|
||||
file := map[string]any{"type": "file", "content": "${seat:mesh-store:99999}"}
|
||||
if err := seatInto(file, "x", Rendering{}); err == nil {
|
||||
t.Fatal("99999 was accepted as a port")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFillingASeatLeavesTheManifestAlone(t *testing.T) {
|
||||
env := map[string]any{"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}"}
|
||||
manifest := map[string]any{"type": "container", "id": "server", "env": env}
|
||||
for _, at := range []int{6852, 5432} {
|
||||
copied := map[string]any{}
|
||||
for k, v := range manifest {
|
||||
copied[k] = v
|
||||
}
|
||||
with := Rendering{Seats: map[string]map[int]int{"mesh-store": {5432: at}}}
|
||||
if err := seatInto(copied, "mesh-controller", with); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if env["MESH_STORE_INVENTORY_PORT"] != "${seat:mesh-store:5432}" {
|
||||
t.Fatalf("the module's own manifest was edited: %v", env)
|
||||
}
|
||||
}
|
||||
|
||||
// **The control plane's own manifest, composed through the whole path.**
|
||||
//
|
||||
// The store was given 6852 and the broker's plain port 5679 (the control-node's migration, novox/hq
|
||||
// 04-ISSUES/102). The control plane's own connections are sealed at genesis with the ports genesis
|
||||
// wrote; what its container is told beside them is where this machine put the store and the
|
||||
// broker now, read from the node's settings exactly as every consumer's binding is.
|
||||
func TestTheControlPlanesOwnAddressesFollowTheNodesPorts(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("the control plane's own manifest does not parse:\n%v", err)
|
||||
}
|
||||
// The manifest itself names them now; withSeatPorts is a no-op on it, and this holds it so.
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "container" {
|
||||
continue
|
||||
}
|
||||
env, _ := r["env"].(map[string]any)
|
||||
for key, want := range SeatPorts {
|
||||
if env[key] != want {
|
||||
t.Errorf("module.json says %s=%v, not %q", key, env[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
m = withSeatPorts(m)
|
||||
control, err := m.Resolve([]Built{{
|
||||
Name: "server", Kind: ArtifactImage,
|
||||
Reference: ArtifactStoreScheme + "mesh-controller/server@sha256:" + strings.Repeat("c", 64),
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{control}}
|
||||
needed := map[string]map[string]string{"mesh-controller": {}}
|
||||
for name := range m.OwnSecrets {
|
||||
needed["mesh-controller"][name] = "sealed-" + name
|
||||
}
|
||||
out, err := r.Declaration(Rendering{
|
||||
Needed: needed,
|
||||
ArtifactStore: "anchor.internal:5100",
|
||||
Seats: map[string]map[int]int{
|
||||
"mesh-store": {5432: 6852},
|
||||
"mesh-broker": {5671: 5671, 5672: 5679, 15672: 15673},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("the control plane does not compose: %v", err)
|
||||
}
|
||||
server := fileNamed(out, "mesh-controller.server")
|
||||
if server == nil {
|
||||
t.Fatalf("the control plane's container is not in the declaration: %v", out)
|
||||
}
|
||||
env, _ := server["env"].(map[string]any)
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
"MESH_STORE_IDENTITY_PORT": "6852",
|
||||
"MESH_STORE_LICENCES_PORT": "6852",
|
||||
"MESH_BROKER_AMQP_PORT": "5679",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "15673",
|
||||
"MESH_BROKER_ADDRESS_PORT": "5671",
|
||||
} {
|
||||
if env[key] != want {
|
||||
t.Errorf("the control plane is told %s=%v; the node put it on %s", key, env[key], want)
|
||||
}
|
||||
}
|
||||
if got := server["image"]; got != "anchor.internal:5100/mesh-controller/server@sha256:"+strings.Repeat("c", 64) {
|
||||
t.Errorf("the control plane's own image is %v, not routed through the store", got)
|
||||
}
|
||||
|
||||
// And on a mesh where the foundation is where genesis raised it, nothing is added.
|
||||
out, err = r.Declaration(Rendering{Needed: needed, ArtifactStore: "anchor.internal:5100"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env, _ = fileNamed(out, "mesh-controller.server")["env"].(map[string]any)
|
||||
if env["MESH_STORE_INVENTORY_PORT"] != "" || env["MESH_BROKER_AMQP_PORT"] != "" {
|
||||
t.Errorf("with no settings, the control plane is told %v", env)
|
||||
}
|
||||
}
|
||||
|
||||
// SeatPorts is what the control plane's manifest says beside each sealed connection: the port
|
||||
// this machine put the seat's holder at (novox/hq 04-ISSUES/102).
|
||||
var SeatPorts = map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}",
|
||||
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}",
|
||||
}
|
||||
|
||||
// withSeatPorts is the control plane's manifest with SeatPorts in its container's environment.
|
||||
//
|
||||
// **The manifest lands one commit after the code that fills it**, deliberately: a control plane
|
||||
// still running the previous build passes `${seat:…}` through unfilled, and the manifest may only
|
||||
// name the placeholder once every control plane that could compose it knows it. So the test does
|
||||
// not depend on module.json carrying these yet, and is a no-op once it does.
|
||||
func withSeatPorts(m Manifest) Manifest {
|
||||
out := m
|
||||
out.Resources = nil
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "container" {
|
||||
out.Resources = append(out.Resources, r)
|
||||
continue
|
||||
}
|
||||
copied := map[string]any{}
|
||||
for k, v := range r {
|
||||
copied[k] = v
|
||||
}
|
||||
env := map[string]any{}
|
||||
if had, ok := r["env"].(map[string]any); ok {
|
||||
for k, v := range had {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range SeatPorts {
|
||||
if _, said := env[k]; !said {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
copied["env"] = env
|
||||
out.Resources = append(out.Resources, copied)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
package envfile
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The port a machine put something on, said beside the value that names it.
|
||||
//
|
||||
// **An address written down when a port was decided does not follow the port** (novox/hq
|
||||
// 04-ISSUES/102). The control plane's own store and broker connections are written at genesis —
|
||||
// full connection strings, password and all — and sealed, so the mesh cannot open them to move the
|
||||
// port inside. When the node's settings move that port, every consumer's binding follows and the
|
||||
// control plane's own connection does not: it goes on dialling the number genesis wrote, and the
|
||||
// mesh is headless.
|
||||
//
|
||||
// So a setting has a third twin. `NAME_FILE` says where the value is; `NAME_PORT` says which port
|
||||
// this machine put the thing at, composed into the control plane's environment from the node's
|
||||
// settings exactly as a consumer's binding is. The value's own port is what stands when the twin
|
||||
// says nothing — a mesh whose foundation is still where genesis raised it needs no override, and
|
||||
// an empty answer is the mesh saying it has nothing to add, not the mesh saying zero.
|
||||
//
|
||||
// Only the port. The host inside the value is the machine's own loopback, or the broker's public
|
||||
// name — a fact of the genesis, not of any node's settings — and the mesh has no better opinion
|
||||
// of it. What moves under ADR 0100 is the port.
|
||||
|
||||
// PortVar is the twin saying which port this machine put the named thing at.
|
||||
func PortVar(name string) string { return name + "_PORT" }
|
||||
|
||||
// Port is what NAME_PORT says, checked to be a port, or "" when it says nothing.
|
||||
//
|
||||
// Blank counts as unset, as it does for every other variable here: a container given an empty
|
||||
// string was given nothing, and refusing it as ambiguous would turn an omission into a puzzle.
|
||||
func Port(name string) (string, error) {
|
||||
raw := strings.TrimSpace(os.Getenv(PortVar(name)))
|
||||
if raw == "" {
|
||||
return "", nil
|
||||
}
|
||||
if unfilled.MatchString(raw) {
|
||||
// **A placeholder the mesh never filled, and this is the one place it must not be a
|
||||
// fault.** The control plane composes its own declaration, so a manifest naming
|
||||
// `${seat:…}` reaches a control plane one build older than the manifest — one that does
|
||||
// not know the placeholder and passes it through as the value. Refusing it here would
|
||||
// leave every command and the daemon unable to open a store: headless, on the machine
|
||||
// that cannot be repaired through the mesh (novox/hq 04-ISSUES/102). So it is what an
|
||||
// empty answer is — nothing said, the sealed value stands — and it is said aloud, because
|
||||
// a manifest ahead of its binary is worth a line on stderr and not worth an outage.
|
||||
fmt.Fprintf(os.Stderr, "%s is %q, a placeholder nothing filled in — ignored; the port in "+
|
||||
"%s stands. The control plane composing this declaration is older than the manifest "+
|
||||
"naming it: rebuild and push it\n", PortVar(name), raw, name)
|
||||
return "", nil
|
||||
}
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil || n < 1 || n > 65535 {
|
||||
return "", fmt.Errorf("%s is %q, which is not a port", PortVar(name), raw)
|
||||
}
|
||||
return strconv.Itoa(n), nil
|
||||
}
|
||||
|
||||
// Placed is Value, with its port moved to where NAME_PORT says this machine put it.
|
||||
func Placed(name string) (string, error) {
|
||||
value, err := Value(name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
port, err := Port(name)
|
||||
if err != nil || port == "" {
|
||||
return value, err
|
||||
}
|
||||
placed, err := WithPort(value, port)
|
||||
if err != nil {
|
||||
// Where it came from is said; what it says is not. The value is a connection string with
|
||||
// a password in it, and every error here is written on the assumption it will be logged.
|
||||
return "", fmt.Errorf("%s names a port to move %s to, and %s could not be read as "+
|
||||
"something with a port in it: %w", PortVar(name), name, name, err)
|
||||
}
|
||||
return placed, nil
|
||||
}
|
||||
|
||||
// keywordPort is `port=…` in a `key=value` connection string.
|
||||
var keywordPort = regexp.MustCompile(`(^|\s)port=\S*`)
|
||||
|
||||
// unfilled is a value that is still a placeholder — `${…}` — rather than something a person or the
|
||||
// mesh wrote as a port.
|
||||
var unfilled = regexp.MustCompile(`^\$\{[^}]*\}$`)
|
||||
|
||||
// WithPort is the value with its port replaced by `port`.
|
||||
//
|
||||
// Three shapes, because the control plane's settings come in three: a URL
|
||||
// (`scheme://[user:password@]host[:port][/…]`), a bare `host[:port]` (the broker's address) and
|
||||
// a `key=value` connection string (`host=… port=…`), which is what the store's driver accepts
|
||||
// beside a URL. An IPv6 host stays in its brackets. Nothing here parses the URL properly — a
|
||||
// password with a character a URL parser dislikes is still a working connection string, and
|
||||
// refusing it would refuse a value that has been dialling fine since genesis.
|
||||
func WithPort(value, port string) (string, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("the value is empty")
|
||||
}
|
||||
if scheme, rest, isURL := strings.Cut(value, "://"); isURL {
|
||||
// **The password may hold any of `/ ? # @`, so the host begins after the LAST `@`**, and
|
||||
// the path only after that. Cutting at the first `/` would take a password's slash for the
|
||||
// path's and put the new port on the user name — a connection string that dials the wrong
|
||||
// host without a word. Genesis makes passwords that cannot do this; an accepted one can.
|
||||
// The connection strings this reads — a store's, a broker's, a management API's — carry
|
||||
// no `@` after their userinfo, which is what makes the last one the boundary.
|
||||
userinfo, hostAndTail := "", rest
|
||||
if at := strings.LastIndex(rest, "@"); at >= 0 {
|
||||
userinfo, hostAndTail = rest[:at+1], rest[at+1:]
|
||||
}
|
||||
authority, tail := hostAndTail, ""
|
||||
if end := strings.IndexAny(hostAndTail, "/?#"); end >= 0 {
|
||||
authority, tail = hostAndTail[:end], hostAndTail[end:]
|
||||
}
|
||||
host, err := hostWithPort(authority, port)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return scheme + "://" + userinfo + host + tail, nil
|
||||
}
|
||||
if strings.Contains(value, "=") && !strings.ContainsAny(value, "/") {
|
||||
if keywordPort.MatchString(value) {
|
||||
return keywordPort.ReplaceAllString(value, "${1}port="+port), nil
|
||||
}
|
||||
return value + " port=" + port, nil
|
||||
}
|
||||
return hostWithPort(value, port)
|
||||
}
|
||||
|
||||
// hostWithPort is `host[:port]` with the port set, brackets kept around an IPv6 host.
|
||||
func hostWithPort(authority, port string) (string, error) {
|
||||
if authority == "" {
|
||||
return "", fmt.Errorf("there is no host to put a port on")
|
||||
}
|
||||
host, _, err := net.SplitHostPort(authority)
|
||||
if err != nil {
|
||||
// No port yet. A bracketed IPv6 host without a port is a shape SplitHostPort refuses, and
|
||||
// a bare one carries colons of its own — both are a host, not an error.
|
||||
host = strings.TrimSuffix(strings.TrimPrefix(authority, "["), "]")
|
||||
}
|
||||
return net.JoinHostPort(host, port), nil
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package envfile
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The control plane's own connections follow the port the node moved (novox/hq 04-ISSUES/102).
|
||||
|
||||
func TestAPortTwinMovesTheValuesPort(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"postgres://mesh:s3cret@127.0.0.1:5432/inventory?sslmode=disable": "postgres://mesh:s3cret@127.0.0.1:6852/inventory?sslmode=disable",
|
||||
"postgres://mesh:s3cret@127.0.0.1/inventory": "postgres://mesh:s3cret@127.0.0.1:6852/inventory",
|
||||
"amqp://control:p%40ss@127.0.0.1:5672/": "amqp://control:p%40ss@127.0.0.1:6852/",
|
||||
"amqp://control:p@ss:with@127.0.0.1:5672/": "amqp://control:p@ss:with@127.0.0.1:6852/",
|
||||
"http://guest:guest@127.0.0.1:15672": "http://guest:guest@127.0.0.1:6852",
|
||||
"postgres://mesh:a/b?c#d@e@127.0.0.1:5432/inventory": "postgres://mesh:a/b?c#d@e@127.0.0.1:6852/inventory",
|
||||
"http://[::1]:15672/api": "http://[::1]:6852/api",
|
||||
"broker.example:5671": "broker.example:6852",
|
||||
"broker.example": "broker.example:6852",
|
||||
"host=127.0.0.1 port=5432 dbname=inventory": "host=127.0.0.1 port=6852 dbname=inventory",
|
||||
"host=127.0.0.1 dbname=inventory": "host=127.0.0.1 dbname=inventory port=6852",
|
||||
}
|
||||
for value, want := range cases {
|
||||
got, err := WithPort(value, "6852")
|
||||
if err != nil || got != want {
|
||||
t.Errorf("WithPort(%q) = %q, %v; want %q", value, got, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlacedLeavesTheValueAloneWhenNothingSaysAPort(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "value")
|
||||
if err := os.WriteFile(path, []byte("postgres://m:p@127.0.0.1:5432/inventory\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("MESH_P_FILE", path)
|
||||
t.Setenv("MESH_P_PORT", "")
|
||||
got, err := Placed("MESH_P")
|
||||
if err != nil || got != "postgres://m:p@127.0.0.1:5432/inventory" {
|
||||
t.Fatalf("got %q, %v", got, err)
|
||||
}
|
||||
t.Setenv("MESH_P_PORT", " 6852 ")
|
||||
got, err = Placed("MESH_P")
|
||||
if err != nil || got != "postgres://m:p@127.0.0.1:6852/inventory" {
|
||||
t.Fatalf("got %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPortTwinThatIsNotAPortIsRefusedWithoutQuotingTheValue(t *testing.T) {
|
||||
t.Setenv("MESH_Q", "postgres://m:hunter2@127.0.0.1:5432/inventory")
|
||||
t.Setenv("MESH_Q_PORT", "many")
|
||||
_, err := Placed("MESH_Q")
|
||||
if err == nil {
|
||||
t.Fatal("a port that is not a number was accepted")
|
||||
}
|
||||
if strings.Contains(err.Error(), "hunter2") {
|
||||
t.Fatalf("the value was quoted back: %v", err)
|
||||
}
|
||||
t.Setenv("MESH_Q", "")
|
||||
t.Setenv("MESH_Q_PORT", "6852")
|
||||
if _, err := Placed("MESH_Q"); err == nil {
|
||||
t.Fatal("a port with no value to put it on was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// A placeholder nothing filled is nothing said, not a fault: the control plane composing the
|
||||
// declaration may be one build behind the manifest, and refusing would leave it headless.
|
||||
func TestAnUnfilledPlaceholderIsNothingSaid(t *testing.T) {
|
||||
t.Setenv("MESH_R", "postgres://m:p@127.0.0.1:5432/inventory")
|
||||
t.Setenv("MESH_R_PORT", "${seat:mesh-store:5432}")
|
||||
got, err := Placed("MESH_R")
|
||||
if err != nil || got != "postgres://m:p@127.0.0.1:5432/inventory" {
|
||||
t.Fatalf("an unfilled placeholder was not ignored: %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package identity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/store"
|
||||
)
|
||||
|
||||
// ForTest is a fresh, migrated identity store in a database of its own, dropped when the test
|
||||
// ends. Exported for the same reason inventory.ForTest is: the check that a node's signed word
|
||||
// is verified against the key the mesh recorded lives beside the link, and a second copy of this
|
||||
// would be a second thing to keep true. It takes a *testing.T, so nothing that is not a test can
|
||||
// call it.
|
||||
func ForTest(t *testing.T) *Identity {
|
||||
t.Helper()
|
||||
admin := os.Getenv("MESH_TEST_POSTGRES")
|
||||
if admin == "" {
|
||||
t.Skip("no MESH_TEST_POSTGRES; run `make check` to raise one")
|
||||
}
|
||||
name := fmt.Sprintf("ident_%d_%s", time.Now().UnixNano()%1_000_000,
|
||||
strings.ToLower(strings.NewReplacer("/", "", "-", "").Replace(t.Name())))
|
||||
if len(name) > 60 {
|
||||
name = name[:60]
|
||||
}
|
||||
conn, err := pgx.Connect(t.Context(), admin)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot reach the test PostgreSQL: %v", err)
|
||||
}
|
||||
if _, err := conn.Exec(t.Context(), "create database "+name); err != nil {
|
||||
t.Fatalf("cannot create %s: %v", name, err)
|
||||
}
|
||||
conn.Close(t.Context())
|
||||
|
||||
cut := strings.LastIndex(admin, "/")
|
||||
t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
|
||||
|
||||
ident, err := Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
ident.Close()
|
||||
c, err := pgx.Connect(context.Background(), admin)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close(context.Background())
|
||||
_, _ = c.Exec(context.Background(), "drop database if exists "+name+" with (force)")
|
||||
})
|
||||
if err := ident.Ready(t.Context(), 20*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
migrations, err := Migrations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ident.store.Migrate(t.Context(), migrations); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ident
|
||||
}
|
||||
@@ -16,64 +16,25 @@ import (
|
||||
// node's peer list to chase it, and an address that moves is the thing declaring the hub was
|
||||
// meant to stop.
|
||||
//
|
||||
// **The adopted tunnel's addresses come first** (novox/hq ADR 0105). The hub that took over a
|
||||
// tunnel is at the tunnel's own address. A node enrolling with a key the tunnel already routed
|
||||
// to keeps the address the tunnel had for it — nothing a peer knows changes. And an address the
|
||||
// tunnel holds for a peer that has not enrolled is never handed to anyone else: that peer is
|
||||
// still reaching the hub at it.
|
||||
//
|
||||
// The rest is allocated in order from the range, taking the lowest free one. Not random: a person
|
||||
// reading a peer list should be able to guess which node an address belongs to, and reuse of a
|
||||
// released address is a smaller problem than a list nobody can hold in their head.
|
||||
// Allocated in order from the range, taking the lowest free one. Not random: a person reading a
|
||||
// peer list should be able to guess which node an address belongs to, and reuse of a released
|
||||
// address is a smaller problem than a list nobody can hold in their head.
|
||||
func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (string, error) {
|
||||
prefix, err := netip.ParsePrefix(cidr)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%q is not a network the mesh can allocate from: %w", cidr, err)
|
||||
}
|
||||
|
||||
var existing, key *string
|
||||
var name string
|
||||
var hub bool
|
||||
var existing *string
|
||||
if err := i.store.Pool().QueryRow(ctx,
|
||||
`select name, host(overlay_address), overlay_key, is_hub from node where id = $1`, node).
|
||||
Scan(&name, &existing, &key, &hub); err != nil {
|
||||
`select host(overlay_address) from node where id = $1`, node).Scan(&existing); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if existing != nil && *existing != "" {
|
||||
return *existing, nil
|
||||
}
|
||||
|
||||
tunnel, hubName, adopted, err := i.AdoptedTunnel(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if adopted && hub && hubName == name {
|
||||
address, err := netip.ParsePrefix(tunnel.Address)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the adopted tunnel's address %q: %w", tunnel.Address, err)
|
||||
}
|
||||
return i.place(ctx, node, address.Addr().String())
|
||||
}
|
||||
carried, err := i.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
taken := map[string]bool{}
|
||||
if adopted {
|
||||
// The tunnel's own address is the hub's whether or not the hub has been placed yet.
|
||||
if address, err := netip.ParsePrefix(tunnel.Address); err == nil {
|
||||
taken[address.Addr().String()] = true
|
||||
}
|
||||
}
|
||||
for _, p := range carried {
|
||||
if key != nil && p.PublicKey == *key {
|
||||
// The tunnel already routes to this key: the node keeps that address, and the peer
|
||||
// notices nothing when its machine enrols.
|
||||
return i.place(ctx, node, p.Address)
|
||||
}
|
||||
taken[p.Address] = true
|
||||
}
|
||||
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select host(overlay_address) from node where overlay_address is not null`)
|
||||
if err != nil {
|
||||
@@ -97,7 +58,12 @@ func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (strin
|
||||
candidate := prefix.Masked().Addr().Next()
|
||||
for prefix.Contains(candidate) {
|
||||
if !taken[candidate.String()] {
|
||||
return i.place(ctx, node, candidate.String())
|
||||
if _, err := i.store.Pool().Exec(ctx,
|
||||
`update node set overlay_address = $2::inet where id = $1`,
|
||||
node, candidate.String()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return candidate.String(), nil
|
||||
}
|
||||
candidate = candidate.Next()
|
||||
}
|
||||
@@ -106,13 +72,5 @@ func (i *Inventory) AssignAddress(ctx context.Context, node, cidr string) (strin
|
||||
// halfway through assigning one node.
|
||||
return "", fmt.Errorf(
|
||||
"every address in %s is taken, so %s cannot be given one. The mesh has outgrown its "+
|
||||
"range and renumbering it is a deliberate act", cidr, name)
|
||||
}
|
||||
|
||||
func (i *Inventory) place(ctx context.Context, node, address string) (string, error) {
|
||||
if _, err := i.store.Pool().Exec(ctx,
|
||||
`update node set overlay_address = $2::inet where id = $1`, node, address); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return address, nil
|
||||
"range and renumbering it is a deliberate act", cidr, node)
|
||||
}
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
-- The tunnel a node found on its machine, and the peers it carried (novox/hq ADR 0105).
|
||||
--
|
||||
-- On an adopted node that is the hub, the private network takes over the tunnel it finds: its
|
||||
-- key, its port, its address and range, and every peer. The node presents what it found when it
|
||||
-- enrols -- the same moment it presents its keys, because the found tunnel's key IS its key on the
|
||||
-- private network from then on -- and the mesh composes every address from it.
|
||||
|
||||
-- What the node presented: interface, unit and configuration path, port, address and range, and
|
||||
-- the tunnel's public key. The private key never travels; the node keeps it as its own overlay
|
||||
-- key. Null on a node that found no tunnel, which is every converged one.
|
||||
alter table node add column tunnel jsonb;
|
||||
|
||||
-- The node's last account of carrying it: the found interface down and disabled, the mesh's up
|
||||
-- in its place. Null until the node says so.
|
||||
alter table node add column tunnel_carried jsonb;
|
||||
|
||||
-- The peers the found tunnel had: a public key and the address the tunnel routed to it. Peers of
|
||||
-- the tunnel, not nodes of the mesh, until they enrol -- a machine the mesh has no record of, whose
|
||||
-- identity precedes its enrolment. One row per key, and one address per key on one tunnel.
|
||||
create table tunnel_peer (
|
||||
node uuid not null references node(id) on delete cascade,
|
||||
public_key text not null,
|
||||
address inet not null,
|
||||
since timestamptz not null default now(),
|
||||
primary key (node, public_key),
|
||||
unique (node, address)
|
||||
);
|
||||
@@ -1,375 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// The tunnel a node found on its machine, and the peers it carried (novox/hq ADR 0105).
|
||||
//
|
||||
// On an adopted node that is the hub, the private network takes over the tunnel it finds: its
|
||||
// private key, its port, its address and range, and every peer the found interface had. The node
|
||||
// presents what it found when it enrols, in the same breath as its keys — the found tunnel's key is
|
||||
// its key on the private network from then on — and the mesh composes every address from it: the
|
||||
// hub's is the tunnel's, the range is the tunnel's, and a peer that enrols keeps the address the
|
||||
// tunnel already had for its key.
|
||||
|
||||
// Tunnel is what a node found on its machine, as it presented it. No private key: the node keeps
|
||||
// it as its own overlay key, sealed like any own secret, and the mesh records only the public half
|
||||
// — which is then the node's overlay key too.
|
||||
type Tunnel struct {
|
||||
// Interface, Unit and Config are what the host takes over: the found interface, the unit
|
||||
// that raised it, and its configuration file, which is kept like any held file.
|
||||
Interface string `json:"interface"`
|
||||
Unit string `json:"unit"`
|
||||
Config string `json:"config"`
|
||||
// Port is the port the found interface listened on — one the hosting provider already lets
|
||||
// through, which is why it is worth taking.
|
||||
Port int `json:"port"`
|
||||
// Address is the interface's own address with its prefix length, 192.0.2.1/24; Range is the
|
||||
// network that prefix names, 192.0.2.0/24.
|
||||
Address string `json:"address"`
|
||||
Range string `json:"range"`
|
||||
// PublicKey is the found interface's, which every peer knows the tunnel by.
|
||||
PublicKey string `json:"public_key"`
|
||||
// Peers are the found interface's peers: each a public key and the address the tunnel routed
|
||||
// to it.
|
||||
Peers []TunnelPeer `json:"peers,omitempty"`
|
||||
// At is when the node presented it; zero on a tunnel not yet recorded.
|
||||
At time.Time `json:"at,omitempty"`
|
||||
}
|
||||
|
||||
// TunnelPeer is one peer of a found tunnel.
|
||||
type TunnelPeer struct {
|
||||
PublicKey string `json:"public_key"`
|
||||
// Address is the one host address the tunnel routed to the peer, without a prefix.
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
// Carried is what a node last said about carrying the tunnel it found: the found interface down
|
||||
// and disabled, the mesh's up in its place with the found key.
|
||||
type Carried struct {
|
||||
Interface string `json:"interface"`
|
||||
Port int `json:"port"`
|
||||
Range string `json:"range"`
|
||||
Peers int `json:"peers"`
|
||||
// State is one of the CarriedStates: the found interface is still up and the mesh's is not
|
||||
// (not taken), the found one is down and the mesh's up with its key (taken), or the found one
|
||||
// is down and the mesh's is not up — the one state where the peers reach nothing. Note is
|
||||
// what the host did about it, when it did something. Kept is where the found configuration's
|
||||
// original was kept.
|
||||
State string `json:"state"`
|
||||
Note string `json:"note,omitempty"`
|
||||
Kept string `json:"kept,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// The states a carried tunnel's account can be in, as the host says them.
|
||||
const (
|
||||
CarriedNotTaken = "not-taken"
|
||||
CarriedTaken = "taken"
|
||||
CarriedDown = "down"
|
||||
)
|
||||
|
||||
// CarriedPeer is one peer of the adopted tunnel as the mesh holds it: a peer of the tunnel, and
|
||||
// — once a node enrols with that key — a node of the mesh as well.
|
||||
type CarriedPeer struct {
|
||||
PublicKey string
|
||||
Address string
|
||||
// EnrolledAs names the node that enrolled with this key, or is empty while none has.
|
||||
EnrolledAs string
|
||||
}
|
||||
|
||||
// ErrNoTunnel is asking about a tunnel on a node that presented none.
|
||||
var ErrNoTunnel = errors.New("that node presented no tunnel")
|
||||
|
||||
// RecordTunnel keeps what a node presented, replacing what was there: the question is the tunnel
|
||||
// as the node found it now. The peers are replaced whole for the same reason.
|
||||
func (i *Inventory) RecordTunnel(ctx context.Context, nodeID string, t Tunnel) error {
|
||||
if strings.TrimSpace(t.Interface) == "" || strings.TrimSpace(t.PublicKey) == "" {
|
||||
return errors.New("a found tunnel names its interface and its public key, and this names neither")
|
||||
}
|
||||
if _, err := netip.ParsePrefix(t.Range); err != nil {
|
||||
return fmt.Errorf("the found tunnel's range %q is not a range: %w", t.Range, err)
|
||||
}
|
||||
address, err := netip.ParsePrefix(t.Address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the found tunnel's address %q is not an address with a prefix: %w", t.Address, err)
|
||||
}
|
||||
peers := make([]TunnelPeer, 0, len(t.Peers))
|
||||
for _, p := range t.Peers {
|
||||
host, single := peerHost(p.Address)
|
||||
if !single {
|
||||
// A peer routed a range rather than one address is a spoke's view of its hub — the
|
||||
// predecessor gives a spoke the whole subnet through the hub — and a hub is not a peer
|
||||
// the mesh carries. Skipped, not refused: a spoke enrols with what it found, and only
|
||||
// the hub's peers are ever carried (novox/hq ADR 0105).
|
||||
continue
|
||||
}
|
||||
if !address.Masked().Contains(host) {
|
||||
return fmt.Errorf("the found tunnel's peer %s is routed at %s, outside the tunnel's %s",
|
||||
shortKey(p.PublicKey), host, t.Range)
|
||||
}
|
||||
peers = append(peers, TunnelPeer{PublicKey: p.PublicKey, Address: host.String()})
|
||||
}
|
||||
t.Peers = nil
|
||||
t.At = time.Now().UTC()
|
||||
raw, err := json.Marshal(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
if _, err := tx.Exec(ctx, `update node set tunnel = $2 where id = $1`, nodeID, raw); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `delete from tunnel_peer where node = $1`, nodeID); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range peers {
|
||||
if _, err := tx.Exec(ctx,
|
||||
`insert into tunnel_peer (node, public_key, address) values ($1, $2, $3::inet)`,
|
||||
nodeID, p.PublicKey, p.Address); err != nil {
|
||||
return fmt.Errorf("recording the found tunnel's peer %s: %w", shortKey(p.PublicKey), err)
|
||||
}
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// peerHost is the one host address a peer's allowed address names — a bare address, or a /32
|
||||
// (or /128) — and false for anything wider or unreadable: a peer routed a whole range is not a
|
||||
// machine with an address the mesh could give a node.
|
||||
func peerHost(allowed string) (netip.Addr, bool) {
|
||||
allowed = strings.TrimSpace(allowed)
|
||||
if a, err := netip.ParseAddr(allowed); err == nil {
|
||||
return a, true
|
||||
}
|
||||
p, err := netip.ParsePrefix(allowed)
|
||||
if err != nil || !p.IsSingleIP() {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
return p.Addr(), true
|
||||
}
|
||||
|
||||
func shortKey(key string) string {
|
||||
if len(key) > 8 {
|
||||
return key[:8] + "…"
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// TunnelOf is the tunnel a node presented, with its peers, or ErrNoTunnel.
|
||||
func (i *Inventory) TunnelOf(ctx context.Context, name string) (Tunnel, error) {
|
||||
var raw []byte
|
||||
var id string
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select id, tunnel from node where name = $1`, name).Scan(&id, &raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Tunnel{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
||||
}
|
||||
if err != nil {
|
||||
return Tunnel{}, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return Tunnel{}, fmt.Errorf("%w: %s", ErrNoTunnel, name)
|
||||
}
|
||||
var t Tunnel
|
||||
if err := json.Unmarshal(raw, &t); err != nil {
|
||||
return Tunnel{}, err
|
||||
}
|
||||
t.Peers, err = i.tunnelPeers(ctx, id)
|
||||
return t, err
|
||||
}
|
||||
|
||||
func (i *Inventory) tunnelPeers(ctx context.Context, nodeID string) ([]TunnelPeer, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select public_key, host(address) from tunnel_peer where node = $1 order by address`, nodeID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []TunnelPeer
|
||||
for rows.Next() {
|
||||
var p TunnelPeer
|
||||
if err := rows.Scan(&p.PublicKey, &p.Address); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// AdoptedTunnel is the tunnel the mesh's private network runs over, if the hub adopted one: the
|
||||
// hub's found tunnel, when the hub's overlay key is the tunnel's. Absent, the mesh runs on its own
|
||||
// range — and a hub that found a tunnel but holds another key did not adopt it, which `overlay
|
||||
// show` says.
|
||||
//
|
||||
// The condition on the key is the condition of the whole record: a hub raised with a key of its
|
||||
// own would drop every peer's packets on the found port (novox/hq ADR 0105, option 2), so the
|
||||
// tunnel is adopted only when the hub answers to the key its peers know. **Not a condition on the
|
||||
// node's mode**: the range and the carried peers are facts of the mesh once the tunnel is taken,
|
||||
// and converging the hub — which flips its mode — must not renumber the mesh or drop the peers
|
||||
// still reaching it.
|
||||
func (i *Inventory) AdoptedTunnel(ctx context.Context) (Tunnel, string, bool, error) {
|
||||
var name string
|
||||
var key *string
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select name, overlay_key from node where is_hub and tunnel is not null`).
|
||||
Scan(&name, &key)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Tunnel{}, "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Tunnel{}, "", false, err
|
||||
}
|
||||
t, err := i.TunnelOf(ctx, name)
|
||||
if err != nil {
|
||||
return Tunnel{}, "", false, err
|
||||
}
|
||||
if key == nil || *key != t.PublicKey {
|
||||
return t, name, false, nil
|
||||
}
|
||||
return t, name, true, nil
|
||||
}
|
||||
|
||||
// CarriedPeers is every peer of the adopted tunnel, with the node that enrolled under its key
|
||||
// where one has: peers of the tunnel, and nodes of the mesh once they enrol. Empty when the hub
|
||||
// adopted no tunnel.
|
||||
func (i *Inventory) CarriedPeers(ctx context.Context) ([]CarriedPeer, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select p.public_key, host(p.address), coalesce(n.name, '')
|
||||
from tunnel_peer p
|
||||
join node hub on hub.id = p.node and hub.is_hub
|
||||
and hub.tunnel is not null and hub.overlay_key = hub.tunnel->>'public_key'
|
||||
left join node n on n.overlay_key = p.public_key
|
||||
order by p.address`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []CarriedPeer
|
||||
for rows.Next() {
|
||||
var p CarriedPeer
|
||||
if err := rows.Scan(&p.PublicKey, &p.Address, &p.EnrolledAs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// FoundTunnel is a node's found tunnel with the node's mode, for composing: the takeover is
|
||||
// declared to an adopted node only, since only there is a found unit kept to be stopped.
|
||||
type FoundTunnel struct {
|
||||
Tunnel
|
||||
NodeAdopted bool
|
||||
}
|
||||
|
||||
// Tunnels is every node's found tunnel by node name, for the ones whose overlay key is the
|
||||
// tunnel's — the ones whose private network takes it over. A found tunnel under another key is
|
||||
// left running beside the mesh's, and ADR 0100's rule that the ranges differ applies to it.
|
||||
func (i *Inventory) Tunnels(ctx context.Context) (map[string]FoundTunnel, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select name, tunnel, adopted from node
|
||||
where tunnel is not null and overlay_key = tunnel->>'public_key'`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]FoundTunnel{}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var raw []byte
|
||||
var adopted bool
|
||||
if err := rows.Scan(&name, &raw, &adopted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var t Tunnel
|
||||
if err := json.Unmarshal(raw, &t); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[name] = FoundTunnel{Tunnel: t, NodeAdopted: adopted}
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ErrStaleRekey is a rekey that names a previous overlay key other than the one recorded: a
|
||||
// replay of a rekey already done, or one made against a record that has since moved on.
|
||||
var ErrStaleRekey = errors.New("the rekey names a previous overlay key that is not the node's current one")
|
||||
|
||||
// Rekey records that a node took a found tunnel's key as its overlay key after enrolling (novox/hq
|
||||
// ADR 0105): the key and the tunnel are recorded as enrolment would have, and a hub is moved to the
|
||||
// tunnel's address so nothing derived from it is stale. The caller has verified the node signed
|
||||
// for this; what is checked here is that it follows the record — `previous` is the overlay key the
|
||||
// node holds now — so the same message cannot be applied twice.
|
||||
func (i *Inventory) Rekey(ctx context.Context, nodeID, previous, key string, t Tunnel) error {
|
||||
if key != t.PublicKey {
|
||||
return errors.New("a rekey takes a tunnel over with the tunnel's own key, and this names another")
|
||||
}
|
||||
var current *string
|
||||
var hub bool
|
||||
if err := i.store.Pool().QueryRow(ctx,
|
||||
`select overlay_key, is_hub from node where id = $1`, nodeID).Scan(¤t, &hub); err != nil {
|
||||
return err
|
||||
}
|
||||
if (current == nil && previous != "") || (current != nil && *current != previous) {
|
||||
return ErrStaleRekey
|
||||
}
|
||||
if err := i.RecordOverlayKey(ctx, nodeID, key); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := i.RecordTunnel(ctx, nodeID, t); err != nil {
|
||||
return err
|
||||
}
|
||||
if hub {
|
||||
address, err := netip.ParsePrefix(t.Address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := i.place(ctx, nodeID, address.Addr().String()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecordCarriedTunnel keeps what a node last said about carrying its found tunnel.
|
||||
func (i *Inventory) RecordCarriedTunnel(ctx context.Context, nodeID string, c Carried) error {
|
||||
c.At = time.Now().UTC()
|
||||
raw, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx, `update node set tunnel_carried = $2 where id = $1`, nodeID, raw)
|
||||
return err
|
||||
}
|
||||
|
||||
// CarriedTunnelOf is a node's last account of carrying its found tunnel, and whether it ever gave one.
|
||||
func (i *Inventory) CarriedTunnelOf(ctx context.Context, name string) (Carried, bool, error) {
|
||||
var raw []byte
|
||||
err := i.store.Pool().QueryRow(ctx, `select tunnel_carried from node where name = $1`, name).Scan(&raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Carried{}, false, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
||||
}
|
||||
if err != nil {
|
||||
return Carried{}, false, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return Carried{}, false, nil
|
||||
}
|
||||
var c Carried
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return Carried{}, false, err
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
@@ -1,279 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0105: the mesh adopts the predecessor's tunnel in place. The controller reads the
|
||||
// hub's address and range from the adopted tunnel, assigns an enrolling node the address its key
|
||||
// already had, and refuses to hand out an address the tunnel already holds.
|
||||
|
||||
const (
|
||||
tunnelKey = "TUNNEL-KEY-the-found-interfaces-public-key="
|
||||
peerTwo = "PEER-KEY-two============================="
|
||||
peerThree = "PEER-KEY-three==========================="
|
||||
)
|
||||
|
||||
// theFoundTunnel is what a hub presents at enrolment: the predecessor's interface on a
|
||||
// documentation range, with two peers each routed one address.
|
||||
func theFoundTunnel() Tunnel {
|
||||
return Tunnel{
|
||||
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf",
|
||||
Port: 51900, Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: tunnelKey,
|
||||
Peers: []TunnelPeer{{PublicKey: peerTwo, Address: "192.0.2.2/32"},
|
||||
{PublicKey: peerThree, Address: "192.0.2.3"}},
|
||||
}
|
||||
}
|
||||
|
||||
// anAdoptedHub is an adopted node that enrolled with the found tunnel's key and presented the
|
||||
// tunnel, then was placed as the hub — the order genesis does it in.
|
||||
func anAdoptedHub(t *testing.T, inv *Inventory) Node {
|
||||
t.Helper()
|
||||
hub, err := inv.AddNodeAs(t.Context(), "anchor", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(t.Context(), hub.ID, tunnelKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordTunnel(t.Context(), hub.ID, theFoundTunnel()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetPlace(t.Context(), "anchor", "anchor.example:51900", "hosting", true, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return hub
|
||||
}
|
||||
|
||||
func TestTheHubsAddressAndRangeComeFromTheAdoptedTunnel(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
hub := anAdoptedHub(t, inv)
|
||||
|
||||
tunnel, name, adopted, err := inv.AdoptedTunnel(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !adopted || name != "anchor" || tunnel.Range != "192.0.2.0/24" || tunnel.Port != 51900 {
|
||||
t.Fatalf("the adopted tunnel did not read back: adopted=%t on %s, %+v", adopted, name, tunnel)
|
||||
}
|
||||
if len(tunnel.Peers) != 2 || tunnel.Peers[0].Address != "192.0.2.2" || tunnel.Peers[1].Address != "192.0.2.3" {
|
||||
t.Fatalf("the peers did not read back as one host address each: %+v", tunnel.Peers)
|
||||
}
|
||||
|
||||
// Whatever range the caller would allocate from, the hub is at the tunnel's own address.
|
||||
address, err := inv.AssignAddress(t.Context(), hub.ID, "10.42.0.0/16")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if address != "192.0.2.1" {
|
||||
t.Fatalf("the hub was given %s, not the address the tunnel it took over had", address)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEnrollingNodeKeepsTheAddressTheTunnelHadForItsKey(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
anAdoptedHub(t, inv)
|
||||
|
||||
// A predecessor machine enrols: its host took its found interface's key as its overlay key,
|
||||
// which is the key the hub's tunnel already routes to.
|
||||
peer, err := inv.AddNodeAs(t.Context(), "home-server", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(t.Context(), peer.ID, peerThree); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
address, err := inv.AssignAddress(t.Context(), peer.ID, "192.0.2.0/24")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if address != "192.0.2.3" {
|
||||
t.Fatalf("the enrolling peer was given %s, not the 192.0.2.3 the tunnel had for its key", address)
|
||||
}
|
||||
|
||||
carried, err := inv.CarriedPeers(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
byKey := map[string]CarriedPeer{}
|
||||
for _, c := range carried {
|
||||
byKey[c.PublicKey] = c
|
||||
}
|
||||
if byKey[peerThree].EnrolledAs != "home-server" || byKey[peerTwo].EnrolledAs != "" {
|
||||
t.Fatalf("the registry cannot say which peer is a node now: %+v", carried)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFreshNodeIsNeverGivenAnAddressTheTunnelHolds(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
anAdoptedHub(t, inv)
|
||||
|
||||
// .1 is the hub, .2 and .3 are peers of the tunnel that have not enrolled: a new machine with
|
||||
// a key of its own gets the next one, from the same range.
|
||||
fresh, err := inv.AddNode(t.Context(), "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(t.Context(), fresh.ID, "A-KEY-OF-ITS-OWN========================"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
address, err := inv.AssignAddress(t.Context(), fresh.ID, "192.0.2.0/24")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if address != "192.0.2.4" {
|
||||
t.Fatalf("a fresh node was given %s; 192.0.2.2 and .3 are the tunnel's peers and .1 its hub", address)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATunnelUnderAnotherKeyIsNotAdopted(t *testing.T) {
|
||||
// A hub whose overlay key is not the found tunnel's would drop every peer's packets on the
|
||||
// found port (ADR 0105, option 2). Such a tunnel is recorded and not adopted: the mesh keeps
|
||||
// its own range, and ADR 0100's non-overlap rule stands for it.
|
||||
inv := fresh(t)
|
||||
hub, err := inv.AddNodeAs(t.Context(), "anchor", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(t.Context(), hub.ID, "THE-MESHS-OWN-KEY======================="); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordTunnel(t.Context(), hub.ID, theFoundTunnel()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetPlace(t.Context(), "anchor", "anchor.example:51820", "hosting", true, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, adopted, err := inv.AdoptedTunnel(t.Context()); err != nil || adopted {
|
||||
t.Fatalf("a tunnel under another key was adopted (err %v)", err)
|
||||
}
|
||||
if carried, err := inv.CarriedPeers(t.Context()); err != nil || len(carried) != 0 {
|
||||
t.Fatalf("peers of a tunnel that was not adopted are carried: %+v (err %v)", carried, err)
|
||||
}
|
||||
if address, err := inv.AssignAddress(t.Context(), hub.ID, "10.42.0.0/16"); err != nil || address != "10.42.0.1" {
|
||||
t.Fatalf("the hub was given %s (err %v); it should allocate from the mesh's own range", address, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPeerRoutedARangeIsNotCarried(t *testing.T) {
|
||||
// A peer routed a whole range is a spoke's view of its hub, never a machine with an address
|
||||
// the mesh could carry: skipped, and the single-address peers beside it kept.
|
||||
inv := fresh(t)
|
||||
hub, err := inv.AddNodeAs(t.Context(), "anchor", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := theFoundTunnel()
|
||||
found.Peers = append(found.Peers, TunnelPeer{PublicKey: "WIDE", Address: "192.0.2.0/24"})
|
||||
if err := inv.RecordTunnel(t.Context(), hub.ID, found); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := inv.TunnelOf(t.Context(), "anchor")
|
||||
if err != nil || len(got.Peers) != 2 {
|
||||
t.Fatalf("the range-routed peer was carried, or the others dropped: %+v %v", got.Peers, err)
|
||||
}
|
||||
// A single address outside the tunnel's range is still refused: it is not a peer this tunnel
|
||||
// routes to.
|
||||
found.Peers = []TunnelPeer{{PublicKey: "ELSEWHERE", Address: "198.51.100.7/32"}}
|
||||
if err := inv.RecordTunnel(t.Context(), hub.ID, found); err == nil || !strings.Contains(err.Error(), "outside") {
|
||||
t.Fatalf("a peer outside the range was recorded: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A predecessor spoke's tunnel has one peer — the hub — routed the whole range. Its enrolment must
|
||||
// not fail on it: only the hub's peers are ever carried, so a range-routed peer is skipped.
|
||||
func TestASpokesTunnelEnrolsWithItsHubPeerSkipped(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
anAdoptedHub(t, inv)
|
||||
spoke, err := inv.AddNodeAs(t.Context(), "home-server", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(t.Context(), spoke.ID, peerThree); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordTunnel(t.Context(), spoke.ID, Tunnel{
|
||||
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
|
||||
Address: "192.0.2.3/24", Range: "192.0.2.0/24", PublicKey: peerThree,
|
||||
Peers: []TunnelPeer{{PublicKey: tunnelKey, Address: "192.0.2.0/24"}},
|
||||
}); err != nil {
|
||||
t.Fatalf("a spoke-shaped tunnel was refused: %v", err)
|
||||
}
|
||||
got, err := inv.TunnelOf(t.Context(), "home-server")
|
||||
if err != nil || len(got.Peers) != 0 {
|
||||
t.Fatalf("the spoke's hub was recorded as a peer to carry: %+v %v", got.Peers, err)
|
||||
}
|
||||
// Nothing about the hub's carried peers changed: still two, one now enrolled.
|
||||
carried, err := inv.CarriedPeers(t.Context())
|
||||
if err != nil || len(carried) != 2 {
|
||||
t.Fatalf("carried peers: %+v %v", carried, err)
|
||||
}
|
||||
if address, err := inv.AssignAddress(t.Context(), spoke.ID, "192.0.2.0/24"); err != nil || address != "192.0.2.3" {
|
||||
t.Fatalf("the spoke did not keep its address: %s %v", address, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Converging the hub flips its mode and nothing else: the range stays the tunnel's and the peers
|
||||
// stay carried, or the mesh would renumber itself and drop the peers still reaching it.
|
||||
func TestConvergingTheHubKeepsTheRangeAndTheCarriedPeers(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
hub := anAdoptedHub(t, inv)
|
||||
if _, err := inv.AssignAddress(t.Context(), hub.ID, "192.0.2.0/24"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Converge(t.Context(), "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tunnel, _, adopted, err := inv.AdoptedTunnel(t.Context())
|
||||
if err != nil || !adopted || tunnel.Range != "192.0.2.0/24" {
|
||||
t.Fatalf("converging renumbered the mesh: adopted=%t %+v %v", adopted, tunnel, err)
|
||||
}
|
||||
if carried, err := inv.CarriedPeers(t.Context()); err != nil || len(carried) != 2 {
|
||||
t.Fatalf("converging dropped the carried peers: %+v %v", carried, err)
|
||||
}
|
||||
found, err := inv.Tunnels(t.Context())
|
||||
if err != nil || found["anchor"].NodeAdopted {
|
||||
t.Fatalf("a converged hub still reads as adopted for the takeover: %+v %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A hub that enrolled with a key of its own takes the tunnel over afterwards by rekeying: the key
|
||||
// and the tunnel are recorded, the hub moves to the tunnel's address, and the same rekey applied
|
||||
// again is stale.
|
||||
func TestARekeyTakesTheTunnelOverAfterEnrolment(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
hub, err := inv.AddNodeAs(t.Context(), "anchor", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const own = "THE-MESHS-OWN-KEY======================="
|
||||
if err := inv.RecordOverlayKey(t.Context(), hub.ID, own); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetPlace(t.Context(), "anchor", "anchor.example:51900", "hosting", true, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if address, err := inv.AssignAddress(t.Context(), hub.ID, "10.42.0.0/16"); err != nil || address != "10.42.0.1" {
|
||||
t.Fatalf("before the rekey the hub is on the mesh's own range: %s %v", address, err)
|
||||
}
|
||||
|
||||
if err := inv.Rekey(t.Context(), hub.ID, own, tunnelKey, theFoundTunnel()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, adopted, err := inv.AdoptedTunnel(t.Context())
|
||||
if err != nil || !adopted {
|
||||
t.Fatalf("the tunnel is not adopted after the rekey (%v)", err)
|
||||
}
|
||||
placed, err := inv.Overlays(t.Context())
|
||||
if err != nil || len(placed) != 1 || placed[0].Address != "192.0.2.1" || placed[0].Key != tunnelKey {
|
||||
t.Fatalf("the hub did not move to the tunnel's address under the tunnel's key: %+v %v", placed, err)
|
||||
}
|
||||
if err := inv.Rekey(t.Context(), hub.ID, own, tunnelKey, theFoundTunnel()); !errors.Is(err, ErrStaleRekey) {
|
||||
t.Fatalf("the same rekey applied again was not refused as stale: %v", err)
|
||||
}
|
||||
if err := inv.Rekey(t.Context(), hub.ID, tunnelKey, "ANOTHER-KEY=============================", theFoundTunnel()); err == nil {
|
||||
t.Fatal("a rekey to a key that is not the tunnel's was accepted")
|
||||
}
|
||||
}
|
||||
@@ -124,29 +124,6 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol
|
||||
"%s's overlay key could not be recorded: %w", node.Name, err)
|
||||
}
|
||||
}
|
||||
// And the tunnel it found, whose key is the overlay key above (novox/hq ADR 0105). Recorded
|
||||
// before the token is spent for the same reason as the keys: the first declaration this node
|
||||
// receives is composed from it, and a hub enrolled without its tunnel would be placed at an
|
||||
// address of the mesh's choosing rather than the tunnel's.
|
||||
if request.Tunnel != nil {
|
||||
if request.Tunnel.PublicKey != request.OverlayKey {
|
||||
return EnrolReply{}, fmt.Errorf("%s presented a tunnel under key %s and an overlay key "+
|
||||
"that is not it; a tunnel is taken over with its own key or not at all", node.Name,
|
||||
request.Tunnel.PublicKey)
|
||||
}
|
||||
peers := make([]inventory.TunnelPeer, 0, len(request.Tunnel.Peers))
|
||||
for _, p := range request.Tunnel.Peers {
|
||||
peers = append(peers, inventory.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address})
|
||||
}
|
||||
if err := e.Inventory.RecordTunnel(ctx, node.ID, inventory.Tunnel{
|
||||
Interface: request.Tunnel.Interface, Unit: request.Tunnel.Unit,
|
||||
Config: request.Tunnel.Config, Port: request.Tunnel.Port,
|
||||
Address: request.Tunnel.Address, Range: request.Tunnel.Range,
|
||||
PublicKey: request.Tunnel.PublicKey, Peers: peers,
|
||||
}); err != nil {
|
||||
return EnrolReply{}, fmt.Errorf("%s's found tunnel could not be recorded: %w", node.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Spent once the node is complete in the store.
|
||||
if err := e.Inventory.Spend(ctx, secret, by); err != nil {
|
||||
@@ -181,34 +158,6 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol
|
||||
return reply, nil
|
||||
}
|
||||
|
||||
// rekey applies a verified rekey: the node's overlay key and tunnel are recorded as enrolment
|
||||
// would have recorded them, and a hub moves to the tunnel's address.
|
||||
func (e Enrolment) rekey(ctx context.Context, node inventory.Node, r Rekey) error {
|
||||
if r.Tunnel == nil || r.OverlayKey == "" {
|
||||
return fmt.Errorf("%s sent a rekey naming no tunnel or no key; refused", node.Name)
|
||||
}
|
||||
if e.Identity == nil {
|
||||
return fmt.Errorf("%s sent a rekey and this mesh has no identity store to verify it against", node.Name)
|
||||
}
|
||||
if err := e.Identity.VerifyNode(ctx, node.ID,
|
||||
RekeyProof(node.Name, r.Previous, r.OverlayKey, r.Tunnel), r.Proof); err != nil {
|
||||
return fmt.Errorf("%s's rekey is not signed by %s's identity key; refused: %w", node.Name, node.Name, err)
|
||||
}
|
||||
peers := make([]inventory.TunnelPeer, 0, len(r.Tunnel.Peers))
|
||||
for _, p := range r.Tunnel.Peers {
|
||||
peers = append(peers, inventory.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address})
|
||||
}
|
||||
err := e.Inventory.Rekey(ctx, node.ID, r.Previous, r.OverlayKey, inventory.Tunnel{
|
||||
Interface: r.Tunnel.Interface, Unit: r.Tunnel.Unit, Config: r.Tunnel.Config, Port: r.Tunnel.Port,
|
||||
Address: r.Tunnel.Address, Range: r.Tunnel.Range, PublicKey: r.Tunnel.PublicKey, Peers: peers,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's rekey was not recorded: %w", node.Name, err)
|
||||
}
|
||||
log.Printf("%s took over the tunnel on %s: its overlay key is the tunnel's now", node.Name, r.Tunnel.Interface)
|
||||
return nil
|
||||
}
|
||||
|
||||
// claimant names the key presenting a token, so a claim can be held for it alone.
|
||||
func claimant(public ed25519.PublicKey) string {
|
||||
sum := sha256.Sum256(public)
|
||||
@@ -270,26 +219,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (err error) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// What it says about the tunnel it carried (novox/hq ADR 0105), whenever it says it.
|
||||
if report.Tunnel != nil {
|
||||
if err := e.Inventory.RecordCarriedTunnel(ctx, node.ID, inventory.Carried{
|
||||
Interface: report.Tunnel.Interface, Port: report.Tunnel.Port, Range: report.Tunnel.Range,
|
||||
Peers: report.Tunnel.Peers, State: report.Tunnel.State, Note: report.Tunnel.Note,
|
||||
Kept: report.Tunnel.Kept,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// A node taking a found tunnel's key after enrolment (novox/hq ADR 0105). Verified against the
|
||||
// node's live identity key before anything is written: the broker account authenticates the
|
||||
// connection, the signature proves the node itself said it. Refused outright when the proof
|
||||
// does not verify or is stale — a refusal, not "not now", so the node hears why.
|
||||
if report.Rekey != nil {
|
||||
if err := e.rekey(ctx, node, *report.Rekey); err != nil {
|
||||
return err
|
||||
}
|
||||
return e.Inventory.Seen(ctx, node.ID)
|
||||
}
|
||||
|
||||
// A bare word that a node is there is not an account of what the machine did or holds: it
|
||||
// moves last_seen and touches nothing else. This arrives every minute (link.AliveEvery),
|
||||
|
||||
@@ -8,8 +8,6 @@ package link
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -73,39 +71,12 @@ type EnrolRequest struct {
|
||||
// node's public key could replay the spent token (novox/hq issue 083, on review).
|
||||
Proof []byte `json:"proof,omitempty"`
|
||||
|
||||
// Tunnel is the tunnel this node found on its machine and whose key it took as its overlay
|
||||
// key (novox/hq ADR 0105): the interface, its port, address and range, and its peers. Presented
|
||||
// with the keys because it is one of them — OverlayKey above is this tunnel's public key when
|
||||
// it is set — and the mesh composes the hub's address, the range and every carried peer from
|
||||
// it. Nil from a node that found none, which is every converged one.
|
||||
Tunnel *Tunnel `json:"tunnel,omitempty"`
|
||||
|
||||
// Redelivered is set by the control plane, never sent: the broker handed this request over a
|
||||
// second time. Such a request does not finish an enrolment already spent — the first time may
|
||||
// have answered, and the node holds what it was told.
|
||||
Redelivered bool `json:"-"`
|
||||
}
|
||||
|
||||
// Tunnel is a found tunnel as a node presents it: everything but its private key, which the node
|
||||
// keeps as its own overlay key and never sends.
|
||||
type Tunnel struct {
|
||||
Interface string `json:"interface"`
|
||||
Unit string `json:"unit"`
|
||||
Config string `json:"config"`
|
||||
Port int `json:"port"`
|
||||
Address string `json:"address"`
|
||||
Range string `json:"range"`
|
||||
PublicKey string `json:"public_key"`
|
||||
Peers []TunnelPeer `json:"peers,omitempty"`
|
||||
}
|
||||
|
||||
// TunnelPeer is one peer of a found tunnel: its public key and the address the tunnel routed to
|
||||
// it.
|
||||
type TunnelPeer struct {
|
||||
PublicKey string `json:"public_key"`
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
// Signed is a declaration and the signature over it.
|
||||
//
|
||||
// The signature is over Declaration exactly as it will arrive, bytes unchanged — a node verifies
|
||||
@@ -158,60 +129,6 @@ type Report struct {
|
||||
// Reachable is what can be reached on the machine now: every listening socket and every
|
||||
// published container port. Only an adopted node reports it; it is what converging previews.
|
||||
Reachable []Reach `json:"reachable,omitempty"`
|
||||
|
||||
// Tunnel is what an adopted node says about the tunnel it found and carried (novox/hq ADR
|
||||
// 0105): the interface, its port, range and peer count, whether the found interface is down
|
||||
// and the mesh's up in its place, and where the found configuration's original was kept.
|
||||
Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
|
||||
|
||||
// Rekey is a node taking a found tunnel's key as its overlay key after enrolment (novox/hq
|
||||
// ADR 0105). A report carrying one is not an account of the machine: it moves the node's
|
||||
// overlay key and tunnel and nothing else.
|
||||
Rekey *Rekey `json:"rekey,omitempty"`
|
||||
}
|
||||
|
||||
// CarriedTunnel is a node's account of the tunnel it took over. State is "not-taken" (the found
|
||||
// interface still up, the mesh's not), "taken" (the found one down, the mesh's up with its key) or
|
||||
// "down" (the found one down and the mesh's not up: the peers reach nothing); Note is what the host
|
||||
// did about it.
|
||||
type CarriedTunnel struct {
|
||||
Interface string `json:"interface"`
|
||||
Port int `json:"port"`
|
||||
Range string `json:"range"`
|
||||
Peers int `json:"peers"`
|
||||
State string `json:"state"`
|
||||
Note string `json:"note,omitempty"`
|
||||
Kept string `json:"kept,omitempty"`
|
||||
}
|
||||
|
||||
// Rekey is a node saying it took a found tunnel's key as its overlay key after enrolling (novox/hq
|
||||
// ADR 0105) — the path for a hub that enrolled before the mesh knew to take a tunnel over, since
|
||||
// re-enrolling would rotate every key the node holds. Carried in a report, on the node's own
|
||||
// authenticated connection, and signed with its identity key over RekeyProof, so a report forged
|
||||
// on a stolen broker account cannot move a node's overlay key.
|
||||
type Rekey struct {
|
||||
// Previous is the overlay key the node holds now, as the mesh records it. A rekey naming
|
||||
// another is stale — a replay, or made against a record that moved on — and is refused.
|
||||
Previous string `json:"previous"`
|
||||
OverlayKey string `json:"overlay_key"`
|
||||
Tunnel *Tunnel `json:"tunnel"`
|
||||
Proof []byte `json:"proof"`
|
||||
}
|
||||
|
||||
// RekeyProof is what a node signs when it rekeys: the node, the key it leaves, the key it takes
|
||||
// and the tunnel it took it from, so a proof cannot be moved to another node or another tunnel.
|
||||
func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte {
|
||||
var t Tunnel
|
||||
if tunnel != nil {
|
||||
t = *tunnel
|
||||
}
|
||||
peers := make([]string, 0, len(t.Peers))
|
||||
for _, p := range t.Peers {
|
||||
peers = append(peers, p.PublicKey+"@"+p.Address)
|
||||
}
|
||||
return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" +
|
||||
t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" +
|
||||
t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ","))
|
||||
}
|
||||
|
||||
// Held is one file or container found on an adopted node and kept as it was.
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
package link_test
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0105: a hub that enrolled before the mesh knew to take a tunnel over rekeys onto the
|
||||
// found tunnel's key without re-enrolling — which would rotate every key it holds and remake every
|
||||
// credential the mesh sealed to it. The rekey rides in a report and is signed with the node's
|
||||
// identity key; the mesh verifies it against the key it recorded, and refuses one signed by
|
||||
// another key or one already applied.
|
||||
|
||||
const (
|
||||
ownKey = "THE-MESHS-OWN-KEY======================="
|
||||
tunnelKey = "TUNNEL-KEY-the-found-interfaces-public-key="
|
||||
)
|
||||
|
||||
func theTunnel() *link.Tunnel {
|
||||
return &link.Tunnel{Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf",
|
||||
Port: 51900, Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: tunnelKey,
|
||||
Peers: []link.TunnelPeer{{PublicKey: "PEER-A=", Address: "192.0.2.2/32"}}}
|
||||
}
|
||||
|
||||
// anEnrolledHub is a hub the way it stands before the feature: adopted, placed, its overlay key its
|
||||
// own, its identity key recorded — and a mesh holding both stores.
|
||||
func anEnrolledHub(t *testing.T) (link.Enrolment, inventory.Node, ed25519.PrivateKey) {
|
||||
t.Helper()
|
||||
inv := inventory.ForTest(t)
|
||||
ident := identity.ForTest(t)
|
||||
ctx := t.Context()
|
||||
hub, err := inv.AddNodeAs(ctx, "anchor", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
public, private, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ident.RecordNodeKey(ctx, hub.ID, public); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordOverlayKey(ctx, hub.ID, ownKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetPlace(ctx, "anchor", "anchor.example:51900", "hosting", true, "10.42.0.1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return link.Enrolment{Inventory: inv, Identity: ident}, hub, private
|
||||
}
|
||||
|
||||
func TestASignedRekeyMovesTheHubOntoItsTunnel(t *testing.T) {
|
||||
e, hub, private := anEnrolledHub(t)
|
||||
ctx := t.Context()
|
||||
rekey := &link.Rekey{Previous: ownKey, OverlayKey: tunnelKey, Tunnel: theTunnel()}
|
||||
rekey.Proof = ed25519.Sign(private, link.RekeyProof("anchor", ownKey, tunnelKey, theTunnel()))
|
||||
|
||||
if err := e.Heard(ctx, link.Report{Node: "anchor", Rekey: rekey}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
placed, err := e.Inventory.Overlays(ctx)
|
||||
if err != nil || len(placed) != 1 {
|
||||
t.Fatal(placed, err)
|
||||
}
|
||||
if placed[0].Key != tunnelKey || placed[0].Address != "192.0.2.1" {
|
||||
t.Fatalf("the hub is not on the tunnel's key and address: %+v", placed[0])
|
||||
}
|
||||
tunnel, _, adopted, err := e.Inventory.AdoptedTunnel(ctx)
|
||||
if err != nil || !adopted || tunnel.Range != "192.0.2.0/24" || len(tunnel.Peers) != 1 {
|
||||
t.Fatalf("the tunnel is not adopted after the rekey: %+v %t %v", tunnel, adopted, err)
|
||||
}
|
||||
_ = hub
|
||||
|
||||
// Replayed, it is stale: the previous key it names is no longer the node's.
|
||||
err = e.Heard(ctx, link.Report{Node: "anchor", Rekey: rekey})
|
||||
if err == nil || !strings.Contains(err.Error(), "previous overlay key") {
|
||||
t.Fatalf("a replayed rekey was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARekeySignedByAnotherKeyIsRefusedAndChangesNothing(t *testing.T) {
|
||||
e, _, _ := anEnrolledHub(t)
|
||||
ctx := t.Context()
|
||||
_, stranger, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rekey := &link.Rekey{Previous: ownKey, OverlayKey: tunnelKey, Tunnel: theTunnel()}
|
||||
rekey.Proof = ed25519.Sign(stranger, link.RekeyProof("anchor", ownKey, tunnelKey, theTunnel()))
|
||||
|
||||
err = e.Heard(ctx, link.Report{Node: "anchor", Rekey: rekey})
|
||||
if err == nil || !strings.Contains(err.Error(), "not signed by anchor's identity key") {
|
||||
t.Fatalf("a rekey signed by a stranger was accepted: %v", err)
|
||||
}
|
||||
placed, _ := e.Inventory.Overlays(ctx)
|
||||
if placed[0].Key != ownKey || placed[0].Address != "10.42.0.1" {
|
||||
t.Fatalf("a refused rekey changed the record: %+v", placed[0])
|
||||
}
|
||||
if _, _, adopted, _ := e.Inventory.AdoptedTunnel(ctx); adopted {
|
||||
t.Fatal("a refused rekey recorded a tunnel")
|
||||
}
|
||||
|
||||
// And a proof moved to another tunnel — the signature was over one tunnel, the message names
|
||||
// another — does not verify either.
|
||||
moved := &link.Rekey{Previous: ownKey, OverlayKey: tunnelKey, Tunnel: theTunnel()}
|
||||
other := theTunnel()
|
||||
other.Port = 51820
|
||||
moved.Proof = ed25519.Sign(mustPrivate(t, e, "anchor"), link.RekeyProof("anchor", ownKey, tunnelKey, other))
|
||||
if err := e.Heard(ctx, link.Report{Node: "anchor", Rekey: moved}); err == nil {
|
||||
t.Fatal("a proof over another tunnel was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// mustPrivate is a fresh key recorded as the node's live one, for signing in a test that needs
|
||||
// the node's own signature after the fixture's key is out of scope.
|
||||
func mustPrivate(t *testing.T, e link.Enrolment, node string) ed25519.PrivateKey {
|
||||
t.Helper()
|
||||
public, private, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := e.Inventory.NodeByName(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := e.Identity.RecordNodeKey(t.Context(), n.ID, public); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return private
|
||||
}
|
||||
@@ -146,12 +146,8 @@ func (s *Server) Answers(r Replayer) error {
|
||||
}
|
||||
|
||||
// Connect opens the control plane's own connection to the broker.
|
||||
//
|
||||
// On the port MESH_BROKER_AMQP_PORT names when the node's settings moved the broker (novox/hq
|
||||
// 04-ISSUES/102) — the URL is genesis's, sealed, and its port is the one thing in it the node may
|
||||
// have moved since.
|
||||
func Connect(enroller Enroller, listener Listener) (*Server, error) {
|
||||
url, err := envfile.Placed(AMQPVar)
|
||||
url, err := envfile.Value(AMQPVar)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -43,40 +43,6 @@ func Declaration(node Node, peers []Peer, keyPath string) ([]byte, error) {
|
||||
keyPath = DefaultKeyPath
|
||||
}
|
||||
|
||||
up := Resource{
|
||||
"id": "overlay-up", "type": "service", "unit": Unit,
|
||||
"state": "running",
|
||||
// Enabled, so the node comes back onto the network after a reboot without waiting to
|
||||
// be told again. A node whose overlay only exists while something is watching is not
|
||||
// a node that survives being switched off and on.
|
||||
"boot": "enabled",
|
||||
// And restarted when the peer list changes, because a running interface does not
|
||||
// re-read its configuration.
|
||||
//
|
||||
// This is the whole of it: a node joins, every existing node's peer list changes,
|
||||
// each file is replaced — and without this the service is already running, nothing
|
||||
// reloads it, and every node keeps a network that no longer matches the mesh. It
|
||||
// reports complete success. The lab found it the moment a third node arrived.
|
||||
//
|
||||
// Declared state rather than a command. The service must reflect the file; the host
|
||||
// works out that it does not. A command to restart would be an action, and the link
|
||||
// may not carry one (novox/hq ADR 0005) — the host refused exactly that, correctly,
|
||||
// which is how this shape was arrived at.
|
||||
"restart-on": []string{"overlay-config"},
|
||||
}
|
||||
if node.TakesOver != nil {
|
||||
// The private network takes over the tunnel it found (novox/hq ADR 0105): before this
|
||||
// unit starts, the host stops and disables the found one — never flushing it — and keeps
|
||||
// its configuration like any held file. The key is already the found one: the node took
|
||||
// it as its own overlay key when it enrolled, which is why the mesh's peer list for it
|
||||
// carries the found peers under the key they know.
|
||||
up["takes-over"] = map[string]any{
|
||||
"interface": node.TakesOver.Interface,
|
||||
"unit": node.TakesOver.Unit,
|
||||
"config": node.TakesOver.Config,
|
||||
}
|
||||
}
|
||||
|
||||
resources := []Resource{
|
||||
{
|
||||
"id": "overlay-tools", "type": "package", "package": "wireguard-tools",
|
||||
@@ -89,7 +55,27 @@ func Declaration(node Node, peers []Peer, keyPath string) ([]byte, error) {
|
||||
"mode": "0600",
|
||||
"content": config(node, peers, keyPath),
|
||||
},
|
||||
up,
|
||||
{
|
||||
"id": "overlay-up", "type": "service", "unit": Unit,
|
||||
"state": "running",
|
||||
// Enabled, so the node comes back onto the network after a reboot without waiting to
|
||||
// be told again. A node whose overlay only exists while something is watching is not
|
||||
// a node that survives being switched off and on.
|
||||
"boot": "enabled",
|
||||
// And restarted when the peer list changes, because a running interface does not
|
||||
// re-read its configuration.
|
||||
//
|
||||
// This is the whole of it: a node joins, every existing node's peer list changes,
|
||||
// each file is replaced — and without this the service is already running, nothing
|
||||
// reloads it, and every node keeps a network that no longer matches the mesh. It
|
||||
// reports complete success. The lab found it the moment a third node arrived.
|
||||
//
|
||||
// Declared state rather than a command. The service must reflect the file; the host
|
||||
// works out that it does not. A command to restart would be an action, and the link
|
||||
// may not carry one (novox/hq ADR 0005) — the host refused exactly that, correctly,
|
||||
// which is how this shape was arrived at.
|
||||
"restart-on": []string{"overlay-config"},
|
||||
},
|
||||
}
|
||||
|
||||
// The names used to be appended here, on the argument that a node with peers and no names is
|
||||
|
||||
@@ -223,40 +223,3 @@ func TestTheHubForwardsAndNobodyElseDoes(t *testing.T) {
|
||||
"compromised one could do")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0105: a node whose private network takes over the tunnel it found is told so on
|
||||
// the interface's service, and nothing else about the declaration changes — the key is already
|
||||
// the found one, taken at enrolment.
|
||||
func TestTakingOverAFoundTunnelIsSaidOnTheInterfacesService(t *testing.T) {
|
||||
node := Node{Name: "anchor", Key: "PUB", Address: "192.0.2.1", Hub: true,
|
||||
Endpoint: "198.51.100.1:51900",
|
||||
TakesOver: &TakeOver{Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf"}}
|
||||
config, resources := declarationFor(t, node, nil)
|
||||
|
||||
var up map[string]any
|
||||
for _, r := range resources {
|
||||
if r["type"] == "service" {
|
||||
up = r
|
||||
}
|
||||
}
|
||||
takes, ok := up["takes-over"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("the interface's service does not say what it takes over: %+v", up)
|
||||
}
|
||||
if takes["unit"] != "wg-quick@wg0" || takes["config"] != "/etc/wireguard/wg0.conf" || takes["interface"] != "wg0" {
|
||||
t.Errorf("the takeover names the wrong tunnel: %+v", takes)
|
||||
}
|
||||
if !strings.Contains(config, "ListenPort = 51900") {
|
||||
t.Errorf("the hub's interface does not listen on the tunnel's port:\n%s", config)
|
||||
}
|
||||
if strings.Contains(config, "PrivateKey") {
|
||||
t.Error("the found key travelled in the configuration; it is the node's own, set from its key file")
|
||||
}
|
||||
|
||||
_, plain := declarationFor(t, Node{Name: "laptop", Key: "PUB", Address: "192.0.2.4"}, nil)
|
||||
for _, r := range plain {
|
||||
if _, says := r["takes-over"]; says {
|
||||
t.Error("a node taking over nothing was told to take something over")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,48 +26,6 @@ type Node struct {
|
||||
Site string
|
||||
Hub bool
|
||||
Address string
|
||||
|
||||
// Carried are the peers of the tunnel this node took over (novox/hq ADR 0105): machines the
|
||||
// mesh has no record of, each known by the public key and the address the found tunnel routed
|
||||
// to it. Only a hub has any. They stay in its peer list until a node enrols with that key —
|
||||
// from then on the node is the peer.
|
||||
Carried []Carried
|
||||
// TakesOver names the found tunnel this node's private network replaces: its unit is stopped
|
||||
// and disabled, never flushed, and its configuration kept, before the mesh's interface comes
|
||||
// up with the found key. Nil on a node that raises the mesh's interface beside whatever it has.
|
||||
TakesOver *TakeOver
|
||||
}
|
||||
|
||||
// Carried is one peer of an adopted tunnel that has not enrolled: a peer of the tunnel, not a
|
||||
// node of the mesh.
|
||||
type Carried struct {
|
||||
Key string
|
||||
Address string
|
||||
}
|
||||
|
||||
// TakeOver is the found tunnel a node's private network takes over, as the host is told it.
|
||||
type TakeOver struct {
|
||||
Interface string
|
||||
Unit string
|
||||
Config string
|
||||
}
|
||||
|
||||
// HostPrefix is one address as a route: /32 for IPv4, /128 for IPv6.
|
||||
func HostPrefix(address string) string {
|
||||
if strings.Contains(address, ":") {
|
||||
return address + "/128"
|
||||
}
|
||||
return address + "/32"
|
||||
}
|
||||
|
||||
// CarriedName is how a carried peer is named in a peer list: it has no node name, so it is named
|
||||
// by the key its packets arrive under.
|
||||
func CarriedName(key string) string {
|
||||
short := key
|
||||
if len(short) > 8 {
|
||||
short = short[:8] + "…"
|
||||
}
|
||||
return "a peer of the tunnel (" + short + ")"
|
||||
}
|
||||
|
||||
// Reachable reports whether other nodes can dial this one. Declared, never inferred.
|
||||
@@ -187,9 +145,7 @@ func Compute(nodes []Node, overlayCIDR string) (Graph, error) {
|
||||
// The hub holds every node that does not share a site with it, because those nodes
|
||||
// route through it and it must know where to send the replies. Ones it cannot dial
|
||||
// will dial it.
|
||||
enrolled := map[string]bool{}
|
||||
for _, other := range usable {
|
||||
enrolled[other.Key] = true
|
||||
if other.Name == self.Name || (self.Site != "" && self.Site == other.Site) {
|
||||
continue
|
||||
}
|
||||
@@ -200,21 +156,6 @@ func Compute(nodes []Node, overlayCIDR string) (Graph, error) {
|
||||
Why: "routes through this hub",
|
||||
})
|
||||
}
|
||||
// And every peer of the tunnel it took over that has not enrolled (novox/hq ADR
|
||||
// 0105): the same key and the same address the found tunnel had for it, so the
|
||||
// machine behind it cannot tell the tunnel changed hands. No endpoint — it dials in,
|
||||
// as it always did. Once a node enrols with that key, the node's entry above is the
|
||||
// peer, and WireGuard takes one entry per key.
|
||||
for _, c := range self.Carried {
|
||||
if enrolled[c.Key] {
|
||||
continue
|
||||
}
|
||||
peers = append(peers, Peer{
|
||||
Name: CarriedName(c.Key), Key: c.Key,
|
||||
Allowed: HostPrefix(c.Address),
|
||||
Why: "carried from the tunnel this hub took over — a peer of the tunnel, not yet a node of the mesh",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(peers, func(i, j int) bool { return peers[i].Name < peers[j].Name })
|
||||
|
||||
@@ -304,39 +304,3 @@ func TestOneReachableNodeIsEnoughToPeerDirectly(t *testing.T) {
|
||||
"nowhere to go")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0105: a hub that took over the predecessor's tunnel carries every peer that tunnel
|
||||
// had, under the key and at the address the peer knows, until a node enrols with that key.
|
||||
func TestTheHubCarriesTheTunnelsPeersUntilTheyEnrol(t *testing.T) {
|
||||
hub := at("anchor", "hosting", "192.0.2.1", "198.51.100.1:51900", true)
|
||||
hub.Carried = []Carried{
|
||||
{Key: "key-home", Address: "192.0.2.2"},
|
||||
{Key: "key-workstation", Address: "192.0.2.3"},
|
||||
}
|
||||
// The machine behind key-home enrolled: it is a node now, at the address it kept.
|
||||
home := at("home", "house", "192.0.2.2", "", false)
|
||||
home.Key = "key-home"
|
||||
|
||||
g, err := Compute([]Node{hub, home}, "192.0.2.0/24")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
peers := peersOf(t, g, "anchor")
|
||||
carried, ok := peers[CarriedName("key-workstation")]
|
||||
if !ok {
|
||||
t.Fatalf("the hub does not carry the peer that has not enrolled: %+v", g["anchor"])
|
||||
}
|
||||
if carried.Allowed != "192.0.2.3/32" || carried.Endpoint != "" || carried.Key != "key-workstation" {
|
||||
t.Errorf("a carried peer is not the tunnel's own entry — same key, its one address, no endpoint: %+v", carried)
|
||||
}
|
||||
if _, twice := peers[CarriedName("key-home")]; twice {
|
||||
t.Error("a peer that enrolled is carried as well as listed as a node: WireGuard takes one entry per key")
|
||||
}
|
||||
if node, ok := peers["home"]; !ok || node.Key != "key-home" || node.Allowed != "192.0.2.2/32" {
|
||||
t.Errorf("the enrolled peer is not the node it became: %+v", node)
|
||||
}
|
||||
// Carried peers are the hub's business only: a spoke routes everything through the hub.
|
||||
if _, leaked := peersOf(t, g, "home")[CarriedName("key-workstation")]; leaked {
|
||||
t.Error("a carried peer appeared in a spoke's peer list")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The manifest's placeholder, unfilled, reaches a store reader and changes nothing.**
|
||||
//
|
||||
// The control plane composes its own declaration, so the manifest naming `${seat:…}` can be
|
||||
// composed by a control plane one build older than it — one that passes the literal through as
|
||||
// the value. That is the state of the live control-node between this manifest landing and its
|
||||
// next build being pushed, and a reader that refused the literal would leave it headless
|
||||
// (novox/hq 04-ISSUES/102, finding F1). So the reader is held to ignoring exactly what
|
||||
// module.json says, not a placeholder shaped like it.
|
||||
func TestTheManifestsOwnPlaceholderUnfilledLeavesTheStoreWhereTheFileSays(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Resources []struct {
|
||||
Type string `json:"type"`
|
||||
Env map[string]string `json:"env"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var written string
|
||||
for _, r := range m.Resources {
|
||||
if r.Type == "container" {
|
||||
written = r.Env["MESH_STORE_INVENTORY_PORT"]
|
||||
}
|
||||
}
|
||||
if written == "" {
|
||||
t.Fatal("module.json no longer names MESH_STORE_INVENTORY_PORT")
|
||||
}
|
||||
|
||||
alone(t)
|
||||
t.Setenv(Variable(example), dsn)
|
||||
t.Setenv(PortVariable(example), written)
|
||||
opened, err := Open(t.Context(), example)
|
||||
if err != nil {
|
||||
t.Fatalf("the unfilled placeholder was refused, which is a headless control plane: %v", err)
|
||||
}
|
||||
defer opened.Close()
|
||||
if got := opened.Pool().Config().ConnConfig.Port; got != 5432 {
|
||||
t.Fatalf("the store is on %d; with the placeholder unfilled, the file's port stands", got)
|
||||
}
|
||||
}
|
||||
@@ -213,55 +213,3 @@ func mustNotLeak(t *testing.T, err error) {
|
||||
t.Fatalf("the password is in the error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The node moved the store, and the control plane's own connection follows (novox/hq 04-ISSUES/102).
|
||||
//
|
||||
// The connection string is what genesis wrote, port and all; the port twin is what the node's
|
||||
// settings say now. The pool's configuration is the one place both meet.
|
||||
func TestThePortTwinMovesTheStoresPort(t *testing.T) {
|
||||
alone(t)
|
||||
t.Setenv(PortVariable(example), "")
|
||||
path := filepath.Join(t.TempDir(), "inventory")
|
||||
if err := os.WriteFile(path, []byte(dsn+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(FileVariable(example), path)
|
||||
|
||||
opened, err := Open(t.Context(), example)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := opened.Pool().Config().ConnConfig.Port; got != 5432 {
|
||||
t.Fatalf("with nothing said, the store is on %d rather than what the file says", got)
|
||||
}
|
||||
opened.Close()
|
||||
|
||||
t.Setenv(PortVariable(example), "6852")
|
||||
opened, err = Open(t.Context(), example)
|
||||
if err != nil {
|
||||
t.Fatalf("a moved port was refused: %v", err)
|
||||
}
|
||||
defer opened.Close()
|
||||
if got := opened.Pool().Config().ConnConfig.Port; got != 6852 {
|
||||
t.Fatalf("the store is on %d, and the node put it on 6852", got)
|
||||
}
|
||||
if got := opened.Pool().Config().ConnConfig.Password; got != "s3cret-in-here" {
|
||||
t.Fatalf("moving the port changed the password to %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPortTwinThatIsNotAPortNamesItselfAndNotTheSecret(t *testing.T) {
|
||||
alone(t)
|
||||
t.Setenv(Variable(example), dsn)
|
||||
t.Setenv(PortVariable(example), "six")
|
||||
_, err := Open(t.Context(), example)
|
||||
if err == nil {
|
||||
t.Fatal("a port that is not a number was accepted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), PortVariable(example)) {
|
||||
t.Errorf("the error does not name the variable: %v", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "s3cret") {
|
||||
t.Errorf("the error quotes the password: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-37
@@ -25,8 +25,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/envfile"
|
||||
)
|
||||
|
||||
// contextName is what a context may be called.
|
||||
@@ -69,17 +67,6 @@ func Variable(context string) string {
|
||||
// raises the first one.
|
||||
func FileVariable(context string) string { return Variable(context) + "_FILE" }
|
||||
|
||||
// PortVariable is the environment variable naming the PORT this machine put the store at — the
|
||||
// third twin, beside the value and the file.
|
||||
//
|
||||
// **The connection string is sealed and the port inside it is genesis's** (novox/hq 04-ISSUES/102).
|
||||
// The control plane cannot open its own store secret to move the port, and a node's settings can
|
||||
// move the store (ADR 0100: the foundation's ports are the node's). Every consumer's binding
|
||||
// followed that setting; the control plane's own connection did not, and the mesh was headless. So
|
||||
// the port is composed into the control plane's environment from the node's settings, exactly as a
|
||||
// consumer's binding is, and the connection string's own port stands only when this says nothing.
|
||||
func PortVariable(context string) string { return envfile.PortVar(Variable(context)) }
|
||||
|
||||
// Database is what a context's database is called.
|
||||
//
|
||||
// Named after the context, so that a person looking at a PostgreSQL server can see which
|
||||
@@ -98,7 +85,7 @@ func Open(ctx context.Context, name string) (*Store, error) {
|
||||
"name, so it must be lower-case letters and digits, starting with a letter", name)
|
||||
}
|
||||
|
||||
dsn, from, err := placed(name)
|
||||
dsn, from, err := settingsFor(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -182,29 +169,6 @@ func settingsFor(name string) (dsn, from string, err error) {
|
||||
return direct, Variable(name), nil
|
||||
}
|
||||
|
||||
// placed is a context's connection string with its port moved to where this machine put the
|
||||
// store, when the node's settings say so (PortVariable), and as it was otherwise.
|
||||
func placed(name string) (dsn, from string, err error) {
|
||||
dsn, from, err = settingsFor(name)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
port, err := envfile.Port(Variable(name))
|
||||
if err != nil || port == "" {
|
||||
return dsn, from, err
|
||||
}
|
||||
moved, err := envfile.WithPort(dsn, port)
|
||||
if err != nil {
|
||||
// The variable and the port are named; the connection string is not, for the same reason
|
||||
// as everywhere else in this file.
|
||||
return "", "", fmt.Errorf(
|
||||
"%s says this machine put the %s store on port %s, and the connection settings in %s "+
|
||||
"could not be read as something with a port in them: %w",
|
||||
PortVariable(name), name, port, from, err)
|
||||
}
|
||||
return moved, from + ", on port " + port + " as " + PortVariable(name) + " says", nil
|
||||
}
|
||||
|
||||
// Context is which context this store belongs to.
|
||||
func (s *Store) Context() string { return s.context }
|
||||
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
# Lab bed: the hub adopts the predecessor's tunnel (novox/hq ADR 0105)
|
||||
|
||||
A scenario and an integration-test skeleton for the mesh-lab repository, kept here because this
|
||||
branch changes only the controller and the host. Move `adopt-the-tunnel.yml` to
|
||||
`mesh-lab/scenarios/` and `adopt-the-tunnel.test.ts` to `mesh-lab/test/integration/` when the
|
||||
feature lands; neither has been run. The skeleton follows `adoption.test.ts` and reuses its
|
||||
harness. Documentation addresses throughout; the bed is node-agnostic.
|
||||
|
||||
## The bed, precisely
|
||||
|
||||
Three machines on one public segment, `hosting` (192.0.2.0/24), inbound allowed on all (the
|
||||
anchor's firewall is the predecessor's, installed by the bed):
|
||||
|
||||
| machine | address | role |
|
||||
|---|---|---|
|
||||
| `anchor` | 192.0.2.10 | the machine in use: the predecessor's hub, then the mesh adopted on it |
|
||||
| `peer-a` | 192.0.2.20 | a predecessor machine: reaches a service on the anchor through the tunnel; later **enrols and keeps its address** |
|
||||
| `peer-b` | 192.0.2.30 | a second predecessor machine: reaches the same service; **never enrols** — the peer that must notice nothing throughout |
|
||||
| `fresh` | 192.0.2.40 | a new machine: enrols later and **gets a fresh address from the same range** |
|
||||
|
||||
**Prepared the way the predecessor leaves a hub** (before genesis, by the bed, on `anchor`):
|
||||
|
||||
- `wireguard-tools` installed; a keypair made on each of `anchor`, `peer-a`, `peer-b`.
|
||||
- `/etc/wireguard/wg0.conf` on the anchor: `[Interface]` `PrivateKey = <anchor's>`,
|
||||
`ListenPort = 51900`, `Address = 10.10.0.1/24`; two `[Peer]` sections — `peer-a`'s public
|
||||
key with `AllowedIPs = 10.10.0.2/32`, `peer-b`'s with `AllowedIPs = 10.10.0.3/32`. Raised with
|
||||
`systemctl enable --now wg-quick@wg0`. **10.10.0.0/24 is deliberately not the mesh's default
|
||||
range** (10.42.0.0/16), so a hub address in 10.10.0.0/24 can only have come from the tunnel.
|
||||
- `wg0.conf` on each peer: its own key, `Address = 10.10.0.2/24` (resp. `.3/24`), one
|
||||
`[Peer]` — the anchor's public key, `Endpoint = 192.0.2.10:51900`,
|
||||
`AllowedIPs = 10.10.0.0/24`, `PersistentKeepalive = 25`. Raised the same way.
|
||||
- A service on the anchor the peers reach **only over the tunnel**: a container publishing
|
||||
`10.10.0.1:8081:80` (bound to the tunnel address, so a call from 192.0.2.20 to 10.10.0.1:8081
|
||||
proves the tunnel carried it). Under a name no catalogue module uses — this bed is about the
|
||||
tunnel, not about taking a service.
|
||||
- The predecessor's firewall (`ufw`) allowing `51900/udp` and `22/tcp`, denying the rest — as ADR
|
||||
0100's bed prepares it.
|
||||
- A record of the anchor's `wg0` public key and of `sha256sum /etc/wireguard/wg0.conf`, taken
|
||||
before genesis, for the assertions below.
|
||||
|
||||
**Genesis**, adopted, on the anchor: `mesh-bootstrap --adopted --node anchor --site hosting
|
||||
--endpoint 192.0.2.10:51900 …` — no `--hub-port`, no `--overlay-range`, no `--tunnel`: the
|
||||
installer finds `wg0` itself (one interface besides `mesh0`) and takes its port and range. The
|
||||
bed asserts genesis **says** it found and took the tunnel.
|
||||
|
||||
**Review changes (2026-09-24).** A spoke's `wg0.conf` names one peer — the hub — routed the
|
||||
whole range; the controller skips range-routed peers, so T2's enrolment carries no peer from the
|
||||
spoke. A hub that enrolled *before* this feature (a generated key) takes the tunnel over without
|
||||
re-enrolling: `mesh-host overlay take --tunnel wg0` on the machine rekeys the overlay key only and
|
||||
sends a signed rekey; the bed adds **R0** for it below. A takeover is composed only for a hub
|
||||
placed at the tunnel's address on the tunnel's port, and the host stops nothing until the declared
|
||||
interface matches the found one and the key file holds the found key; a mesh interface that fails
|
||||
to start gives the found unit back. The host's account has three states: `not-taken`, `taken`,
|
||||
`down`.
|
||||
|
||||
## Assertions, in the record's order
|
||||
|
||||
- **R0 — a hub enrolled with its own key takes the tunnel over by rekeying.** Genesis is run
|
||||
adopted *without* the tunnel being found (the bed stops `wg-quick@wg0` for the run, so the
|
||||
installer sees no tunnel, then starts it again — the pre-feature shape). Then on the anchor:
|
||||
`mesh-host overlay take --tunnel wg0`; `node show anchor` says "tunnel found wg0 …"; `overlay
|
||||
place anchor --hub --endpoint 192.0.2.10:51900 --site hosting` (with `:51820` first, which must
|
||||
be refused naming 51900); `plan anchor --json` names `Address = 10.10.0.1/32`, `ListenPort =
|
||||
51900`, two `/32` peers, `takes-over` wg0 and nothing in 10.42.0.0/16; then `push anchor --wait
|
||||
2m` and T1's assertions hold. `overlay take` run a second time is refused by the controller as
|
||||
stale and changes nothing.
|
||||
|
||||
- **T1 — the tunnel changes hands and the peers notice nothing.** After genesis and the push
|
||||
that raises the private network on the anchor:
|
||||
- `wg show interfaces` on the anchor lists `mesh0` and not `wg0`;
|
||||
`systemctl is-active wg-quick@wg0` is inactive and `is-enabled` disabled;
|
||||
- `/etc/wireguard/wg0.conf` is on disk with the recorded digest (kept, never flushed), and
|
||||
`node show anchor` names where its original was kept;
|
||||
- `wg show mesh0 public-key` is the anchor's recorded `wg0` public key; `wg show mesh0
|
||||
listen-port` is 51900; `ip -o addr show dev mesh0` carries `10.10.0.1`; `wg show mesh0 peers`
|
||||
lists both peers' public keys with their `/32` allowed addresses;
|
||||
- a loop on `peer-a` and `peer-b` calling `http://10.10.0.1:8081/` every second, started before
|
||||
genesis, records **no window of failure longer than one WireGuard re-handshake** (measure and
|
||||
assert an upper bound — the switch is one unit stop plus one unit start on the anchor); the
|
||||
peers' `wg0.conf` digests are unchanged; the peers' `wg show wg0 latest-handshakes` advance
|
||||
after the switch.
|
||||
- `overlay show` lists `anchor` as the hub over the tunnel it took over, and both peers under
|
||||
"peers of the tunnel … not nodes of the mesh", not yet enrolled.
|
||||
- **T2 — a peer enrols and keeps its address.** On `peer-a`: `node add peer-a --adopted`,
|
||||
token issued, `mesh-host enrol --token …` **with the broker reached over the tunnel** (the
|
||||
broker address in the token is `10.10.0.1:<bus>`, which only the tunnel routes); then `overlay
|
||||
place peer-a --site house` and a push. Assert: `node show peer-a` says a tunnel `wg0` was
|
||||
found and `overlay show` puts `peer-a` at **10.10.0.2**; the carried-peers list now says
|
||||
`enrolled as peer-a`; the anchor's `mesh0` still has exactly one entry for `peer-a`'s key;
|
||||
`peer-a`'s `wg0` is down and `mesh0` up with the same key; `peer-a` still reaches
|
||||
`10.10.0.1:8081` and `peer-b` still does too, uninterrupted.
|
||||
- **T3 — a new machine gets a fresh address from the same range.** `fresh` enrols converged
|
||||
(no tunnel), is placed, pushed. Assert its address is **10.10.0.4** (`.1` hub, `.2` and `.3`
|
||||
the tunnel's), that it reaches `10.10.0.1` (the hub) and `10.10.0.2` (the enrolled peer) —
|
||||
`ping -c1` over `mesh0` — and that `peer-b`, never enrolled, is still served.
|
||||
- **T4 — nothing derived from the address is stale.** `plan anchor --json` and `plan peer-a
|
||||
--json` (and the rendered `/etc/hosts` on each node) name `10.10.0.1` for the anchor and
|
||||
`10.10.0.2` for `peer-a`, and no address in `10.42.0.0/16`; the broker address handed to a
|
||||
module issued on `peer-a` is `anchor.internal:<bus>` resolving to `10.10.0.1`; the same after a
|
||||
second `push` of every node, byte for byte.
|
||||
- **N — the narrowed ADR 0100 check.** On `fresh`, a converged genesis dry-run with
|
||||
`--overlay-range 10.10.0.0/24` while a *second* tunnel the bed raises there (`wg1` at
|
||||
10.10.0.9/24, not adopted because the node is converged) is up, is refused naming `wg1` —
|
||||
the non-overlap rule still applies where a tunnel is not adopted.
|
||||
|
||||
## What is not asserted here
|
||||
|
||||
- Taking a service over the tunnel (ADR 0100's bed does that).
|
||||
- IPv6 tunnels: the parser reads them, the bed prepares only IPv4.
|
||||
@@ -1,174 +0,0 @@
|
||||
/**
|
||||
* THE HUB ADOPTS THE PREDECESSOR'S TUNNEL (novox/hq ADR 0105). Skeleton — NOT YET RUN.
|
||||
*
|
||||
* The bed and every assertion are described in README.md beside this file; the numbered
|
||||
* assertions here are that document's. Follows adoption.test.ts: same harness, same `on`/`must`
|
||||
* helpers, same genesis wrapper.
|
||||
*
|
||||
* MESH_LAB_INCUS='sudo -n incus'
|
||||
* MESH_LAB_HOST_BINARY=.../mesh-host/mesh-host
|
||||
* MESH_LAB_BUNDLE=.../mesh-host/examples/foundation-first-node.lock
|
||||
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||
*/
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync } from "node:fs";
|
||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||
import { raise } from "../../src/lifecycle/raise.ts";
|
||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||
import { hostBinaryPath } from "../../src/lifecycle/place.ts";
|
||||
import { labIsUsable, destroyAll, catalogueIsPresent } from "./harness.ts";
|
||||
import { genesis } from "./genesis.ts";
|
||||
|
||||
const capability = await labIsUsable();
|
||||
const binary = hostBinaryPath();
|
||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||
const skip = !capability.usable ? `lab not usable: ${capability.why}`
|
||||
: !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||
: !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle"
|
||||
: catalogueIsPresent();
|
||||
|
||||
const SCENARIO = "adopt-the-tunnel";
|
||||
const ANCHOR = "anchor", PEER_A = "peer-a", PEER_B = "peer-b", FRESH = "fresh";
|
||||
const TUNNEL = { port: 51900, range: "10.10.0.0/24", hub: "10.10.0.1", a: "10.10.0.2", b: "10.10.0.3", fresh: "10.10.0.4" };
|
||||
const SERVICE = `http://${TUNNEL.hub}:8081/`;
|
||||
|
||||
let instanceId = "";
|
||||
let wg0Key = ""; // the anchor's wg0 public key, recorded before genesis
|
||||
let wg0Digest = ""; // sha256 of /etc/wireguard/wg0.conf before genesis
|
||||
|
||||
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||
const { stdout } = await exec(instanceId, machine, ["sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`], timeoutMs);
|
||||
const marker = stdout.lastIndexOf("__exit=");
|
||||
if (marker < 0) return { out: stdout, ok: false };
|
||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||
}
|
||||
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||
const { out, ok } = await on(machine, command, timeoutMs);
|
||||
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||
return out;
|
||||
}
|
||||
/** The controller, a container on the anchor. */
|
||||
async function control(args: string): Promise<string> {
|
||||
return must(ANCHOR, `docker exec mesh-controller mesh-controller ${args}`);
|
||||
}
|
||||
|
||||
/** Prepares a machine the way the predecessor leaves it: a keypair and a wg0 — see README.md. */
|
||||
async function predecessorTunnelOn(machine: string, conf: (keys: Record<string, string>) => string, keys: Record<string, string>): Promise<void> {
|
||||
await must(machine, "apt-get install -y wireguard-tools >/dev/null 2>&1 || pacman -S --noconfirm wireguard-tools >/dev/null");
|
||||
await must(machine, `umask 077; printf '%s' '${conf(keys)}' > /etc/wireguard/wg0.conf`);
|
||||
await must(machine, "systemctl enable --now wg-quick@wg0");
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
if (skip) return;
|
||||
await destroyAll(SCENARIO);
|
||||
const scenario = loadScenario(`scenarios/${SCENARIO}.yml`);
|
||||
instanceId = (await raise(scenario)).instanceId;
|
||||
|
||||
// Keys for the three predecessor machines, made where they live and never moved.
|
||||
const keys: Record<string, string> = {};
|
||||
for (const m of [ANCHOR, PEER_A, PEER_B]) {
|
||||
await must(m, "umask 077; wg genkey > /etc/wireguard/predecessor.key");
|
||||
keys[m] = (await must(m, "wg pubkey < /etc/wireguard/predecessor.key")).trim();
|
||||
}
|
||||
await predecessorTunnelOn(ANCHOR, k => [
|
||||
"[Interface]", `PrivateKey = $(cat /etc/wireguard/predecessor.key)`, `ListenPort = ${TUNNEL.port}`, `Address = ${TUNNEL.hub}/24`,
|
||||
"[Peer]", `PublicKey = ${k[PEER_A]}`, `AllowedIPs = ${TUNNEL.a}/32`,
|
||||
"[Peer]", `PublicKey = ${k[PEER_B]}`, `AllowedIPs = ${TUNNEL.b}/32`,
|
||||
].join("\n"), keys);
|
||||
for (const [m, addr] of [[PEER_A, TUNNEL.a], [PEER_B, TUNNEL.b]] as const) {
|
||||
await predecessorTunnelOn(m, k => [
|
||||
"[Interface]", `PrivateKey = $(cat /etc/wireguard/predecessor.key)`, `Address = ${addr}/24`,
|
||||
"[Peer]", `PublicKey = ${k[ANCHOR]}`, `Endpoint = 192.0.2.10:${TUNNEL.port}`, `AllowedIPs = ${TUNNEL.range}`, "PersistentKeepalive = 25",
|
||||
].join("\n"), keys);
|
||||
}
|
||||
// A service reachable only over the tunnel, under a name no catalogue module uses.
|
||||
await must(ANCHOR, `docker run -d --name predecessor-page -p ${TUNNEL.hub}:8081:80 nginx:alpine`);
|
||||
// The predecessor's firewall: the tunnel's port and ssh, nothing else (as ADR 0100's bed).
|
||||
await must(ANCHOR, `ufw --force reset >/dev/null; ufw default deny incoming; ufw allow 22/tcp; ufw allow ${TUNNEL.port}/udp; ufw --force enable`);
|
||||
for (const m of [PEER_A, PEER_B]) assert.ok((await on(m, `curl -fsS --max-time 3 ${SERVICE}`)).ok, `${m} does not reach the service over the tunnel before genesis`);
|
||||
|
||||
wg0Key = (await must(ANCHOR, "wg show wg0 public-key")).trim();
|
||||
wg0Digest = (await must(ANCHOR, "sha256sum /etc/wireguard/wg0.conf")).split(" ")[0];
|
||||
// The probe the peers keep running through the switch: one call a second, failures counted.
|
||||
for (const m of [PEER_A, PEER_B]) await must(m, `nohup sh -c 'while :; do curl -fsS --max-time 1 ${SERVICE} >/dev/null 2>&1 || date +%s >> /tmp/failed; sleep 1; done' >/dev/null 2>&1 &`);
|
||||
});
|
||||
|
||||
after(async () => { if (instanceId) await destroy(instanceId); });
|
||||
|
||||
test("T1 — adopted, the tunnel changes hands and the peers notice nothing", { skip }, async () => {
|
||||
const ran = await genesis({ instanceId, machine: ANCHOR, node: ANCHOR, site: "hosting",
|
||||
flags: ["--adopted", "--endpoint", `192.0.2.10:${TUNNEL.port}`] } as never);
|
||||
assert.match(ran.said, /tunnel\s+wg0/i, `genesis did not say it found and took the tunnel:\n${ran.said}`);
|
||||
|
||||
const ifaces = await must(ANCHOR, "wg show interfaces");
|
||||
assert.match(ifaces, /\bmesh0\b/); assert.doesNotMatch(ifaces, /\bwg0\b/);
|
||||
assert.equal((await on(ANCHOR, "systemctl is-active wg-quick@wg0")).out.trim(), "inactive");
|
||||
assert.equal((await on(ANCHOR, "systemctl is-enabled wg-quick@wg0")).out.trim(), "disabled");
|
||||
assert.equal((await must(ANCHOR, "sha256sum /etc/wireguard/wg0.conf")).split(" ")[0], wg0Digest, "the found configuration was changed or flushed");
|
||||
assert.equal((await must(ANCHOR, "wg show mesh0 public-key")).trim(), wg0Key, "the mesh's interface is not up with the found key");
|
||||
assert.equal((await must(ANCHOR, "wg show mesh0 listen-port")).trim(), String(TUNNEL.port));
|
||||
assert.match(await must(ANCHOR, "ip -o addr show dev mesh0"), new RegExp(TUNNEL.hub.replaceAll(".", "\\.")));
|
||||
const peers = await must(ANCHOR, "wg show mesh0 allowed-ips");
|
||||
assert.match(peers, new RegExp(`${TUNNEL.a}/32`)); assert.match(peers, new RegExp(`${TUNNEL.b}/32`));
|
||||
|
||||
for (const m of [PEER_A, PEER_B]) {
|
||||
const failed = (await on(m, "cat /tmp/failed 2>/dev/null | wc -l")).out.trim();
|
||||
assert.ok(Number(failed) <= 5, `${m} lost the service for ${failed} seconds through the switch`);
|
||||
assert.ok((await on(m, `curl -fsS --max-time 3 ${SERVICE}`)).ok, `${m} does not reach the service after the switch`);
|
||||
}
|
||||
const shown = await control("overlay show");
|
||||
assert.match(shown, /anchor.*hub.*took over on wg0/);
|
||||
assert.match(shown, /peers of the tunnel/); assert.match(shown, /not yet enrolled/);
|
||||
assert.match(await control(`node show ${ANCHOR}`), /tunnel found\s+wg0 on port 51900/);
|
||||
});
|
||||
|
||||
test("T2 — a peer enrols over the tunnel and keeps its address", { skip }, async () => {
|
||||
await control(`node add ${PEER_A} --adopted`);
|
||||
const token = (await control(`token issue --node ${PEER_A}`)).match(/token\s+(\S+)/)?.[1] ?? "";
|
||||
// The token's broker address is the hub's tunnel address: only the tunnel routes it.
|
||||
await must(PEER_A, `mesh-host enrol --token '${token}'`);
|
||||
await control(`overlay place ${PEER_A} --site house`);
|
||||
await control(`assign ${PEER_A} networking`);
|
||||
await control(`push ${PEER_A} --wait 2m`);
|
||||
|
||||
assert.match(await control("overlay show"), new RegExp(`${PEER_A}\\s+${TUNNEL.a.replaceAll(".", "\\.")}`));
|
||||
assert.match(await control("overlay show"), new RegExp(`enrolled as ${PEER_A}`));
|
||||
const onHub = await must(ANCHOR, "wg show mesh0 allowed-ips");
|
||||
assert.equal(onHub.split("\n").filter(l => l.includes(`${TUNNEL.a}/32`)).length, 1, "the enrolled peer's key appears twice on the hub");
|
||||
assert.doesNotMatch(await must(PEER_A, "wg show interfaces"), /\bwg0\b/);
|
||||
assert.ok((await on(PEER_A, `curl -fsS --max-time 3 ${SERVICE}`)).ok);
|
||||
assert.ok((await on(PEER_B, `curl -fsS --max-time 3 ${SERVICE}`)).ok, "the peer that never enrols lost the service");
|
||||
});
|
||||
|
||||
test("T3 — a new machine gets a fresh address from the same range", { skip }, async () => {
|
||||
await control(`node add ${FRESH}`);
|
||||
const token = (await control(`token issue --node ${FRESH}`)).match(/token\s+(\S+)/)?.[1] ?? "";
|
||||
await must(FRESH, `mesh-host enrol --token '${token}'`);
|
||||
await control(`overlay place ${FRESH} --nothing`);
|
||||
await control(`assign ${FRESH} networking`);
|
||||
await control(`push ${FRESH} --wait 2m`);
|
||||
assert.match(await control("overlay show"), new RegExp(`${FRESH}\\s+${TUNNEL.fresh.replaceAll(".", "\\.")}`));
|
||||
assert.ok((await on(FRESH, `ping -c1 -W2 ${TUNNEL.hub}`)).ok, "the new machine does not reach the hub");
|
||||
assert.ok((await on(FRESH, `ping -c1 -W2 ${TUNNEL.a}`)).ok, "the new machine does not reach the enrolled peer");
|
||||
assert.ok((await on(PEER_B, `curl -fsS --max-time 3 ${SERVICE}`)).ok);
|
||||
});
|
||||
|
||||
test("T4 — nothing derived from the address is stale", { skip }, async () => {
|
||||
for (const n of [ANCHOR, PEER_A, FRESH]) {
|
||||
const plan = await control(`plan ${n} --json`);
|
||||
assert.doesNotMatch(plan, /10\.42\./, `${n}'s plan names the mesh's default range`);
|
||||
assert.match(plan, new RegExp(TUNNEL.hub.replaceAll(".", "\\.")));
|
||||
assert.match(await must(n, "cat /etc/hosts"), new RegExp(`${TUNNEL.hub.replaceAll(".", "\\.")}\\s+anchor\\.internal`));
|
||||
}
|
||||
const first = await control(`plan ${PEER_A} --json`);
|
||||
await control("push");
|
||||
assert.equal(await control(`plan ${PEER_A} --json`), first, "a push changed what the plan says");
|
||||
});
|
||||
|
||||
test("N — where a tunnel is not adopted, the ranges must still differ (ADR 0100)", { skip }, async () => {
|
||||
await must(FRESH, "umask 077; printf '[Interface]\\nPrivateKey = %s\\nAddress = 10.10.0.9/24\\n' \"$(wg genkey)\" > /etc/wireguard/wg1.conf; systemctl start wg-quick@wg1");
|
||||
const ran = await genesis({ instanceId, machine: FRESH, node: FRESH, flags: ["--dry-run", "--overlay-range", TUNNEL.range], attempts: 1, verify: false, hostService: false } as never);
|
||||
assert.match(ran.said, /wg1/, `a converged genesis did not refuse the overlapping tunnel it does not adopt:\n${ran.said}`);
|
||||
});
|
||||
@@ -1,50 +0,0 @@
|
||||
# THE HUB ADOPTS THE PREDECESSOR'S TUNNEL (novox/hq ADR 0105). See README.md beside this file.
|
||||
#
|
||||
# hosting (public)
|
||||
# anchor 192.0.2.10 the predecessor's hub: wg0 on udp/51900, 10.10.0.1/24, two peers; then the
|
||||
# mesh adopted on it, taking the tunnel over
|
||||
# peer-a 192.0.2.20 a predecessor machine reaching a service on the anchor over the tunnel;
|
||||
# enrols later and keeps 10.10.0.2
|
||||
# peer-b 192.0.2.30 a predecessor machine that never enrols: must notice nothing, ever
|
||||
# fresh 192.0.2.40 a new machine: enrols later and gets 10.10.0.4
|
||||
#
|
||||
# inbound: allow on every machine — the anchor's firewall is the predecessor's, installed by the bed.
|
||||
scenario: adopt-the-tunnel
|
||||
|
||||
segments:
|
||||
hosting:
|
||||
kind: public
|
||||
cidr: [192.0.2.0/24]
|
||||
|
||||
machines:
|
||||
anchor:
|
||||
at: { segment: hosting, address: [192.0.2.10] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 12GiB
|
||||
cpus: 6
|
||||
disk: 60GiB
|
||||
peer-a:
|
||||
at: { segment: hosting, address: [192.0.2.20] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 3GiB
|
||||
cpus: 2
|
||||
disk: 20GiB
|
||||
peer-b:
|
||||
at: { segment: hosting, address: [192.0.2.30] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 2GiB
|
||||
cpus: 2
|
||||
disk: 15GiB
|
||||
fresh:
|
||||
at: { segment: hosting, address: [192.0.2.40] }
|
||||
egress: true
|
||||
inbound: allow
|
||||
memory: 3GiB
|
||||
cpus: 2
|
||||
disk: 20GiB
|
||||
|
||||
place:
|
||||
all: [host, runtime]
|
||||
+1
-7
@@ -42,13 +42,7 @@
|
||||
"MESH_STORE_LICENCES_FILE": "/run/secrets/licences",
|
||||
"MESH_BROKER_AMQP_FILE": "/run/secrets/broker",
|
||||
"MESH_BROKER_MANAGEMENT_FILE": "/run/secrets/broker-management",
|
||||
"MESH_BROKER_ADDRESS_FILE": "/run/secrets/broker-address",
|
||||
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}",
|
||||
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}"
|
||||
"MESH_BROKER_ADDRESS_FILE": "/run/secrets/broker-address"
|
||||
},
|
||||
"volumes": [
|
||||
"mesh-broker-tls:/broker-tls:ro",
|
||||
|
||||
Reference in New Issue
Block a user