Declare openings and a refusal-only guard on an adopted node in place of the filter (hq ADR 0100)

This commit is contained in:
2026-09-22 17:21:44 +02:00
parent a86a6c2974
commit c3b1617693
6 changed files with 526 additions and 13 deletions
+12 -7
View File
@@ -485,16 +485,21 @@ func declarationWith(ctx context.Context, open *stores, node string,
break
}
composed, err := plan.Compose(catalogue.Rendering{
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept})
// Whether this node is adopted (novox/hq ADR 0100): then the found firewall stays in force, and
// the declaration carries openings and the mesh's guard in place of a filter.
record, err := inv.NodeByName(ctx, node)
if err != nil {
return sendable{}, err
}
// Whether this node is adopted, and what was taken on it, said in every declaration it is
// sent from this one place (novox/hq ADR 0100).
adoption, err := adoptionOf(ctx, inv, node, plan, composed)
composed, err := plan.Compose(catalogue.Rendering{
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted})
if err != nil {
return sendable{}, err
}
// And what was taken on it, said in every declaration it is sent from this one place.
adoption, err := adoptionOf(ctx, inv, record, plan, composed)
if err != nil {
return sendable{}, err
}
+2 -6
View File
@@ -41,16 +41,12 @@ func (s sendable) Body() ([]byte, error) {
}
// adoptionOf is the envelope for a node, nil when it is converged.
func adoptionOf(ctx context.Context, inv *inventory.Inventory, node string,
func adoptionOf(ctx context.Context, inv *inventory.Inventory, record inventory.Node,
plan catalogue.Resolution, composed catalogue.Composed) (*adoptionEnvelope, error) {
record, err := inv.NodeByName(ctx, node)
if err != nil {
return nil, err
}
if !record.Adopted {
return nil, nil
}
taken, err := inv.Taken(ctx, node)
taken, err := inv.Taken(ctx, record.Name)
if err != nil {
return nil, err
}
+218
View File
@@ -0,0 +1,218 @@
package catalogue
import (
"fmt"
"sort"
"strconv"
"strings"
)
// What an adopted node is declared in place of a filter (novox/hq ADR 0100).
//
// On an adopted node the firewall found on the machine stays in force: the mesh loads no table
// that drops by default or holds an accept. What the mesh needs reachable is declared as
// openings, which the host converges through the found firewall in its own terms; and the mesh
// guards its own foundation ports itself, in a table that only refuses.
// AdoptionPrefix is the id prefix of what the mesh declares of its own on an adopted node. It is
// never a module's, so none of it is ever held as found.
const AdoptionPrefix = "adoption."
// Where an opening admits from, on the wire.
const (
OpeningFromEverywhere = "everywhere"
OpeningFromMesh = "mesh"
)
// The two paths a packet reaches a port by: received by the machine, or forwarded to a
// container that publishes it.
const (
PathIncoming = "incoming"
PathForwarded = "forwarded"
)
// Guard resources: the refusal-only table, the unit that loads it, and that unit running.
const (
GuardPath = "/etc/mesh/guard.nft"
GuardUnit = "mesh-guard.service"
// GuardUnitPath is where the unit is written.
GuardUnitPath = "/etc/systemd/system/" + GuardUnit
)
// GuardID, GuardUnitID and GuardRunningID are the guard's resource identities. The installer
// raises the same three on an adopted genesis, so the first push finds them already there.
func GuardID() string { return AdoptionPrefix + "guard" }
func GuardUnitID() string { return AdoptionPrefix + "guard-unit" }
func GuardRunningID() string { return AdoptionPrefix + "guard-running" }
// OpeningID is an opening's resource identity: its protocol, port and path say what it is.
func OpeningID(protocol string, port int, path string) string {
return fmt.Sprintf("%sopening-%s-%d-%s", AdoptionPrefix, protocol, port, path)
}
// Openings are what the mesh needs reachable on an adopted node, from the same inputs as the
// filter it would load were the node converged, each from where that filter would admit it.
//
// `rules` is Filtering's answer — every module's listens, the hub's port, the per-node exposure —
// and `foundation` is the ports the mesh itself needs, from everywhere. A rule for this machine
// only opens nothing. `published` maps a machine port a container publishes to the container's
// port: a published port is forwarded, not received, so its opening names the forwarded path and
// the port the packet is forwarded to.
func Openings(rules []Rule, foundation []int, published map[string]map[int]int) []map[string]any {
type key struct {
protocol string
port int
}
from := map[key]string{}
var order []key
widen := func(k key, f string) {
was, seen := from[k]
if !seen {
order = append(order, k)
}
if !seen || was != OpeningFromEverywhere {
from[k] = f
}
}
for _, rule := range rules {
switch rule.From {
case FromEverywhere:
widen(key{rule.Protocol, rule.Port}, OpeningFromEverywhere)
case FromMesh:
widen(key{rule.Protocol, rule.Port}, OpeningFromMesh)
}
}
for _, port := range foundation {
widen(key{"tcp", port}, OpeningFromEverywhere)
}
sort.Slice(order, func(a, b int) bool {
if order[a].port != order[b].port {
return order[a].port < order[b].port
}
return order[a].protocol < order[b].protocol
})
out := make([]map[string]any, 0, len(order))
for _, k := range order {
opening := map[string]any{"type": "opening", "port": k.port, "protocol": k.protocol,
"from": from[k]}
if to, forwarded := published[k.protocol][k.port]; forwarded {
opening["id"] = OpeningID(k.protocol, k.port, PathForwarded)
opening["path"] = PathForwarded
opening["to"] = to
} else {
opening["id"] = OpeningID(k.protocol, k.port, PathIncoming)
opening["path"] = PathIncoming
}
out = append(out, opening)
}
return out
}
// Published is every port the given containers publish on the machine, by protocol and machine
// port, mapped to the container's own port. A mapping bound to loopback is left out: nothing off
// the machine reaches it, forwarded or not.
func Published(resources []map[string]any) map[string]map[int]int {
out := map[string]map[int]int{}
for _, r := range resources {
if fmt.Sprint(r["type"]) != "container" {
continue
}
listed, _ := r["ports"].([]any)
for _, entry := range listed {
written := strings.TrimSpace(fmt.Sprint(entry))
protocol := "tcp"
if cut := strings.LastIndex(written, "/"); cut >= 0 {
protocol = written[cut+1:]
written = written[:cut]
}
parts := strings.Split(written, ":")
if len(parts) < 2 {
continue
}
if len(parts) == 3 && (parts[0] == "127.0.0.1" || parts[0] == "localhost" ||
parts[0] == "[::1]") {
continue
}
outer, err := strconv.Atoi(parts[len(parts)-2])
if err != nil {
continue
}
inner, err := strconv.Atoi(parts[len(parts)-1])
if err != nil {
continue
}
if out[protocol] == nil {
out[protocol] = map[int]int{}
}
out[protocol][outer] = inner
}
}
return out
}
// AsGuard renders the mesh's refusal-only table for the given machine ports.
//
// It passes everything by default and holds nothing but a refusal, so it cannot close anything
// the machine serves; and it is the mesh's own table, so the found firewall reloading does not
// touch it. It refuses the ports except from the machine itself — its loopback and the container
// runtime's own networks — and from the private network, known by the interface a packet arrives
// on and never by its source address. At prerouting, ahead of the runtime's destination
// translation, so it matches the port the packet was sent to; in the inet family, so both address
// families.
//
// The same text the installer raises on an adopted genesis; a test holds both to it.
func AsGuard(ports []int) string {
sorted := append([]int{}, ports...)
sort.Ints(sorted)
listed := make([]string, len(sorted))
for i, p := range sorted {
listed[i] = strconv.Itoa(p)
}
var b strings.Builder
b.WriteString("table inet mesh_guard {}\n")
b.WriteString("delete table inet mesh_guard\n")
b.WriteString("table inet mesh_guard {\n")
b.WriteString("\tchain prerouting {\n")
b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n")
fmt.Fprintf(&b, "\t\tiifname != \"lo\" iifname != \"docker0\" iifname != \"br-*\" "+
"iifname != \"mesh0\" tcp dport { %s } drop\n", strings.Join(listed, ", "))
b.WriteString("\t}\n")
b.WriteString("}\n")
return b.String()
}
// GuardUnitText is the unit that loads the guard. Stopping it deletes only its own table: never
// a flush, which would take the container runtime's rules and the found firewall with it.
func GuardUnitText() string {
return "[Unit]\n" +
"Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" +
"After=network-pre.target\n" +
"Wants=network-pre.target\n" +
"\n" +
"[Service]\n" +
"Type=oneshot\n" +
"RemainAfterExit=yes\n" +
"ExecStart=nft -f " + GuardPath + "\n" +
"ExecReload=nft -f " + GuardPath + "\n" +
"ExecStop=nft delete table inet mesh_guard\n" +
"\n" +
"[Install]\n" +
"WantedBy=multi-user.target\n"
}
// GuardResources are the guard as three resources of the existing kinds: the table, the unit, and
// the unit running, restarted when the table changes. Nothing when there is nothing to guard: an
// empty set is not a table nft loads.
func GuardResources(ports []int) []map[string]any {
if len(ports) == 0 {
return nil
}
return []map[string]any{
{"id": GuardID(), "type": "file", "path": GuardPath, "content": AsGuard(ports),
"mode": "0644"},
{"id": GuardUnitID(), "type": "file", "path": GuardUnitPath, "content": GuardUnitText(),
"mode": "0644"},
{"id": GuardRunningID(), "type": "service", "unit": GuardUnit, "state": "running",
"boot": "enabled", "restart-on": []any{GuardID(), GuardUnitID()}},
}
}
+223
View File
@@ -0,0 +1,223 @@
package catalogue
import (
"encoding/json"
"reflect"
"strings"
"testing"
)
// novox/hq ADR 0100: on an adopted node the found firewall stays in force. The mesh declares
// openings where it would have loaded a filter, and guards its own ports in a table that only
// refuses.
// hub is the private network's generator on the hub: it opens the hub's port from anywhere.
type hub struct{}
func (hub) Resources(string) ([]map[string]any, bool, error) {
return []map[string]any{{"id": "config", "type": "file", "path": "/etc/wireguard/mesh0.conf",
"content": "[Interface]\n"}}, true, nil
}
func (hub) Listens(string) ([]Listening, error) {
return []Listening{{Port: 51820, Protocol: "udp", From: FromEverywhere}}, nil
}
// anAdoptedAnchor is the control-node's set: the store, the bus, the registry, the private network,
// a served module and the filter module.
func anAdoptedAnchor() Resolution {
return Resolution{Node: "anchor", Modules: []Manifest{
{Module: "network", Computed: "overlay"},
{Module: "postgres", Guards: []int{5432},
Listens: []Listening{{Port: 5432, From: FromMesh}},
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
"ports": []any{"5432:5432"}}}},
{Module: "lavinmq", Guards: []int{15672},
Listens: []Listening{{Port: 5671, From: FromMesh}, {Port: 5672, From: FromMesh}},
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}}}},
{Module: "distribution",
Listens: []Listening{{Port: 5000, From: FromMesh}},
Resources: []map[string]any{{"id": "store", "type": "container", "name": "registry",
"ports": []any{"5000"}}}},
{Module: "hello-web",
Listens: []Listening{{Port: 8080, From: FromEverywhere}},
Resources: []map[string]any{{"id": "server", "type": "container", "name": "hello-web",
"ports": []any{"8080"}}}},
{Module: "helper", Listens: []Listening{{Port: 9000, From: FromMachine}}},
{Module: "nftables", Filtering: &Filtering{Into: "/etc/nftables.conf"},
Resources: []map[string]any{{"id": "load", "type": "service", "unit": "mesh-filter.service",
"state": "running", "restart-on": []any{"filtering"}}}},
}}
}
func anchorRendering(adopted bool) Rendering {
return Rendering{
Generators: map[string]Generator{"overlay": hub{}},
Ports: map[string]map[int]int{"distribution": {5000: 5000}, "hello-web": {8080: 20001}},
Settings: SettingsBy{"distribution": {{From: "node anchor",
Values: map[string]any{ExposeSetting: map[string]any{"5000": FromEverywhere}}}}},
Mesh: []string{"10.42.0.1"},
Foundation: []int{5671},
Adopted: adopted,
}
}
func byID(resources []map[string]any) map[string]map[string]any {
out := map[string]map[string]any{}
for _, r := range resources {
out[r["id"].(string)] = r
}
return out
}
func TestAnAdoptedNodeIsDeclaredOpeningsFromTheSameInputsAsTheFilter(t *testing.T) {
composed, err := anAdoptedAnchor().Compose(anchorRendering(true))
if err != nil {
t.Fatal(err)
}
got := byID(composed.Resources)
want := map[string]map[string]any{
// The hub's port, from anywhere, received.
"adoption.opening-udp-51820-incoming": {"port": 51820, "protocol": "udp",
"from": "everywhere", "path": "incoming"},
// The store's port from the private network only, and forwarded: a container publishes it.
"adoption.opening-tcp-5432-forwarded": {"port": 5432, "protocol": "tcp", "from": "mesh",
"path": "forwarded", "to": 5432},
// The bus from anywhere: a node enrols over it before it has a private address.
"adoption.opening-tcp-5671-forwarded": {"port": 5671, "protocol": "tcp",
"from": "everywhere", "path": "forwarded", "to": 5671},
"adoption.opening-tcp-5672-forwarded": {"port": 5672, "protocol": "tcp", "from": "mesh",
"path": "forwarded", "to": 5672},
// The registry from anywhere, by its node's exposure setting.
"adoption.opening-tcp-5000-forwarded": {"port": 5000, "protocol": "tcp",
"from": "everywhere", "path": "forwarded", "to": 5000},
// A published port names the machine port and the container port it is forwarded to.
"adoption.opening-tcp-20001-forwarded": {"port": 20001, "protocol": "tcp",
"from": "everywhere", "path": "forwarded", "to": 8080},
}
for id, fields := range want {
opening, ok := got[id]
if !ok {
t.Errorf("no %s among %v", id, keys(got))
continue
}
if opening["type"] != "opening" {
t.Errorf("%s is a %v", id, opening["type"])
}
for k, v := range fields {
if opening[k] != v {
t.Errorf("%s: %s is %v, want %v", id, k, opening[k], v)
}
}
}
for id := range got {
if strings.HasPrefix(id, "adoption.opening-") && want[id] == nil {
t.Errorf("an opening nothing asked for: %s", id)
}
}
// A port for this machine only opens nothing, and the management port is not opened at all.
for id := range got {
if strings.Contains(id, "-9000-") || strings.Contains(id, "-15672-") {
t.Errorf("%s is opened", id)
}
}
// And openings come first, in the order the machine applies them.
if !strings.HasPrefix(composed.Resources[0]["id"].(string), "adoption.opening-") {
t.Errorf("openings are not first: %v", composed.Resources[0]["id"])
}
}
func TestAnAdoptedNodeLoadsNoFilterOfTheMeshs(t *testing.T) {
composed, err := anAdoptedAnchor().Compose(anchorRendering(true))
if err != nil {
t.Fatal(err)
}
for _, r := range composed.Resources {
if r["path"] == "/etc/nftables.conf" || strings.HasPrefix(r["id"].(string), "nftables.") {
t.Fatalf("an adopted node is declared the filter module's %v", r["id"])
}
if content, _ := r["content"].(string); strings.Contains(content, "policy drop") {
t.Fatalf("an adopted node is declared a table that drops by default: %v", r["id"])
}
// Nothing but refusals: the only accept in the guard is its policy.
if content, _ := r["content"].(string); r["id"] == GuardID() &&
strings.Count(content, "accept") != 1 {
t.Fatalf("the guard holds an accept:\n%s", content)
}
}
got := byID(composed.Resources)
guard := got[GuardID()]
if guard == nil || got[GuardUnitID()] == nil || got[GuardRunningID()] == nil {
t.Fatalf("no guard: %v", keys(got))
}
if guard["content"] != AsGuard([]int{5432, 15672}) {
t.Fatalf("the guard does not guard the store and the management port:\n%s", guard["content"])
}
if !reflect.DeepEqual(got[GuardRunningID()]["restart-on"], []any{GuardID(), GuardUnitID()}) {
t.Fatalf("the guard is not reloaded when its table changes: %v", got[GuardRunningID()])
}
// Nothing of the mesh's own is anybody's to hold.
for id, module := range composed.Owner {
if strings.HasPrefix(id, AdoptionPrefix) {
t.Fatalf("%s is owned by %s", id, module)
}
}
}
func TestAConvergedNodeIsDeclaredItsFilterAndNoOpenings(t *testing.T) {
composed, err := anAdoptedAnchor().Compose(anchorRendering(false))
if err != nil {
t.Fatal(err)
}
got := byID(composed.Resources)
if got["nftables.filtering"] == nil || got["nftables.load"] == nil {
t.Fatalf("a converged node lost its filter: %v", keys(got))
}
for id := range got {
if strings.HasPrefix(id, AdoptionPrefix) {
t.Fatalf("a converged node is declared %s", id)
}
}
plain, err := anAdoptedAnchor().Declaration(anchorRendering(false))
if err != nil {
t.Fatal(err)
}
a, _ := json.Marshal(plain)
b, _ := json.Marshal(composed.Resources)
if string(a) != string(b) {
t.Fatal("Compose and Declaration disagree on a converged node")
}
}
// The table the installer raises and the controller declares, character for character.
func TestTheGuardIsExactlyThisTable(t *testing.T) {
const golden = `table inet mesh_guard {}
delete table inet mesh_guard
table inet mesh_guard {
chain prerouting {
type filter hook prerouting priority raw; policy accept;
iifname != "lo" iifname != "docker0" iifname != "br-*" iifname != "mesh0" tcp dport { 5432, 15672 } drop
}
}
`
if got := AsGuard([]int{15672, 5432}); got != golden {
t.Fatalf("the guard changed:\n%s", got)
}
if GuardResources(nil) != nil {
t.Fatal("a guard with nothing to guard is an empty set nft refuses to load")
}
}
func TestAGuardedPortMustBeAPort(t *testing.T) {
if _, err := ParseManifest([]byte(`{"module":"postgres","guards":[5432]}`)); err != nil {
t.Fatalf("guards is refused: %v", err)
}
if _, err := ParseManifest([]byte(`{"module":"postgres","guards":[0]}`)); err == nil {
t.Fatal("guarding port 0 was accepted")
}
}
func keys[V any](m map[string]V) []string {
return sortedKeys(m)
}
+57
View File
@@ -128,6 +128,11 @@ type Rendering struct {
// set, for what a consumer is told, and for what the runtime publishes — and nothing checked
// that the three agreed. They are all derived from this.
Ports map[string]map[int]int
// Adopted says the node is adopted (novox/hq ADR 0100): the firewall found on it stays in
// force, so no module that loads a filter is declared there, and what the mesh needs
// reachable is declared as openings, with its own ports guarded by a table that only refuses.
Adopted bool
}
// machinePort is where a module's port lives on this machine, or the port itself when the mesh has
@@ -257,6 +262,13 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
var out []map[string]any
for _, m := range r.Modules {
if with.Adopted && m.Filtering != nil {
// Nothing of a module that loads a filter, on an adopted node: its table would drop
// by default and hold accepts, and the found firewall stays in force. Every resource,
// not only the rule set — its service must not run, and a node returned to adopted
// stops it by the ordinary removal of what is no longer declared.
continue
}
resources := m.Resources
// What the mesh computes for this module goes FIRST, before the module's own resources.
@@ -567,9 +579,54 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri
out = append(out, fact)
}
}
if with.Adopted {
// First, before anything a module declares: what the mesh needs reachable, then its guard.
// The order a machine applies is the order written here.
ours := Openings(rules, with.Foundation, Published(out))
ours = append(ours, GuardResources(r.guarded(out, owner, with))...)
out = append(ours, out...)
}
return out, nil
}
// guarded is the machine ports of every guarded port of the modules here: where each module's
// container publishes it, as composed — or where the machine put it when no container does.
func (r Resolution) guarded(out []map[string]any, owner map[string]string, with Rendering) []int {
seen := map[int]bool{}
var ports []int
for _, m := range r.Modules {
for _, want := range m.Guards {
at := with.machinePort(m.Module, want)
for _, resource := range out {
if owner[fmt.Sprint(resource["id"])] != m.Module ||
fmt.Sprint(resource["type"]) != "container" {
continue
}
listed, _ := resource["ports"].([]any)
for _, entry := range listed {
parts := strings.Split(strings.TrimSpace(fmt.Sprint(entry)), ":")
if len(parts) < 2 {
continue
}
inner, err := strconv.Atoi(strings.SplitN(parts[len(parts)-1], "/", 2)[0])
if err != nil || inner != want {
continue
}
if outer, err := strconv.Atoi(parts[len(parts)-2]); err == nil {
at = outer
}
}
}
if !seen[at] {
seen[at] = true
ports = append(ports, at)
}
}
}
sort.Ints(ports)
return ports
}
// Contribution is one module telling the answer to a requirement what it needs from it.
type Contribution struct {
// From is the module that said it, so the provider and a person reading the file can tell
+14
View File
@@ -346,6 +346,14 @@ type Manifest struct {
// that could only see its own ports would write a rule set that closed everything else.
Filtering *Filtering `json:"filtering,omitempty"`
// Guards are ports of this module's the mesh refuses on an adopted node except from the
// private network and from the machine itself (novox/hq ADR 0100) — the store's port and the
// broker's management port. The ports the software uses; the mesh guards where the machine
// publishes them. On an adopted node the found firewall stays in force and the mesh loads no
// filter of its own, so this is what keeps them unreachable from outside whatever that
// firewall does. Ignored on a converged node, whose derived filter already closes them.
Guards []int `json:"guards,omitempty"`
// Facts are things only the mesh knows, written where this module asks for them.
//
// **The graph is the control plane's; how a machine uses it is the module's.** The mesh knows
@@ -950,6 +958,12 @@ func ParseManifest(raw []byte) (Manifest, error) {
"%s listens on %d over %q, which is tcp or udp", m.Module, l.Port, p))
}
}
for _, port := range m.Guards {
if port < 1 || port > 65535 {
problems = append(problems, fmt.Sprintf(
"%s guards port %d, which is not a port", m.Module, port))
}
}
if c := m.Certificate; c != nil {
if !strings.HasPrefix(c.Into, "/") {
problems = append(problems, fmt.Sprintf(