Converge openings through the firewall an adopted node was found with, and retire it only when the node converges (hq ADR 0100)

This commit is contained in:
2026-09-22 17:19:49 +02:00
parent 3a613113be
commit 3c90d155b3
10 changed files with 1522 additions and 7 deletions
+29 -2
View File
@@ -150,8 +150,21 @@ func ApplyKeeping(
declared[r.Identity()] = true
}
// Which firewall is found here, before anything else, since an unsupported one refuses the
// whole declaration (novox/hq ADR 0100). Nothing for a converged node.
fw, err := foundFirewall(ctx, d, &known, run, log)
if err != nil {
return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report}
}
for _, orphan := range known.Orphans(declared, origin) {
action, detail, err := remove(ctx, sys, orphan, run)
var action, detail string
var err error
if declaration.Type(orphan.Type) == declaration.TypeOpening {
action, detail, err = removeOpening(ctx, orphan, run, known.Firewall)
} else {
action, detail, err = remove(ctx, sys, orphan, run)
}
if err != nil {
return report, known, &Error{Resource: orphan.ID, Err: err, Done: report}
}
@@ -235,7 +248,13 @@ func ApplyKeeping(
}
was, _ := known.Find(resource.Identity())
outcome, err := applyOne(ctx, sys, resource, run, changed, declares, was, unseal)
var outcome Outcome
var err error
if o, isOpening := resource.(*declaration.Opening); isOpening {
outcome, err = applyOpening(ctx, o, run, fw)
} else {
outcome, err = applyOne(ctx, sys, resource, run, changed, declares, was, unseal)
}
if err != nil {
failed := &Error{Resource: resource.Identity(), Err: err, Done: report}
failures = append(failures, failed)
@@ -293,6 +312,14 @@ func ApplyKeeping(
}
}
// A converged node whose found firewall was in force retires it only now, once everything —
// the mesh's derived filter among it — applied cleanly (novox/hq ADR 0100).
if len(failures) == 0 {
if err := retireFirewall(ctx, d, &known, run, log); err != nil {
return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report}
}
}
if len(failures) > 0 {
// The first, carrying everything that did happen. One error is what the caller reports
// and what a person reads first; the rest are in the report, which is what the mesh
+109
View File
@@ -0,0 +1,109 @@
package apply
import (
"context"
"fmt"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/firewall"
"github.com/novox/mesh-host/internal/store"
)
// foundFirewall settles, before anything else in an apply, which firewall this node has — and on
// an adopted node that the mesh had converged, puts it back in force first (novox/hq ADR 0100).
//
// Only an adopted node asks. It is detected on every apply rather than remembered, so a firewall
// switched on after adoption is spoken to from the next reconcile; what is remembered is what was
// found first, and whether the mesh retired it. An unsupported firewall refuses the whole
// declaration: the mesh could neither open what it needs through it nor say what it would close.
func foundFirewall(ctx context.Context, d *declaration.Declaration, known *store.State, run Runner,
log func(string)) (firewall.Kind, error) {
if d.Adoption == nil {
return "", nil
}
rec := known.Firewall
if rec != nil && rec.DisabledByMesh && rec.Kind == string(firewall.UFW) {
// Returned to adopted: the found firewall is enabled again before the openings are
// converged through it, and the derived filter is gone with this declaration.
if err := firewall.Enable(ctx, run); err != nil {
return "", err
}
rec.DisabledByMesh = false
log(" enabled ufw again: this node is adopted, and the firewall found on it is in force")
}
kind, name, err := firewall.Detect(ctx, run)
if err != nil {
return "", err
}
if kind == firewall.Unsupported {
return "", fmt.Errorf(
"this machine is filtered by %s, and no host speaks that firewall yet. An adopted node "+
"keeps the firewall it was found with, so the mesh could neither open what it needs "+
"through it nor say what it would close; this declaration is refused whole", name)
}
if rec == nil {
rec = &store.FoundFirewall{Kind: string(kind), WasActive: kind == firewall.UFW,
FoundAt: time.Now().UTC()}
} else {
rec.Kind = string(kind)
rec.WasActive = rec.WasActive || kind == firewall.UFW
}
known.Firewall = rec
return kind, nil
}
// retireFirewall disables the found firewall once a converged declaration has applied cleanly,
// which is when the mesh's derived filter has taken its place. Disabled, never flushed: its
// configuration stays on disk for a return to adopted, and the container runtime's rules are not
// its to take.
func retireFirewall(ctx context.Context, d *declaration.Declaration, known *store.State, run Runner,
log func(string)) error {
rec := known.Firewall
if d.Adoption != nil || rec == nil || rec.Kind != string(firewall.UFW) || !rec.WasActive ||
rec.DisabledByMesh {
return nil
}
if err := firewall.Disable(ctx, run); err != nil {
return err
}
rec.DisabledByMesh = true
log(" disabled ufw: this node is converged and filtered by the mesh; ufw's configuration is left on disk")
return nil
}
// applyOpening makes one opening true through the firewall found here.
func applyOpening(ctx context.Context, o *declaration.Opening, run Runner, kind firewall.Kind) (Outcome, error) {
out := begin(o)
switch kind {
case firewall.None:
out.Action = "unchanged"
out.Detail = "no firewall found; nothing filters this port"
return out, nil
case firewall.UFW:
action, err := firewall.Converge(ctx, run, o)
if err != nil {
return out, err
}
out.Action = action
out.Detail = "through ufw, marked " + firewall.Mark(o)
return out, nil
}
return out, fmt.Errorf("no firewall is known for this node, so %s cannot be opened", o.Target())
}
// removeOpening deletes the rules the mesh marked for an opening no longer declared, and nothing
// the machine had before.
func removeOpening(ctx context.Context, a store.Applied, run Runner, rec *store.FoundFirewall) (string, string, error) {
if rec == nil || rec.Kind != string(firewall.UFW) {
return "forgotten", "no firewall held a rule for it", nil
}
n, err := firewall.Remove(ctx, run, a.ID)
if err != nil {
return "", "", err
}
if n == 0 {
return "forgotten", "ufw held no rule marked for it", nil
}
return "removed", fmt.Sprintf("%d ufw rule(s) marked as the mesh's deleted", n), nil
}
+204
View File
@@ -0,0 +1,204 @@
package apply
import (
"context"
"errors"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force; converging the
// node retires it by disabling it, and returning the node to adopted enables it again.
type ufwMachine struct {
installed, active bool
rules []string
ruleset string
asked []string
}
func (u *ufwMachine) run(_ context.Context, name string, args ...string) (string, error) {
u.asked = append(u.asked, name+" "+strings.Join(args, " "))
switch name {
case "nft":
return u.ruleset, nil
case "ufw":
if !u.installed {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
default:
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
switch args[0] {
case "status":
if u.active {
return "Status: active\n", nil
}
return "Status: inactive\n", nil
case "show":
out := "Added user rules (see 'ufw status' for running firewall):\n"
for _, r := range u.rules {
out += "ufw " + r + "\n"
}
return out, nil
case "--force":
u.active = true
return "", nil
case "disable":
u.active = false
return "", nil
case "delete":
want := strings.Join(args[1:], " ")
for i, r := range u.rules {
if strings.ReplaceAll(r, "'", "") == want {
u.rules = append(u.rules[:i], u.rules[i+1:]...)
return "", nil
}
}
return "", errors.New("Could not delete non-existent rule")
default:
// Printed back the way it was given, with the comment quoted as ufw does.
line := strings.Join(args[:len(args)-1], " ") + " '" + args[len(args)-1] + "'"
u.rules = append(u.rules, line)
return "", nil
}
}
func (u *ufwMachine) index(prefix string) int {
for i, a := range u.asked {
if strings.HasPrefix(a, prefix) {
return i
}
}
return -1
}
const busOpening = `{"id":"adoption.opening-tcp-5671-incoming","type":"opening","port":5671,"protocol":"tcp","from":"everywhere","path":"incoming"}`
func withConf(dir string) string {
return `{"id":"x.conf","type":"file","path":"` + filepath.Join(dir, "x.conf") + `","content":"x\n"}`
}
func applyWith(t *testing.T, d *declaration.Declaration, known store.State, run Runner) (Report, store.State, error) {
t.Helper()
return ApplyKeeping(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil,
KeepIn(t.TempDir()))
}
func TestAnOpeningOnAMachineWithNoFirewallChangesNothing(t *testing.T) {
dir := t.TempDir()
u := &ufwMachine{}
report, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
o := outcomeOf(report, "adoption.opening-tcp-5671-incoming")
if o.Action != "unchanged" || !strings.Contains(o.Detail, "nothing filters this port") {
t.Errorf("an opening with no firewall: %+v", o)
}
if state.Firewall == nil || state.Firewall.Kind != "none" {
t.Errorf("the firewall found was not recorded: %+v", state.Firewall)
}
}
func TestAnUnsupportedFirewallRefusesTheWholeDeclaration(t *testing.T) {
dir := t.TempDir()
u := &ufwMachine{ruleset: "table inet filter {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t}\n}\n"}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err == nil || !strings.Contains(err.Error(), "no host speaks that firewall") {
t.Fatalf("an unsupported firewall was not refused: %v", err)
}
if _, statErr := os.Stat(filepath.Join(dir, "x.conf")); !errors.Is(statErr, os.ErrNotExist) {
t.Error("part of a refused declaration was applied")
}
if state.Firewall != nil {
t.Errorf("an unsupported firewall was recorded: %+v", state.Firewall)
}
}
func TestConvergingRetiresTheFoundFirewallAndReturningRestoresIt(t *testing.T) {
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
// Adopted: the opening goes through ufw.
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
if len(u.rules) != 2 || !u.active {
t.Fatalf("adopted: rules %v, active %v", u.rules, u.active)
}
if state.Firewall == nil || state.Firewall.Kind != "ufw" || !state.Firewall.WasActive {
t.Fatalf("adopted: firewall recorded as %+v", state.Firewall)
}
// Converged: the opening's rule goes, and only then is ufw disabled — never reset.
u.asked = nil
converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
_, state, err = applyWith(t, converged, state, u.run)
if err != nil {
t.Fatal(err)
}
if u.active || !state.Firewall.DisabledByMesh {
t.Fatalf("converged: ufw still active (%v) or not recorded as retired (%+v)", u.active, state.Firewall)
}
if len(u.rules) != 1 || u.rules[0] != "allow 22/tcp" {
t.Errorf("converged: the operator's rules were touched, or the mesh's left: %v", u.rules)
}
if del, dis := u.index("ufw delete"), u.index("ufw disable"); del < 0 || dis < del {
t.Errorf("converged: the opening was not removed before ufw was disabled: %v", u.asked)
}
for _, a := range u.asked {
if strings.Contains(a, "reset") {
t.Errorf("converged: ufw was reset: %s", a)
}
}
// Converged again: nothing more to retire.
u.asked = nil
if _, state, err = applyWith(t, converged, state, u.run); err != nil {
t.Fatal(err)
}
if u.index("ufw") >= 0 {
t.Errorf("a converged node kept talking to a retired ufw: %v", u.asked)
}
// Returned to adopted: ufw is enabled before the opening is converged through it.
u.asked = nil
_, state, err = applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), state, u.run)
if err != nil {
t.Fatal(err)
}
if !u.active || state.Firewall.DisabledByMesh {
t.Fatalf("returned: ufw active %v, record %+v", u.active, state.Firewall)
}
if en, add := u.index("ufw --force enable"), u.index("ufw allow"); en < 0 || add < en {
t.Errorf("returned: ufw was not enabled before the opening was added: %v", u.asked)
}
if len(u.rules) != 2 {
t.Errorf("returned: the opening was not converged again: %v", u.rules)
}
}
func TestAConvergedNodeThatWasNeverAdoptedNeverAsksAboutAFirewall(t *testing.T) {
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true}
if _, _, err := applyWith(t, parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`), store.State{}, u.run); err != nil {
t.Fatal(err)
}
if len(u.asked) != 0 {
t.Errorf("a converged apply asked the machine about its firewall: %v", u.asked)
}
}
func TestAnOpeningOnAConvergedNodeIsRefused(t *testing.T) {
if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` + busOpening + `]}`)); err == nil {
t.Error("an opening was accepted on a node the declaration does not say is adopted")
}
}
+89 -1
View File
@@ -78,6 +78,12 @@ const (
// cadence. Tools, hooks and event consumers are not separate modes: they are loaded by a tool
// host, which is itself a process that stays up.
TypeProcess Type = "process"
// TypeOpening is a port the mesh needs reachable on an adopted node, converged through the
// firewall found there in that firewall's own terms (novox/hq ADR 0100). A state, not a
// command: the host adds the rule it marks as the mesh's when it is missing, and removes only
// what it marked — which is what lets it travel over the link.
TypeOpening Type = "opening"
)
// Resource is one thing that should be true of the machine.
@@ -603,6 +609,74 @@ func (s *Service) validate(where string, _ bool) []string {
return problems
}
// Opening is a port reachable on an adopted node, from where, and on which path.
//
// **From** is everywhere or mesh — the private network, by its interface. **Path** is incoming,
// for something listening on the machine, or forwarded, for a published container port: the found
// firewall sees a published port after the runtime has translated it, so a forwarded opening names
// the container's own port in To as well as the machine's in Port.
type Opening struct {
ID string `json:"id"`
Type Type `json:"type"`
Port int `json:"port"`
Protocol string `json:"protocol"`
From string `json:"from"`
Path string `json:"path"`
To int `json:"to,omitempty"`
}
// Where an opening admits from, and the path it is on.
const (
FromEverywhere = "everywhere"
FromMesh = "mesh"
PathIncoming = "incoming"
PathForwarded = "forwarded"
)
func (o *Opening) Identity() string { return o.ID }
func (o *Opening) Kind() Type { return TypeOpening }
func (o *Opening) Target() string {
if o.Path == PathForwarded {
return fmt.Sprintf("%s/%d forwarded to %d from %s", o.Protocol, o.Port, o.To, o.From)
}
return fmt.Sprintf("%s/%d %s from %s", o.Protocol, o.Port, o.Path, o.From)
}
func (o *Opening) validate(where string, _ bool) []string {
var problems []string
if o.Port < 1 || o.Port > 65535 {
problems = append(problems, fmt.Sprintf("%s: an opening's port is 1-65535, not %d", where, o.Port))
}
if o.Protocol != "tcp" && o.Protocol != "udp" {
problems = append(problems, fmt.Sprintf("%s: an opening is tcp or udp, not %q", where, o.Protocol))
}
if o.From != FromEverywhere && o.From != FromMesh {
problems = append(problems, fmt.Sprintf(
"%s: an opening is from %q or %q, not %q", where, FromEverywhere, FromMesh, o.From))
}
switch o.Path {
case PathIncoming:
if o.To != 0 {
problems = append(problems, where+
": an incoming opening names no container port; only a forwarded one does")
}
case PathForwarded:
if o.To < 1 || o.To > 65535 {
problems = append(problems, where+
": a forwarded opening names the container's port it reaches, as to, 1-65535")
}
default:
problems = append(problems, fmt.Sprintf(
"%s: an opening's path is %q or %q, not %q", where, PathIncoming, PathForwarded, o.Path))
}
if !strings.HasPrefix(o.ID, AdoptionPrefix) {
problems = append(problems, fmt.Sprintf(
"%s: an opening is the mesh's own, so its id starts %q", where, AdoptionPrefix))
}
return problems
}
// Package is a package that should be present.
//
// Present is the whole of what it asserts, never a version: version is the package manager's
@@ -810,6 +884,8 @@ func newOf(t Type) Resource {
return &Access{}
case TypeProcess:
return &Process{}
case TypeOpening:
return &Opening{}
}
return nil
}
@@ -818,7 +894,7 @@ func newOf(t Type) Resource {
func Vocabulary() []Type {
return []Type{
TypeAccess, TypeAction, TypeArchive, TypeContainer, TypeDirectory, TypeFile,
TypeNetwork, TypePackage, TypeProcess, TypeService, TypeUser,
TypeNetwork, TypeOpening, TypePackage, TypeProcess, TypeService, TypeUser,
}
}
@@ -1051,6 +1127,18 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
d.Resources = append(d.Resources, resource)
}
problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...)
if env.Adoption == nil {
for _, r := range d.Resources {
if r.Kind() == TypeOpening {
// On a converged node the mesh's own filter admits what is declared, and the
// found firewall is retired; an opening there would be a rule in a firewall the
// mesh has disabled (novox/hq ADR 0100).
problems = append(problems, fmt.Sprintf(
"resource %q: an opening is for an adopted node, and this declaration does not "+
"say the node is adopted", r.Identity()))
}
}
}
if len(problems) > 0 {
return nil, &RefusalError{Problems: problems}
+8 -4
View File
@@ -266,7 +266,7 @@ func TestAFieldTheNewTypesDoNotUseIsRefused(t *testing.T) {
}
}
func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) {
func TestTheVocabularyIsTheTwelveShapesTheMeshNeeds(t *testing.T) {
// Six of them the bootstrap uses (novox/hq 07-the-foundation.md), and removing one is a
// failing test rather than a discovery during a first-node install.
//
@@ -282,7 +282,7 @@ func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) {
}
for _, want := range []Type{
TypeDirectory, TypeFile, TypeService, TypePackage, TypeContainer, TypeAction,
TypeUser, TypeArchive, TypeNetwork, TypeAccess, TypeProcess,
TypeUser, TypeArchive, TypeNetwork, TypeAccess, TypeProcess, TypeOpening,
} {
if !speaks[want] {
t.Errorf("the host no longer speaks %q", want)
@@ -309,8 +309,12 @@ func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) {
// It is a full-host shape rather than a portable one: it needs a process supervisor to install
// into. It does NOT need a container runtime, which is the point — only software that
// genuinely needs isolation asks for a container.
if len(speaks) != 11 {
t.Errorf("the vocabulary is %d shapes rather than 11; every addition widens what a compromised "+
//
// `opening` is the twelfth, and novox/hq ADR 0100 is its decision: on an adopted node the
// firewall found there stays in force, and what the mesh needs reachable is converged through
// it as a state the host marks as the mesh's — never a command, which the link may not carry.
if len(speaks) != 12 {
t.Errorf("the vocabulary is %d shapes rather than 12; every addition widens what a compromised "+
"control plane can express, so a change here is a decision: %s",
len(speaks), vocabulary())
}
+431
View File
@@ -0,0 +1,431 @@
// Package firewall speaks the firewall found on an adopted node, in that firewall's own terms
// (novox/hq ADR 0100).
//
// **The found firewall stays in force.** On an adopted node the mesh loads nothing that drops by
// default or holds an accept; what it needs reachable it converges as openings through what it
// found, marks each rule as its own, and removes only what it marked. It never resets or flushes:
// the rules the machine already had are the operator's, and they are what keeps it serving.
package firewall
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os/exec"
"regexp"
"strconv"
"strings"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/system"
)
// Runner executes a command.
type Runner = system.Runner
// Kind is what firewall a machine has, as far as the mesh is concerned.
type Kind string
const (
// UFW is an active ufw — the one kind found on the machines measured, and the one spoken.
UFW Kind = "ufw"
// None is a machine where nothing refuses anything, which needs no openings.
None Kind = "none"
// Unsupported is a firewall no host speaks yet. A machine with one is refused adoption: the
// mesh could neither open what it needs nor know what it would be closing.
Unsupported Kind = "unsupported"
)
// MeshInterface is the private network's interface, the way an opening from the mesh is known.
// It must be the controller's overlay interface name.
const MeshInterface = "mesh0"
// Detect says which firewall this machine has. For Unsupported the string names it.
func Detect(ctx context.Context, run Runner) (Kind, string, error) {
if out, err := run(ctx, "firewall-cmd", "--state"); err == nil && strings.TrimSpace(out) == "running" {
return Unsupported, "firewalld", nil
}
ufwActive := false
if out, err := run(ctx, "ufw", "status"); err == nil {
ufwActive = statusActive(out)
}
out, err := run(ctx, "nft", "list", "ruleset")
switch {
case err == nil:
if refusing := Refusing(out, ufwActive); len(refusing) > 0 {
return Unsupported, "nftables rules that refuse traffic, in " + strings.Join(refusing, ", "), nil
}
case !missing(err):
return "", "", fmt.Errorf("cannot read this machine's packet filter to know what it has: %w", err)
}
if !ufwActive {
// iptables with the legacy backend is invisible to nft.
for _, legacy := range []string{"iptables-legacy", "ip6tables-legacy"} {
out, err := run(ctx, legacy, "-S")
if err != nil {
continue
}
if refusing := RefusingLegacy(out); len(refusing) > 0 {
return Unsupported, legacy + " rules that refuse traffic, in " + strings.Join(refusing, ", "), nil
}
}
}
if ufwActive {
return UFW, "ufw", nil
}
return None, "", nil
}
func missing(err error) bool {
return errors.Is(err, exec.ErrNotFound)
}
func statusActive(out string) bool {
for _, line := range strings.Split(out, "\n") {
if strings.HasPrefix(strings.TrimSpace(line), "Status:") {
return strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "Status:")) == "active"
}
}
return false
}
// Refusing names the tables of an `nft list ruleset` holding something that refuses traffic — a
// drop or reject, or a base chain whose policy drops — and that is neither the mesh's own nor the
// container runtime's. With ufw active, the tables iptables-nft manages are ufw's and the runtime's
// and are not counted.
func Refusing(ruleset string, ufwActive bool) []string {
var refusing []string
managed := map[string]bool{}
var table, chain string
counted := map[string]bool{}
note := func() {
if !counted[table] {
counted[table] = true
refusing = append(refusing, "table "+table)
}
}
for _, raw := range strings.Split(ruleset, "\n") {
line := strings.TrimSpace(raw)
switch {
case strings.HasPrefix(line, "# Warning: table ") && strings.Contains(line, "managed by iptables-nft"):
name := strings.TrimPrefix(line, "# Warning: table ")
name, _, _ = strings.Cut(name, " is managed")
managed[name] = true
continue
case strings.HasPrefix(line, "table "):
table = strings.TrimSuffix(strings.TrimSpace(strings.TrimPrefix(line, "table ")), "{")
table = strings.TrimSpace(table)
chain = ""
continue
case strings.HasPrefix(line, "chain "):
chain = strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(line, "chain "), "{"))
continue
case line == "" || line == "}" || strings.HasPrefix(line, "#") || chain == "":
continue
}
if table == "inet mesh" || table == "inet mesh_guard" {
continue
}
iptables := managed[table] || iptablesTable(table)
if iptables && ufwActive {
continue
}
if strings.HasPrefix(line, "type ") {
if strings.Contains(line, "policy drop") && !(iptables && runtimes(table, chain, line)) {
note()
}
continue
}
if !verdictRefuses(line) {
continue
}
if iptables && runtimes(table, chain, line) {
continue
}
note()
}
return refusing
}
// iptablesTable is whether a table is one iptables-nft writes. Named rather than read from the
// warning nft prints above it, because nft does not print that for every such table: a captured
// ruleset carried it on ip filter and not on ip raw, where the runtime keeps its drops.
func iptablesTable(table string) bool {
family, name, _ := strings.Cut(table, " ")
if family != "ip" && family != "ip6" {
return false
}
switch name {
case "filter", "nat", "raw", "mangle", "security":
return true
}
return false
}
// runtimes is whether a refusal in an iptables-nft table is the container runtime's own: in its
// DOCKER chains, its forward policy, or its guard against reaching a container's address directly
// from outside its bridge, in the raw table.
func runtimes(table, chain, line string) bool {
_, name, _ := strings.Cut(table, " ")
switch {
case strings.HasPrefix(chain, "DOCKER"):
return true
case name == "filter" && chain == "FORWARD" && strings.HasPrefix(line, "type "):
return true
case name == "raw" && chain == "PREROUTING":
return strings.Contains(line, "daddr") && strings.Contains(line, "iifname !=")
}
return false
}
var verdict = regexp.MustCompile(`(^|\s)(drop|reject)(\s|$)`)
func verdictRefuses(line string) bool {
return verdict.MatchString(line)
}
// RefusingLegacy names the chains of an `iptables-legacy -S` that refuse traffic outside the
// container runtime's own.
func RefusingLegacy(rules string) []string {
var refusing []string
seen := map[string]bool{}
for _, line := range strings.Split(rules, "\n") {
fields := strings.Fields(line)
if len(fields) < 3 {
continue
}
chain := fields[1]
refuses := false
switch fields[0] {
case "-P":
refuses = fields[2] == "DROP" && chain != "FORWARD"
case "-A":
for i, f := range fields {
if f == "-j" && i+1 < len(fields) && (fields[i+1] == "DROP" || fields[i+1] == "REJECT") {
refuses = !strings.HasPrefix(chain, "DOCKER")
}
}
}
if refuses && !seen[chain] {
seen[chain] = true
refusing = append(refusing, "chain "+chain)
}
}
return refusing
}
// --- ufw ---------------------------------------------------------------------------------------
// Mark is the comment every rule the mesh adds carries: whose it is, which opening, and a digest
// of the rule itself, so a rule the opening no longer describes is recognised as stale without the
// host having to know how ufw prints a rule back.
func Mark(o *declaration.Opening) string {
sum := sha256.Sum256([]byte(strings.Join(Rule(o), " ")))
return marker(o.ID) + " " + hex.EncodeToString(sum[:])[:8]
}
func marker(id string) string { return "mesh-host " + id }
// markedFor is whether a comment is the mesh's, for this opening.
func markedFor(comment, id string) bool {
return comment == marker(id) || strings.HasPrefix(comment, marker(id)+" ")
}
// Rule is the ufw rule an opening becomes, without its comment.
//
// incoming from everywhere allow proto tcp to any port P
// incoming from the mesh allow in on mesh0 proto tcp to any port P
// forwarded route allow [in on mesh0] proto tcp to any port <container port>
//
// A forwarded opening names the container's port because ufw's route rules are matched after the
// runtime's destination translation.
func Rule(o *declaration.Opening) []string {
var rule []string
port := o.Port
if o.Path == declaration.PathForwarded {
rule = append(rule, "route")
port = o.To
}
rule = append(rule, "allow")
if o.From == declaration.FromMesh {
rule = append(rule, "in", "on", MeshInterface)
}
return append(rule, "proto", o.Protocol, "to", "any", "port", strconv.Itoa(port))
}
var commentOf = regexp.MustCompile(`comment '([^']*)'`)
// added is every rule `ufw show added` lists, each without its leading "ufw".
func added(ctx context.Context, run Runner) ([]string, error) {
out, err := run(ctx, "ufw", "show", "added")
if err != nil {
return nil, fmt.Errorf("reading ufw's rules: %w", err)
}
var rules []string
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "ufw ") {
rules = append(rules, strings.TrimPrefix(line, "ufw "))
}
}
return rules, nil
}
func comment(rule string) string {
m := commentOf.FindStringSubmatch(rule)
if m == nil {
return ""
}
return m[1]
}
// words splits a rule as ufw printed it into arguments, keeping a quoted comment whole.
func words(rule string) []string {
var out []string
var cur strings.Builder
quoted, any := false, false
for _, r := range rule {
switch {
case r == '\'':
quoted = !quoted
any = true
case r == ' ' && !quoted:
if any {
out = append(out, cur.String())
cur.Reset()
any = false
}
default:
cur.WriteRune(r)
any = true
}
}
if any {
out = append(out, cur.String())
}
return out
}
// Converge makes one opening true in ufw: its marked rule present, and any rule marked for it that
// no longer describes it deleted. Nothing unmarked is touched. The outcome is created, updated or
// unchanged, read back from ufw rather than assumed.
func Converge(ctx context.Context, run Runner, o *declaration.Opening) (string, error) {
rules, err := added(ctx, run)
if err != nil {
return "", err
}
mark := Mark(o)
present := false
var stale []string
for _, rule := range rules {
c := comment(rule)
switch {
case c == mark:
present = true
case markedFor(c, o.ID):
stale = append(stale, rule)
}
}
if present && len(stale) == 0 {
return "unchanged", nil
}
for _, rule := range stale {
if _, err := run(ctx, "ufw", append([]string{"delete"}, words(rule)...)...); err != nil {
return "", fmt.Errorf("deleting the mesh's stale ufw rule %q: %w", rule, err)
}
}
if !present {
args := append(Rule(o), "comment", mark)
if _, err := run(ctx, "ufw", args...); err != nil {
return "", fmt.Errorf("adding the ufw rule for %s: %w", o.Target(), err)
}
}
after, err := added(ctx, run)
if err != nil {
return "", err
}
found, leftover := false, 0
for _, rule := range after {
c := comment(rule)
if c == mark {
found = true
} else if markedFor(c, o.ID) {
leftover++
}
}
if !found {
return "", fmt.Errorf("ufw was asked for %s and does not list it afterwards", o.Target())
}
if leftover > 0 {
return "", fmt.Errorf("ufw still lists %d stale rule(s) marked for %s after deleting them", leftover, o.ID)
}
if len(stale) > 0 {
return "updated", nil
}
return "created", nil
}
// Remove deletes the rules marked for one opening, and nothing else.
func Remove(ctx context.Context, run Runner, id string) (int, error) {
rules, err := added(ctx, run)
if err != nil {
return 0, err
}
removed := 0
for _, rule := range rules {
if !markedFor(comment(rule), id) {
continue
}
if _, err := run(ctx, "ufw", append([]string{"delete"}, words(rule)...)...); err != nil {
return removed, fmt.Errorf("deleting the mesh's ufw rule %q: %w", rule, err)
}
removed++
}
after, err := added(ctx, run)
if err != nil {
return removed, err
}
for _, rule := range after {
if markedFor(comment(rule), id) {
return removed, fmt.Errorf("ufw still lists a rule marked for %s after deleting it", id)
}
}
return removed, nil
}
// Enable turns ufw back on, as found, and reads back that it is.
func Enable(ctx context.Context, run Runner) error {
if _, err := run(ctx, "ufw", "--force", "enable"); err != nil {
return fmt.Errorf("enabling ufw again: %w", err)
}
return expectActive(ctx, run, true)
}
// Disable retires ufw without flushing it: its configuration stays on disk, and the container
// runtime's rules are not its to remove.
func Disable(ctx context.Context, run Runner) error {
if _, err := run(ctx, "ufw", "disable"); err != nil {
return fmt.Errorf("disabling ufw: %w", err)
}
return expectActive(ctx, run, false)
}
func expectActive(ctx context.Context, run Runner, want bool) error {
out, err := run(ctx, "ufw", "status")
if err != nil {
return fmt.Errorf("reading ufw's status back: %w", err)
}
if statusActive(out) != want {
state := "inactive"
if want {
state = "active"
}
return fmt.Errorf("ufw was asked to be %s and says: %s", state, strings.TrimSpace(out))
}
return nil
}
+335
View File
@@ -0,0 +1,335 @@
package firewall
import (
"context"
"errors"
"fmt"
"os"
"os/exec"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
)
// Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force, the mesh opens
// what it needs through it in its own terms, and removes only what it marked.
func dockerOnly(t *testing.T) string {
t.Helper()
// Captured from a real machine running the container runtime and nothing else that filters:
// its nat, filter and raw tables as iptables-nft writes them.
raw, err := os.ReadFile("testdata/docker-only.nft")
if err != nil {
t.Fatal(err)
}
return string(raw)
}
const aDroppingTable = `
table inet filter {
chain input {
type filter hook input priority filter; policy drop;
ct state established,related accept
tcp dport 22 accept
}
}
`
const ufwChains = `
# Warning: table ip filter is managed by iptables-nft, do not touch!
table ip filter {
chain INPUT {
type filter hook input priority filter; policy drop;
counter packets 0 bytes 0 jump ufw-before-input
}
chain ufw-user-input {
tcp dport 22 counter packets 0 bytes 0 accept
}
chain ufw-reject-input {
counter packets 0 bytes 0 reject
}
}
`
const theMeshsOwn = `
table inet mesh {
chain input {
type filter hook input priority filter; policy drop;
iif lo accept
}
}
table inet mesh_guard {
chain prerouting {
type filter hook prerouting priority raw; policy accept;
iifname != "lo" tcp dport { 5432, 15672 } drop
}
}
`
func TestTheContainerRuntimesOwnRulesAreNotAFirewall(t *testing.T) {
if got := Refusing(dockerOnly(t), false); len(got) != 0 {
t.Errorf("the runtime's own rules read as a firewall: %v", got)
}
}
func TestTheMeshsOwnTablesAreNotAFirewall(t *testing.T) {
if got := Refusing(dockerOnly(t)+theMeshsOwn, false); len(got) != 0 {
t.Errorf("the mesh's own tables read as a found firewall: %v", got)
}
}
func TestATableThatDropsIsAFirewall(t *testing.T) {
got := Refusing(dockerOnly(t)+aDroppingTable, false)
if len(got) != 1 || got[0] != "table inet filter" {
t.Errorf("a dropping table was not named: %v", got)
}
}
func TestUfwsOwnChainsAreUfwsWhenItIsActive(t *testing.T) {
if got := Refusing(dockerOnly(t)+ufwChains, true); len(got) != 0 {
t.Errorf("ufw's own chains read as a second firewall: %v", got)
}
if got := Refusing(dockerOnly(t)+ufwChains, false); len(got) == 0 {
t.Error("iptables rules that refuse, with ufw not active, were not counted")
}
}
func TestLegacyIptablesThatRefusesIsAFirewall(t *testing.T) {
docker := "-P INPUT ACCEPT\n-P FORWARD DROP\n-P OUTPUT ACCEPT\n-N DOCKER\n-A DOCKER -i docker0 -j DROP\n"
if got := RefusingLegacy(docker); len(got) != 0 {
t.Errorf("the runtime's legacy rules read as a firewall: %v", got)
}
if got := RefusingLegacy(docker + "-A INPUT -p tcp --dport 25 -j REJECT\n"); len(got) != 1 {
t.Errorf("a legacy reject was not counted: %v", got)
}
}
// fakeUFW is ufw as far as the host can see it: a status, and user rules it prints back in its
// own canonical form — deliberately not the order the host wrote them in.
type fakeUFW struct {
active bool
installed bool
rules []string
ruleset string
firewalld bool
asked []string
}
func canonical(args []string) string {
var route, in, port, proto, comment string
for i := 0; i < len(args); i++ {
switch args[i] {
case "route":
route = "route "
case "in":
in = "in on " + args[i+2] + " "
i += 2
case "port":
port = args[i+1]
i++
case "proto":
proto = args[i+1]
i++
case "comment":
comment = args[i+1]
i++
}
}
line := route + "allow " + in + port + "/" + proto
if comment != "" {
line += " comment '" + comment + "'"
}
return line
}
func (f *fakeUFW) run(_ context.Context, name string, args ...string) (string, error) {
f.asked = append(f.asked, name+" "+strings.Join(args, " "))
switch name {
case "firewall-cmd":
if f.firewalld {
return "running\n", nil
}
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
case "nft":
return f.ruleset, nil
case "iptables-legacy", "ip6tables-legacy":
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
case "ufw":
default:
return "", fmt.Errorf("unexpected %s", name)
}
if !f.installed {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
switch {
case args[0] == "status":
if f.active {
return "Status: active\n\nTo Action From\n", nil
}
return "Status: inactive\n", nil
case args[0] == "show":
out := "Added user rules (see 'ufw status' for running firewall):\n"
for _, r := range f.rules {
out += "ufw " + r + "\n"
}
return out, nil
case args[0] == "--force" && args[1] == "enable":
f.active = true
return "Firewall is active and enabled on system startup\n", nil
case args[0] == "disable":
f.active = false
return "Firewall stopped and disabled on system startup\n", nil
case args[0] == "delete":
for i, r := range f.rules {
if strings.Join(words(r), "\x00") == strings.Join(args[1:], "\x00") {
f.rules = append(f.rules[:i], f.rules[i+1:]...)
return "Rule deleted\n", nil
}
}
return "", errors.New("Could not delete non-existent rule")
default:
f.rules = append(f.rules, canonical(args))
return "Rule added\n", nil
}
}
func (f *fakeUFW) added() int {
n := 0
for _, a := range f.asked {
if strings.HasPrefix(a, "ufw allow") || strings.HasPrefix(a, "ufw route") {
n++
}
}
return n
}
func opening(id string, port int, from, path string, to int) *declaration.Opening {
return &declaration.Opening{ID: id, Type: declaration.TypeOpening, Port: port, Protocol: "tcp",
From: from, Path: path, To: to}
}
func TestAnOpeningBecomesTheUfwRuleForItsPathAndOrigin(t *testing.T) {
for _, c := range []struct {
o *declaration.Opening
want string
}{
{opening("adoption.a", 5671, "everywhere", "incoming", 0), "allow proto tcp to any port 5671"},
{opening("adoption.b", 5432, "mesh", "incoming", 0), "allow in on mesh0 proto tcp to any port 5432"},
{opening("adoption.c", 20001, "everywhere", "forwarded", 8080), "route allow proto tcp to any port 8080"},
{opening("adoption.d", 20001, "mesh", "forwarded", 8080), "route allow in on mesh0 proto tcp to any port 8080"},
} {
if got := strings.Join(Rule(c.o), " "); got != c.want {
t.Errorf("%s: %q, want %q", c.o.ID, got, c.want)
}
}
}
func TestAnOpeningIsAddedOnceAndMarkedAsTheMeshs(t *testing.T) {
f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp"}}
o := opening("adoption.opening-tcp-5671-incoming", 5671, "everywhere", "incoming", 0)
action, err := Converge(context.Background(), f.run, o)
if err != nil || action != "created" {
t.Fatalf("first converge: %q %v", action, err)
}
if !strings.Contains(f.rules[1], "comment 'mesh-host adoption.opening-tcp-5671-incoming ") {
t.Errorf("the rule is not marked as the mesh's: %v", f.rules)
}
action, err = Converge(context.Background(), f.run, o)
if err != nil || action != "unchanged" {
t.Fatalf("second converge: %q %v", action, err)
}
if f.added() != 1 {
t.Errorf("re-converging added again: %v", f.asked)
}
}
func TestAnOpeningLostToAReloadIsAddedAgain(t *testing.T) {
f := &fakeUFW{installed: true, active: true}
o := opening("adoption.x", 5671, "everywhere", "incoming", 0)
if _, err := Converge(context.Background(), f.run, o); err != nil {
t.Fatal(err)
}
f.rules = nil // what a reload that lost the rule leaves
action, err := Converge(context.Background(), f.run, o)
if err != nil || action != "created" || len(f.rules) != 1 {
t.Fatalf("a lost opening was not put back: %q %v %v", action, err, f.rules)
}
}
func TestAChangedOpeningReplacesOnlyItsOwnRule(t *testing.T) {
f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 8080/tcp comment 'someone else'"}}
if _, err := Converge(context.Background(), f.run, opening("adoption.x", 5671, "everywhere", "incoming", 0)); err != nil {
t.Fatal(err)
}
action, err := Converge(context.Background(), f.run, opening("adoption.x", 5671, "mesh", "incoming", 0))
if err != nil || action != "updated" {
t.Fatalf("%q %v", action, err)
}
if len(f.rules) != 3 || f.rules[0] != "allow 22/tcp" || f.rules[1] != "allow 8080/tcp comment 'someone else'" ||
!strings.Contains(f.rules[2], "in on mesh0") {
t.Errorf("rules afterwards: %v", f.rules)
}
}
func TestRemovingAnOpeningRemovesOnlyWhatWasMarkedForIt(t *testing.T) {
f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 8080/tcp"}}
for _, o := range []*declaration.Opening{
opening("adoption.a", 5671, "everywhere", "incoming", 0),
opening("adoption.ab", 5000, "everywhere", "incoming", 0),
} {
if _, err := Converge(context.Background(), f.run, o); err != nil {
t.Fatal(err)
}
}
n, err := Remove(context.Background(), f.run, "adoption.a")
if err != nil || n != 1 {
t.Fatalf("removed %d: %v", n, err)
}
if len(f.rules) != 3 || f.rules[0] != "allow 22/tcp" || f.rules[1] != "allow 8080/tcp" ||
!strings.Contains(f.rules[2], "adoption.ab") {
t.Errorf("more than the marked rule went: %v", f.rules)
}
}
func TestEnableAndDisableReadBack(t *testing.T) {
f := &fakeUFW{installed: true, active: true}
if err := Disable(context.Background(), f.run); err != nil || f.active {
t.Fatalf("disable: %v", err)
}
if err := Enable(context.Background(), f.run); err != nil || !f.active {
t.Fatalf("enable: %v", err)
}
for _, a := range f.asked {
if strings.Contains(a, "reset") || strings.Contains(a, "flush") {
t.Errorf("the found firewall was reset: %s", a)
}
}
}
func TestDetectingTheFoundFirewall(t *testing.T) {
for _, c := range []struct {
name string
f *fakeUFW
want Kind
}{
{"nothing but the runtime", &fakeUFW{ruleset: dockerOnly(t)}, None},
{"ufw active", &fakeUFW{installed: true, active: true, ruleset: dockerOnly(t) + ufwChains}, UFW},
{"ufw installed and inactive", &fakeUFW{installed: true, ruleset: dockerOnly(t)}, None},
{"firewalld", &fakeUFW{firewalld: true, ruleset: dockerOnly(t)}, Unsupported},
{"an nftables table of its own", &fakeUFW{ruleset: dockerOnly(t) + aDroppingTable}, Unsupported},
{"ufw beside an nftables table", &fakeUFW{installed: true, active: true, ruleset: dockerOnly(t) + ufwChains + aDroppingTable}, Unsupported},
} {
got, name, err := Detect(context.Background(), c.f.run)
if err != nil {
t.Fatalf("%s: %v", c.name, err)
}
if got != c.want {
t.Errorf("%s: detected %s (%s), want %s", c.name, got, name, c.want)
}
if got == Unsupported && name == "" {
t.Errorf("%s: an unsupported firewall was not named", c.name)
}
}
}
+297
View File
@@ -0,0 +1,297 @@
# Warning: table ip nat is managed by iptables-nft, do not touch!
table ip nat {
chain DOCKER {
iifname != "br-c70303d221ee" tcp dport 5680 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-c70303d221ee" tcp dport 15673 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-3636e05760a9" tcp dport 59000 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-3636e05760a9" tcp dport 59001 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-3636e05760a9" tcp dport 55672 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-3636e05760a9" tcp dport 55673 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-3636e05760a9" tcp dport 55432 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "docker0" tcp dport 57732 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "docker0" tcp dport 57733 counter packets 0 bytes 0 xt target "DNAT"
iifname != "docker0" tcp dport 5314 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-613eb68ef5fb" tcp dport 4848 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-b3240c822bce" tcp dport 5679 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-b3240c822bce" tcp dport 15672 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-4b504efd6080" tcp dport 9000 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-4b504efd6080" tcp dport 9001 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-ef6df03f71a0" tcp dport 5432 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-ef6df03f71a0" tcp dport 8081 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-ec480f77ac34" tcp dport 6379 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-af4c9c2aa60a" tcp dport 8001 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-669fda75f1ac" tcp dport 28080 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-af4c9c2aa60a" tcp dport 6789 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-af4c9c2aa60a" tcp dport 8770 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-af4c9c2aa60a" tcp dport 1212 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-af4c9c2aa60a" tcp dport 80 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-af4c9c2aa60a" tcp dport 443 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "docker0" tcp dport 55541 counter packets 0 bytes 0 xt target "DNAT"
}
chain PREROUTING {
type nat hook prerouting priority dstnat; policy accept;
xt match "addrtype" counter packets 437947 bytes 71410534 jump DOCKER
}
chain OUTPUT {
type nat hook output priority dstnat; policy accept;
ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 5761 bytes 423317 jump DOCKER
}
chain POSTROUTING {
type nat hook postrouting priority srcnat; policy accept;
ip saddr 172.22.0.0/16 oifname != "br-65f6dc782562" counter packets 124 bytes 16328 xt target "MASQUERADE"
ip saddr 172.28.0.0/16 oifname != "br-669fda75f1ac" counter packets 127 bytes 16928 xt target "MASQUERADE"
ip saddr 172.31.0.0/16 oifname != "br-ec480f77ac34" counter packets 127 bytes 16928 xt target "MASQUERADE"
ip saddr 192.168.48.0/20 oifname != "br-ef6df03f71a0" counter packets 127 bytes 16928 xt target "MASQUERADE"
ip saddr 172.25.0.0/16 oifname != "br-4b504efd6080" counter packets 129 bytes 17052 xt target "MASQUERADE"
ip saddr 192.168.32.0/20 oifname != "br-613eb68ef5fb" counter packets 1124 bytes 76748 xt target "MASQUERADE"
ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 1833 bytes 150990 xt target "MASQUERADE"
ip saddr 172.18.0.0/16 oifname != "br-07a5e2f2c42f" counter packets 504 bytes 65112 xt target "MASQUERADE"
ip saddr 172.27.0.0/16 oifname != "br-160f55da427c" counter packets 508 bytes 65784 xt target "MASQUERADE"
ip saddr 192.168.16.0/20 oifname != "br-dba077b9b543" counter packets 503 bytes 64784 xt target "MASQUERADE"
ip saddr 192.168.64.0/20 oifname != "br-d44fef8fd602" counter packets 1282 bytes 111308 xt target "MASQUERADE"
ip saddr 172.30.0.0/16 oifname != "br-af4c9c2aa60a" counter packets 2503 bytes 190324 xt target "MASQUERADE"
ip saddr 172.21.0.0/16 oifname != "br-9d6c95e8d80c" counter packets 1289 bytes 112644 xt target "MASQUERADE"
ip saddr 172.23.0.0/16 oifname != "br-679db9b21e00" counter packets 506 bytes 65384 xt target "MASQUERADE"
ip saddr 172.24.0.0/16 oifname != "br-40094534a5ee" counter packets 508 bytes 65784 xt target "MASQUERADE"
ip saddr 172.26.0.0/16 oifname != "br-3dcb6ef83ea1" counter packets 508 bytes 65784 xt target "MASQUERADE"
ip saddr 172.20.0.0/16 oifname != "br-3a760a74f4f6" counter packets 1153 bytes 104344 xt target "MASQUERADE"
ip saddr 192.168.80.0/20 oifname != "br-3636e05760a9" counter packets 546 bytes 70540 xt target "MASQUERADE"
ip saddr 172.29.0.0/16 oifname != "br-b3240c822bce" counter packets 551 bytes 71492 xt target "MASQUERADE"
ip saddr 172.19.0.0/16 oifname != "br-c70303d221ee" counter packets 1136 bytes 106592 xt target "MASQUERADE"
}
}
# Warning: table ip filter is managed by iptables-nft, do not touch!
table ip filter {
chain DOCKER {
ip daddr 172.17.0.5 iifname != "docker0" oifname "docker0" tcp dport 5432 counter packets 0 bytes 0 accept
ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 443 counter packets 0 bytes 0 accept
ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 80 counter packets 0 bytes 0 accept
ip daddr 172.30.0.10 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 3000 counter packets 0 bytes 0 accept
ip daddr 172.30.0.5 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 8000 counter packets 0 bytes 0 accept
ip daddr 172.30.0.3 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 6789 counter packets 0 bytes 0 accept
ip daddr 172.28.0.3 iifname != "br-669fda75f1ac" oifname "br-669fda75f1ac" tcp dport 8080 counter packets 0 bytes 0 accept
ip daddr 172.30.0.4 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 5540 counter packets 0 bytes 0 accept
ip daddr 172.31.0.2 iifname != "br-ec480f77ac34" oifname "br-ec480f77ac34" tcp dport 6379 counter packets 0 bytes 0 accept
ip daddr 192.168.48.3 iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" tcp dport 8081 counter packets 0 bytes 0 accept
ip daddr 192.168.48.2 iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" tcp dport 5432 counter packets 0 bytes 0 accept
ip daddr 172.25.0.2 iifname != "br-4b504efd6080" oifname "br-4b504efd6080" tcp dport 9001 counter packets 0 bytes 0 accept
ip daddr 172.25.0.2 iifname != "br-4b504efd6080" oifname "br-4b504efd6080" tcp dport 9000 counter packets 0 bytes 0 accept
ip daddr 172.29.0.2 iifname != "br-b3240c822bce" oifname "br-b3240c822bce" tcp dport 15672 counter packets 0 bytes 0 accept
ip daddr 172.29.0.2 iifname != "br-b3240c822bce" oifname "br-b3240c822bce" tcp dport 5672 counter packets 0 bytes 0 accept
ip daddr 192.168.32.2 iifname != "br-613eb68ef5fb" oifname "br-613eb68ef5fb" tcp dport 1433 counter packets 0 bytes 0 accept
ip daddr 172.17.0.3 iifname != "docker0" oifname "docker0" tcp dport 5000 counter packets 0 bytes 0 accept
ip daddr 172.17.0.2 iifname != "docker0" oifname "docker0" tcp dport 15672 counter packets 0 bytes 0 accept
ip daddr 172.17.0.2 iifname != "docker0" oifname "docker0" tcp dport 5672 counter packets 0 bytes 0 accept
ip daddr 192.168.80.4 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 5432 counter packets 0 bytes 0 accept
ip daddr 192.168.80.3 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 15672 counter packets 0 bytes 0 accept
ip daddr 192.168.80.3 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 5672 counter packets 0 bytes 0 accept
ip daddr 192.168.80.2 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 9001 counter packets 0 bytes 0 accept
ip daddr 192.168.80.2 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 9000 counter packets 0 bytes 0 accept
ip daddr 172.19.0.2 iifname != "br-c70303d221ee" oifname "br-c70303d221ee" tcp dport 15672 counter packets 0 bytes 0 accept
ip daddr 172.19.0.2 iifname != "br-c70303d221ee" oifname "br-c70303d221ee" tcp dport 5672 counter packets 0 bytes 0 accept
iifname != "br-c70303d221ee" oifname "br-c70303d221ee" counter packets 0 bytes 0 drop
iifname != "br-b3240c822bce" oifname "br-b3240c822bce" counter packets 0 bytes 0 drop
iifname != "br-3636e05760a9" oifname "br-3636e05760a9" counter packets 0 bytes 0 drop
iifname != "br-3a760a74f4f6" oifname "br-3a760a74f4f6" counter packets 0 bytes 0 drop
iifname != "br-3dcb6ef83ea1" oifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 drop
iifname != "br-40094534a5ee" oifname "br-40094534a5ee" counter packets 0 bytes 0 drop
iifname != "br-679db9b21e00" oifname "br-679db9b21e00" counter packets 0 bytes 0 drop
iifname != "br-9d6c95e8d80c" oifname "br-9d6c95e8d80c" counter packets 0 bytes 0 drop
iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
iifname != "br-d44fef8fd602" oifname "br-d44fef8fd602" counter packets 0 bytes 0 drop
iifname != "br-dba077b9b543" oifname "br-dba077b9b543" counter packets 0 bytes 0 drop
iifname != "br-160f55da427c" oifname "br-160f55da427c" counter packets 0 bytes 0 drop
iifname != "br-07a5e2f2c42f" oifname "br-07a5e2f2c42f" counter packets 0 bytes 0 drop
iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop
iifname != "br-613eb68ef5fb" oifname "br-613eb68ef5fb" counter packets 0 bytes 0 drop
iifname != "br-4b504efd6080" oifname "br-4b504efd6080" counter packets 0 bytes 0 drop
iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" counter packets 0 bytes 0 drop
iifname != "br-ec480f77ac34" oifname "br-ec480f77ac34" counter packets 0 bytes 0 drop
iifname != "br-669fda75f1ac" oifname "br-669fda75f1ac" counter packets 0 bytes 0 drop
iifname != "br-65f6dc782562" oifname "br-65f6dc782562" counter packets 0 bytes 0 drop
}
chain DOCKER-FORWARD {
counter packets 6530319 bytes 11196484299 jump DOCKER-CT
counter packets 3312487 bytes 5001091084 jump DOCKER-INTERNAL
counter packets 3312487 bytes 5001091084 jump DOCKER-BRIDGE
iifname "br-c70303d221ee" counter packets 0 bytes 0 accept
iifname "br-b3240c822bce" counter packets 0 bytes 0 accept
iifname "br-3636e05760a9" counter packets 43 bytes 9355 accept
iifname "br-3a760a74f4f6" counter packets 0 bytes 0 accept
iifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 accept
iifname "br-40094534a5ee" counter packets 0 bytes 0 accept
iifname "br-679db9b21e00" counter packets 0 bytes 0 accept
iifname "br-9d6c95e8d80c" counter packets 0 bytes 0 accept
iifname "br-af4c9c2aa60a" counter packets 2761311 bytes 4959915711 accept
iifname "br-d44fef8fd602" counter packets 0 bytes 0 accept
iifname "br-dba077b9b543" counter packets 0 bytes 0 accept
iifname "br-160f55da427c" counter packets 0 bytes 0 accept
iifname "br-07a5e2f2c42f" counter packets 0 bytes 0 accept
iifname "docker0" counter packets 460394 bytes 25754554 accept
iifname "br-613eb68ef5fb" counter packets 10805 bytes 1792862 accept
iifname "br-4b504efd6080" counter packets 33 bytes 2892 accept
iifname "br-ef6df03f71a0" counter packets 0 bytes 0 accept
iifname "br-ec480f77ac34" counter packets 0 bytes 0 accept
iifname "br-669fda75f1ac" counter packets 0 bytes 0 accept
iifname "br-65f6dc782562" counter packets 0 bytes 0 accept
}
chain DOCKER-BRIDGE {
oifname "br-c70303d221ee" counter packets 0 bytes 0 jump DOCKER
oifname "br-b3240c822bce" counter packets 0 bytes 0 jump DOCKER
oifname "br-3636e05760a9" counter packets 0 bytes 0 jump DOCKER
oifname "br-3a760a74f4f6" counter packets 0 bytes 0 jump DOCKER
oifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 jump DOCKER
oifname "br-40094534a5ee" counter packets 0 bytes 0 jump DOCKER
oifname "br-679db9b21e00" counter packets 0 bytes 0 jump DOCKER
oifname "br-9d6c95e8d80c" counter packets 0 bytes 0 jump DOCKER
oifname "br-af4c9c2aa60a" counter packets 118 bytes 8400 jump DOCKER
oifname "br-d44fef8fd602" counter packets 0 bytes 0 jump DOCKER
oifname "br-dba077b9b543" counter packets 0 bytes 0 jump DOCKER
oifname "br-160f55da427c" counter packets 0 bytes 0 jump DOCKER
oifname "br-07a5e2f2c42f" counter packets 0 bytes 0 jump DOCKER
oifname "docker0" counter packets 0 bytes 0 jump DOCKER
oifname "br-613eb68ef5fb" counter packets 0 bytes 0 jump DOCKER
oifname "br-4b504efd6080" counter packets 0 bytes 0 jump DOCKER
oifname "br-ef6df03f71a0" counter packets 0 bytes 0 jump DOCKER
oifname "br-ec480f77ac34" counter packets 0 bytes 0 jump DOCKER
oifname "br-669fda75f1ac" counter packets 0 bytes 0 jump DOCKER
oifname "br-65f6dc782562" counter packets 0 bytes 0 jump DOCKER
}
chain DOCKER-CT {
oifname "br-c70303d221ee" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-b3240c822bce" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-3636e05760a9" xt match "conntrack" counter packets 35 bytes 23113 accept
oifname "br-3a760a74f4f6" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-3dcb6ef83ea1" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-40094534a5ee" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-679db9b21e00" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-9d6c95e8d80c" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-af4c9c2aa60a" xt match "conntrack" counter packets 2488066 bytes 1053784742 accept
oifname "br-d44fef8fd602" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-dba077b9b543" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-160f55da427c" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-07a5e2f2c42f" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "docker0" xt match "conntrack" counter packets 666252 bytes 5079577802 accept
oifname "br-613eb68ef5fb" xt match "conntrack" counter packets 7926 bytes 7636543 accept
oifname "br-4b504efd6080" xt match "conntrack" counter packets 29 bytes 10870 accept
oifname "br-ef6df03f71a0" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-ec480f77ac34" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-669fda75f1ac" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-65f6dc782562" xt match "conntrack" counter packets 0 bytes 0 accept
}
chain DOCKER-INTERNAL {
}
chain FORWARD {
type filter hook forward priority filter; policy drop;
counter packets 33747166 bytes 176750349038 jump DOCKER-USER
counter packets 6530319 bytes 11196484299 jump DOCKER-FORWARD
}
chain DOCKER-USER {
oifname "mlab*" counter packets 15657582 bytes 162801189460 accept
iifname "mlab*" counter packets 10958315 bytes 687322055 accept
oifname "incusbr0" counter packets 388768 bytes 2053271282 accept
iifname "incusbr0" counter packets 212182 bytes 12081942 accept
}
}
# Warning: table ip6 nat is managed by iptables-nft, do not touch!
table ip6 nat {
chain DOCKER {
}
chain PREROUTING {
type nat hook prerouting priority dstnat; policy accept;
xt match "addrtype" counter packets 532 bytes 113834 jump DOCKER
}
chain OUTPUT {
type nat hook output priority dstnat; policy accept;
ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
}
table ip6 filter {
chain DOCKER {
}
chain DOCKER-FORWARD {
counter packets 0 bytes 0 jump DOCKER-CT
counter packets 0 bytes 0 jump DOCKER-INTERNAL
counter packets 0 bytes 0 jump DOCKER-BRIDGE
}
chain DOCKER-BRIDGE {
}
chain DOCKER-CT {
}
chain DOCKER-INTERNAL {
}
chain FORWARD {
type filter hook forward priority filter; policy accept;
counter packets 0 bytes 0 jump DOCKER-USER
counter packets 0 bytes 0 jump DOCKER-FORWARD
}
chain DOCKER-USER {
}
}
table ip raw {
chain PREROUTING {
type filter hook prerouting priority raw; policy accept;
ip daddr 172.19.0.2 iifname != "br-c70303d221ee" counter packets 0 bytes 0 drop
ip daddr 192.168.80.2 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop
ip daddr 192.168.80.3 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 15673 counter packets 0 bytes 0 drop
ip daddr 192.168.80.4 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop
ip daddr 172.17.0.2 iifname != "docker0" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 57732 counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 57733 counter packets 0 bytes 0 drop
ip daddr 172.17.0.3 iifname != "docker0" counter packets 0 bytes 0 drop
ip daddr 172.17.0.4 iifname != "docker0" counter packets 0 bytes 0 drop
ip daddr 192.168.32.2 iifname != "br-613eb68ef5fb" counter packets 0 bytes 0 drop
ip daddr 172.30.0.7 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 172.29.0.2 iifname != "br-b3240c822bce" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 15672 counter packets 0 bytes 0 drop
ip daddr 172.25.0.2 iifname != "br-4b504efd6080" counter packets 0 bytes 0 drop
ip daddr 172.30.0.9 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 192.168.48.2 iifname != "br-ef6df03f71a0" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 5432 counter packets 0 bytes 0 drop
ip daddr 192.168.48.3 iifname != "br-ef6df03f71a0" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 8081 counter packets 0 bytes 0 drop
ip daddr 172.30.0.8 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 172.31.0.2 iifname != "br-ec480f77ac34" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 6379 counter packets 0 bytes 0 drop
ip daddr 172.30.0.4 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 8001 counter packets 0 bytes 0 drop
ip daddr 172.31.0.3 iifname != "br-ec480f77ac34" counter packets 0 bytes 0 drop
ip daddr 172.28.0.2 iifname != "br-669fda75f1ac" counter packets 0 bytes 0 drop
ip daddr 172.30.0.6 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 172.28.0.3 iifname != "br-669fda75f1ac" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 28080 counter packets 0 bytes 0 drop
ip daddr 172.30.0.3 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 6789 counter packets 0 bytes 0 drop
ip daddr 172.30.0.5 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 172.22.0.2 iifname != "br-65f6dc782562" counter packets 0 bytes 0 drop
ip daddr 172.30.0.10 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 172.22.0.3 iifname != "br-65f6dc782562" counter packets 0 bytes 0 drop
ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 172.17.0.5 iifname != "docker0" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 55541 counter packets 0 bytes 0 drop
}
}
table ip mangle {
chain FORWARD {
type filter hook forward priority mangle; policy accept;
tcp flags & (syn | rst) == syn counter packets 13760 bytes 825476 xt target "TCPMSS"
}
}
+17
View File
@@ -83,6 +83,23 @@ type State struct {
// nothing here is ever removed as an orphan — what is held is not the host's to remove, even
// when its module is unassigned.
Held []Held `json:"held,omitempty"`
// Firewall is the firewall found on this machine when it was first adopted, and whether the
// mesh has since retired it (novox/hq ADR 0100). Nil on a node that was never adopted.
Firewall *FoundFirewall `json:"firewall,omitempty"`
}
// FoundFirewall is what the host found filtering this machine, and what it did about it.
type FoundFirewall struct {
// Kind is ufw or none: an unsupported kind is refused adoption, never recorded.
Kind string `json:"kind"`
// WasActive is whether it was in force when found — which is what converging the node
// retires, and returning it to adopted restores.
WasActive bool `json:"was_active,omitempty"`
// DisabledByMesh is set when converging retired it, so returning to adopted enables it again
// and nothing else ever does.
DisabledByMesh bool `json:"disabled_by_mesh,omitempty"`
FoundAt time.Time `json:"found_at"`
}
// Held is one file or container found on an adopted node — present at a declared path or name,
+3
View File
@@ -178,6 +178,9 @@ func everyShape() []declaration.Type {
// it: the mesh's own code runs as a process on the machine, and only software that
// genuinely needs isolation asks for a container.
declaration.TypeProcess,
// An opening is a rule in the firewall found on the machine, which a partial host neither
// has nor can manage (novox/hq ADR 0100).
declaration.TypeOpening,
}
}