diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 2e25d56..d8a4926 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -150,8 +150,21 @@ func ApplyKeeping( declared[r.Identity()] = true } + // Which firewall is found here, before anything else, since an unsupported one refuses the + // whole declaration (novox/hq ADR 0100). Nothing for a converged node. + fw, err := foundFirewall(ctx, d, &known, run, log) + if err != nil { + return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report} + } + for _, orphan := range known.Orphans(declared, origin) { - action, detail, err := remove(ctx, sys, orphan, run) + var action, detail string + var err error + if declaration.Type(orphan.Type) == declaration.TypeOpening { + action, detail, err = removeOpening(ctx, orphan, run, known.Firewall) + } else { + action, detail, err = remove(ctx, sys, orphan, run) + } if err != nil { return report, known, &Error{Resource: orphan.ID, Err: err, Done: report} } @@ -235,7 +248,13 @@ func ApplyKeeping( } was, _ := known.Find(resource.Identity()) - outcome, err := applyOne(ctx, sys, resource, run, changed, declares, was, unseal) + var outcome Outcome + var err error + if o, isOpening := resource.(*declaration.Opening); isOpening { + outcome, err = applyOpening(ctx, o, run, fw) + } else { + outcome, err = applyOne(ctx, sys, resource, run, changed, declares, was, unseal) + } if err != nil { failed := &Error{Resource: resource.Identity(), Err: err, Done: report} failures = append(failures, failed) @@ -293,6 +312,14 @@ func ApplyKeeping( } } + // A converged node whose found firewall was in force retires it only now, once everything — + // the mesh's derived filter among it — applied cleanly (novox/hq ADR 0100). + if len(failures) == 0 { + if err := retireFirewall(ctx, d, &known, run, log); err != nil { + return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report} + } + } + if len(failures) > 0 { // The first, carrying everything that did happen. One error is what the caller reports // and what a person reads first; the rest are in the report, which is what the mesh diff --git a/internal/apply/opening.go b/internal/apply/opening.go new file mode 100644 index 0000000..3adef33 --- /dev/null +++ b/internal/apply/opening.go @@ -0,0 +1,109 @@ +package apply + +import ( + "context" + "fmt" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/firewall" + "github.com/novox/mesh-host/internal/store" +) + +// foundFirewall settles, before anything else in an apply, which firewall this node has — and on +// an adopted node that the mesh had converged, puts it back in force first (novox/hq ADR 0100). +// +// Only an adopted node asks. It is detected on every apply rather than remembered, so a firewall +// switched on after adoption is spoken to from the next reconcile; what is remembered is what was +// found first, and whether the mesh retired it. An unsupported firewall refuses the whole +// declaration: the mesh could neither open what it needs through it nor say what it would close. +func foundFirewall(ctx context.Context, d *declaration.Declaration, known *store.State, run Runner, + log func(string)) (firewall.Kind, error) { + if d.Adoption == nil { + return "", nil + } + rec := known.Firewall + if rec != nil && rec.DisabledByMesh && rec.Kind == string(firewall.UFW) { + // Returned to adopted: the found firewall is enabled again before the openings are + // converged through it, and the derived filter is gone with this declaration. + if err := firewall.Enable(ctx, run); err != nil { + return "", err + } + rec.DisabledByMesh = false + log(" enabled ufw again: this node is adopted, and the firewall found on it is in force") + } + kind, name, err := firewall.Detect(ctx, run) + if err != nil { + return "", err + } + if kind == firewall.Unsupported { + return "", fmt.Errorf( + "this machine is filtered by %s, and no host speaks that firewall yet. An adopted node "+ + "keeps the firewall it was found with, so the mesh could neither open what it needs "+ + "through it nor say what it would close; this declaration is refused whole", name) + } + if rec == nil { + rec = &store.FoundFirewall{Kind: string(kind), WasActive: kind == firewall.UFW, + FoundAt: time.Now().UTC()} + } else { + rec.Kind = string(kind) + rec.WasActive = rec.WasActive || kind == firewall.UFW + } + known.Firewall = rec + return kind, nil +} + +// retireFirewall disables the found firewall once a converged declaration has applied cleanly, +// which is when the mesh's derived filter has taken its place. Disabled, never flushed: its +// configuration stays on disk for a return to adopted, and the container runtime's rules are not +// its to take. +func retireFirewall(ctx context.Context, d *declaration.Declaration, known *store.State, run Runner, + log func(string)) error { + rec := known.Firewall + if d.Adoption != nil || rec == nil || rec.Kind != string(firewall.UFW) || !rec.WasActive || + rec.DisabledByMesh { + return nil + } + if err := firewall.Disable(ctx, run); err != nil { + return err + } + rec.DisabledByMesh = true + log(" disabled ufw: this node is converged and filtered by the mesh; ufw's configuration is left on disk") + return nil +} + +// applyOpening makes one opening true through the firewall found here. +func applyOpening(ctx context.Context, o *declaration.Opening, run Runner, kind firewall.Kind) (Outcome, error) { + out := begin(o) + switch kind { + case firewall.None: + out.Action = "unchanged" + out.Detail = "no firewall found; nothing filters this port" + return out, nil + case firewall.UFW: + action, err := firewall.Converge(ctx, run, o) + if err != nil { + return out, err + } + out.Action = action + out.Detail = "through ufw, marked " + firewall.Mark(o) + return out, nil + } + return out, fmt.Errorf("no firewall is known for this node, so %s cannot be opened", o.Target()) +} + +// removeOpening deletes the rules the mesh marked for an opening no longer declared, and nothing +// the machine had before. +func removeOpening(ctx context.Context, a store.Applied, run Runner, rec *store.FoundFirewall) (string, string, error) { + if rec == nil || rec.Kind != string(firewall.UFW) { + return "forgotten", "no firewall held a rule for it", nil + } + n, err := firewall.Remove(ctx, run, a.ID) + if err != nil { + return "", "", err + } + if n == 0 { + return "forgotten", "ufw held no rule marked for it", nil + } + return "removed", fmt.Sprintf("%d ufw rule(s) marked as the mesh's deleted", n), nil +} diff --git a/internal/apply/opening_test.go b/internal/apply/opening_test.go new file mode 100644 index 0000000..d51bda5 --- /dev/null +++ b/internal/apply/opening_test.go @@ -0,0 +1,204 @@ +package apply + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force; converging the +// node retires it by disabling it, and returning the node to adopted enables it again. + +type ufwMachine struct { + installed, active bool + rules []string + ruleset string + asked []string +} + +func (u *ufwMachine) run(_ context.Context, name string, args ...string) (string, error) { + u.asked = append(u.asked, name+" "+strings.Join(args, " ")) + switch name { + case "nft": + return u.ruleset, nil + case "ufw": + if !u.installed { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + default: + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + switch args[0] { + case "status": + if u.active { + return "Status: active\n", nil + } + return "Status: inactive\n", nil + case "show": + out := "Added user rules (see 'ufw status' for running firewall):\n" + for _, r := range u.rules { + out += "ufw " + r + "\n" + } + return out, nil + case "--force": + u.active = true + return "", nil + case "disable": + u.active = false + return "", nil + case "delete": + want := strings.Join(args[1:], " ") + for i, r := range u.rules { + if strings.ReplaceAll(r, "'", "") == want { + u.rules = append(u.rules[:i], u.rules[i+1:]...) + return "", nil + } + } + return "", errors.New("Could not delete non-existent rule") + default: + // Printed back the way it was given, with the comment quoted as ufw does. + line := strings.Join(args[:len(args)-1], " ") + " '" + args[len(args)-1] + "'" + u.rules = append(u.rules, line) + return "", nil + } +} + +func (u *ufwMachine) index(prefix string) int { + for i, a := range u.asked { + if strings.HasPrefix(a, prefix) { + return i + } + } + return -1 +} + +const busOpening = `{"id":"adoption.opening-tcp-5671-incoming","type":"opening","port":5671,"protocol":"tcp","from":"everywhere","path":"incoming"}` + +func withConf(dir string) string { + return `{"id":"x.conf","type":"file","path":"` + filepath.Join(dir, "x.conf") + `","content":"x\n"}` +} + +func applyWith(t *testing.T, d *declaration.Declaration, known store.State, run Runner) (Report, store.State, error) { + t.Helper() + return ApplyKeeping(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil, + KeepIn(t.TempDir())) +} + +func TestAnOpeningOnAMachineWithNoFirewallChangesNothing(t *testing.T) { + dir := t.TempDir() + u := &ufwMachine{} + report, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + o := outcomeOf(report, "adoption.opening-tcp-5671-incoming") + if o.Action != "unchanged" || !strings.Contains(o.Detail, "nothing filters this port") { + t.Errorf("an opening with no firewall: %+v", o) + } + if state.Firewall == nil || state.Firewall.Kind != "none" { + t.Errorf("the firewall found was not recorded: %+v", state.Firewall) + } +} + +func TestAnUnsupportedFirewallRefusesTheWholeDeclaration(t *testing.T) { + dir := t.TempDir() + u := &ufwMachine{ruleset: "table inet filter {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t}\n}\n"} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err == nil || !strings.Contains(err.Error(), "no host speaks that firewall") { + t.Fatalf("an unsupported firewall was not refused: %v", err) + } + if _, statErr := os.Stat(filepath.Join(dir, "x.conf")); !errors.Is(statErr, os.ErrNotExist) { + t.Error("part of a refused declaration was applied") + } + if state.Firewall != nil { + t.Errorf("an unsupported firewall was recorded: %+v", state.Firewall) + } +} + +func TestConvergingRetiresTheFoundFirewallAndReturningRestoresIt(t *testing.T) { + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} + + // Adopted: the opening goes through ufw. + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + if len(u.rules) != 2 || !u.active { + t.Fatalf("adopted: rules %v, active %v", u.rules, u.active) + } + if state.Firewall == nil || state.Firewall.Kind != "ufw" || !state.Firewall.WasActive { + t.Fatalf("adopted: firewall recorded as %+v", state.Firewall) + } + + // Converged: the opening's rule goes, and only then is ufw disabled — never reset. + u.asked = nil + converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) + _, state, err = applyWith(t, converged, state, u.run) + if err != nil { + t.Fatal(err) + } + if u.active || !state.Firewall.DisabledByMesh { + t.Fatalf("converged: ufw still active (%v) or not recorded as retired (%+v)", u.active, state.Firewall) + } + if len(u.rules) != 1 || u.rules[0] != "allow 22/tcp" { + t.Errorf("converged: the operator's rules were touched, or the mesh's left: %v", u.rules) + } + if del, dis := u.index("ufw delete"), u.index("ufw disable"); del < 0 || dis < del { + t.Errorf("converged: the opening was not removed before ufw was disabled: %v", u.asked) + } + for _, a := range u.asked { + if strings.Contains(a, "reset") { + t.Errorf("converged: ufw was reset: %s", a) + } + } + + // Converged again: nothing more to retire. + u.asked = nil + if _, state, err = applyWith(t, converged, state, u.run); err != nil { + t.Fatal(err) + } + if u.index("ufw") >= 0 { + t.Errorf("a converged node kept talking to a retired ufw: %v", u.asked) + } + + // Returned to adopted: ufw is enabled before the opening is converged through it. + u.asked = nil + _, state, err = applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), state, u.run) + if err != nil { + t.Fatal(err) + } + if !u.active || state.Firewall.DisabledByMesh { + t.Fatalf("returned: ufw active %v, record %+v", u.active, state.Firewall) + } + if en, add := u.index("ufw --force enable"), u.index("ufw allow"); en < 0 || add < en { + t.Errorf("returned: ufw was not enabled before the opening was added: %v", u.asked) + } + if len(u.rules) != 2 { + t.Errorf("returned: the opening was not converged again: %v", u.rules) + } +} + +func TestAConvergedNodeThatWasNeverAdoptedNeverAsksAboutAFirewall(t *testing.T) { + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true} + if _, _, err := applyWith(t, parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`), store.State{}, u.run); err != nil { + t.Fatal(err) + } + if len(u.asked) != 0 { + t.Errorf("a converged apply asked the machine about its firewall: %v", u.asked) + } +} + +func TestAnOpeningOnAConvergedNodeIsRefused(t *testing.T) { + if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` + busOpening + `]}`)); err == nil { + t.Error("an opening was accepted on a node the declaration does not say is adopted") + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index 41605f6..991654f 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -78,6 +78,12 @@ const ( // cadence. Tools, hooks and event consumers are not separate modes: they are loaded by a tool // host, which is itself a process that stays up. TypeProcess Type = "process" + + // TypeOpening is a port the mesh needs reachable on an adopted node, converged through the + // firewall found there in that firewall's own terms (novox/hq ADR 0100). A state, not a + // command: the host adds the rule it marks as the mesh's when it is missing, and removes only + // what it marked — which is what lets it travel over the link. + TypeOpening Type = "opening" ) // Resource is one thing that should be true of the machine. @@ -603,6 +609,74 @@ func (s *Service) validate(where string, _ bool) []string { return problems } +// Opening is a port reachable on an adopted node, from where, and on which path. +// +// **From** is everywhere or mesh — the private network, by its interface. **Path** is incoming, +// for something listening on the machine, or forwarded, for a published container port: the found +// firewall sees a published port after the runtime has translated it, so a forwarded opening names +// the container's own port in To as well as the machine's in Port. +type Opening struct { + ID string `json:"id"` + Type Type `json:"type"` + Port int `json:"port"` + Protocol string `json:"protocol"` + From string `json:"from"` + Path string `json:"path"` + To int `json:"to,omitempty"` +} + +// Where an opening admits from, and the path it is on. +const ( + FromEverywhere = "everywhere" + FromMesh = "mesh" + PathIncoming = "incoming" + PathForwarded = "forwarded" +) + +func (o *Opening) Identity() string { return o.ID } +func (o *Opening) Kind() Type { return TypeOpening } + +func (o *Opening) Target() string { + if o.Path == PathForwarded { + return fmt.Sprintf("%s/%d forwarded to %d from %s", o.Protocol, o.Port, o.To, o.From) + } + return fmt.Sprintf("%s/%d %s from %s", o.Protocol, o.Port, o.Path, o.From) +} + +func (o *Opening) validate(where string, _ bool) []string { + var problems []string + if o.Port < 1 || o.Port > 65535 { + problems = append(problems, fmt.Sprintf("%s: an opening's port is 1-65535, not %d", where, o.Port)) + } + if o.Protocol != "tcp" && o.Protocol != "udp" { + problems = append(problems, fmt.Sprintf("%s: an opening is tcp or udp, not %q", where, o.Protocol)) + } + if o.From != FromEverywhere && o.From != FromMesh { + problems = append(problems, fmt.Sprintf( + "%s: an opening is from %q or %q, not %q", where, FromEverywhere, FromMesh, o.From)) + } + switch o.Path { + case PathIncoming: + if o.To != 0 { + problems = append(problems, where+ + ": an incoming opening names no container port; only a forwarded one does") + } + case PathForwarded: + if o.To < 1 || o.To > 65535 { + problems = append(problems, where+ + ": a forwarded opening names the container's port it reaches, as to, 1-65535") + } + default: + problems = append(problems, fmt.Sprintf( + "%s: an opening's path is %q or %q, not %q", where, PathIncoming, PathForwarded, o.Path)) + } + if !strings.HasPrefix(o.ID, AdoptionPrefix) { + problems = append(problems, fmt.Sprintf( + "%s: an opening is the mesh's own, so its id starts %q", where, AdoptionPrefix)) + } + return problems +} + // Package is a package that should be present. // // Present is the whole of what it asserts, never a version: version is the package manager's @@ -810,6 +884,8 @@ func newOf(t Type) Resource { return &Access{} case TypeProcess: return &Process{} + case TypeOpening: + return &Opening{} } return nil } @@ -818,7 +894,7 @@ func newOf(t Type) Resource { func Vocabulary() []Type { return []Type{ TypeAccess, TypeAction, TypeArchive, TypeContainer, TypeDirectory, TypeFile, - TypeNetwork, TypePackage, TypeProcess, TypeService, TypeUser, + TypeNetwork, TypeOpening, TypePackage, TypeProcess, TypeService, TypeUser, } } @@ -1051,6 +1127,18 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { d.Resources = append(d.Resources, resource) } problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...) + if env.Adoption == nil { + for _, r := range d.Resources { + if r.Kind() == TypeOpening { + // On a converged node the mesh's own filter admits what is declared, and the + // found firewall is retired; an opening there would be a rule in a firewall the + // mesh has disabled (novox/hq ADR 0100). + problems = append(problems, fmt.Sprintf( + "resource %q: an opening is for an adopted node, and this declaration does not "+ + "say the node is adopted", r.Identity())) + } + } + } if len(problems) > 0 { return nil, &RefusalError{Problems: problems} diff --git a/internal/declaration/declaration_test.go b/internal/declaration/declaration_test.go index b2fd73e..d0823e0 100644 --- a/internal/declaration/declaration_test.go +++ b/internal/declaration/declaration_test.go @@ -266,7 +266,7 @@ func TestAFieldTheNewTypesDoNotUseIsRefused(t *testing.T) { } } -func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) { +func TestTheVocabularyIsTheTwelveShapesTheMeshNeeds(t *testing.T) { // Six of them the bootstrap uses (novox/hq 07-the-foundation.md), and removing one is a // failing test rather than a discovery during a first-node install. // @@ -282,7 +282,7 @@ func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) { } for _, want := range []Type{ TypeDirectory, TypeFile, TypeService, TypePackage, TypeContainer, TypeAction, - TypeUser, TypeArchive, TypeNetwork, TypeAccess, TypeProcess, + TypeUser, TypeArchive, TypeNetwork, TypeAccess, TypeProcess, TypeOpening, } { if !speaks[want] { t.Errorf("the host no longer speaks %q", want) @@ -309,8 +309,12 @@ func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) { // It is a full-host shape rather than a portable one: it needs a process supervisor to install // into. It does NOT need a container runtime, which is the point — only software that // genuinely needs isolation asks for a container. - if len(speaks) != 11 { - t.Errorf("the vocabulary is %d shapes rather than 11; every addition widens what a compromised "+ + // + // `opening` is the twelfth, and novox/hq ADR 0100 is its decision: on an adopted node the + // firewall found there stays in force, and what the mesh needs reachable is converged through + // it as a state the host marks as the mesh's — never a command, which the link may not carry. + if len(speaks) != 12 { + t.Errorf("the vocabulary is %d shapes rather than 12; every addition widens what a compromised "+ "control plane can express, so a change here is a decision: %s", len(speaks), vocabulary()) } diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go new file mode 100644 index 0000000..b896c5d --- /dev/null +++ b/internal/firewall/firewall.go @@ -0,0 +1,431 @@ +// Package firewall speaks the firewall found on an adopted node, in that firewall's own terms +// (novox/hq ADR 0100). +// +// **The found firewall stays in force.** On an adopted node the mesh loads nothing that drops by +// default or holds an accept; what it needs reachable it converges as openings through what it +// found, marks each rule as its own, and removes only what it marked. It never resets or flushes: +// the rules the machine already had are the operator's, and they are what keeps it serving. +package firewall + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/system" +) + +// Runner executes a command. +type Runner = system.Runner + +// Kind is what firewall a machine has, as far as the mesh is concerned. +type Kind string + +const ( + // UFW is an active ufw — the one kind found on the machines measured, and the one spoken. + UFW Kind = "ufw" + // None is a machine where nothing refuses anything, which needs no openings. + None Kind = "none" + // Unsupported is a firewall no host speaks yet. A machine with one is refused adoption: the + // mesh could neither open what it needs nor know what it would be closing. + Unsupported Kind = "unsupported" +) + +// MeshInterface is the private network's interface, the way an opening from the mesh is known. +// It must be the controller's overlay interface name. +const MeshInterface = "mesh0" + +// Detect says which firewall this machine has. For Unsupported the string names it. +func Detect(ctx context.Context, run Runner) (Kind, string, error) { + if out, err := run(ctx, "firewall-cmd", "--state"); err == nil && strings.TrimSpace(out) == "running" { + return Unsupported, "firewalld", nil + } + ufwActive := false + if out, err := run(ctx, "ufw", "status"); err == nil { + ufwActive = statusActive(out) + } + + out, err := run(ctx, "nft", "list", "ruleset") + switch { + case err == nil: + if refusing := Refusing(out, ufwActive); len(refusing) > 0 { + return Unsupported, "nftables rules that refuse traffic, in " + strings.Join(refusing, ", "), nil + } + case !missing(err): + return "", "", fmt.Errorf("cannot read this machine's packet filter to know what it has: %w", err) + } + + if !ufwActive { + // iptables with the legacy backend is invisible to nft. + for _, legacy := range []string{"iptables-legacy", "ip6tables-legacy"} { + out, err := run(ctx, legacy, "-S") + if err != nil { + continue + } + if refusing := RefusingLegacy(out); len(refusing) > 0 { + return Unsupported, legacy + " rules that refuse traffic, in " + strings.Join(refusing, ", "), nil + } + } + } + + if ufwActive { + return UFW, "ufw", nil + } + return None, "", nil +} + +func missing(err error) bool { + return errors.Is(err, exec.ErrNotFound) +} + +func statusActive(out string) bool { + for _, line := range strings.Split(out, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "Status:") { + return strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "Status:")) == "active" + } + } + return false +} + +// Refusing names the tables of an `nft list ruleset` holding something that refuses traffic — a +// drop or reject, or a base chain whose policy drops — and that is neither the mesh's own nor the +// container runtime's. With ufw active, the tables iptables-nft manages are ufw's and the runtime's +// and are not counted. +func Refusing(ruleset string, ufwActive bool) []string { + var refusing []string + managed := map[string]bool{} + var table, chain string + counted := map[string]bool{} + note := func() { + if !counted[table] { + counted[table] = true + refusing = append(refusing, "table "+table) + } + } + for _, raw := range strings.Split(ruleset, "\n") { + line := strings.TrimSpace(raw) + switch { + case strings.HasPrefix(line, "# Warning: table ") && strings.Contains(line, "managed by iptables-nft"): + name := strings.TrimPrefix(line, "# Warning: table ") + name, _, _ = strings.Cut(name, " is managed") + managed[name] = true + continue + case strings.HasPrefix(line, "table "): + table = strings.TrimSuffix(strings.TrimSpace(strings.TrimPrefix(line, "table ")), "{") + table = strings.TrimSpace(table) + chain = "" + continue + case strings.HasPrefix(line, "chain "): + chain = strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(line, "chain "), "{")) + continue + case line == "" || line == "}" || strings.HasPrefix(line, "#") || chain == "": + continue + } + if table == "inet mesh" || table == "inet mesh_guard" { + continue + } + iptables := managed[table] || iptablesTable(table) + if iptables && ufwActive { + continue + } + if strings.HasPrefix(line, "type ") { + if strings.Contains(line, "policy drop") && !(iptables && runtimes(table, chain, line)) { + note() + } + continue + } + if !verdictRefuses(line) { + continue + } + if iptables && runtimes(table, chain, line) { + continue + } + note() + } + return refusing +} + +// iptablesTable is whether a table is one iptables-nft writes. Named rather than read from the +// warning nft prints above it, because nft does not print that for every such table: a captured +// ruleset carried it on ip filter and not on ip raw, where the runtime keeps its drops. +func iptablesTable(table string) bool { + family, name, _ := strings.Cut(table, " ") + if family != "ip" && family != "ip6" { + return false + } + switch name { + case "filter", "nat", "raw", "mangle", "security": + return true + } + return false +} + +// runtimes is whether a refusal in an iptables-nft table is the container runtime's own: in its +// DOCKER chains, its forward policy, or its guard against reaching a container's address directly +// from outside its bridge, in the raw table. +func runtimes(table, chain, line string) bool { + _, name, _ := strings.Cut(table, " ") + switch { + case strings.HasPrefix(chain, "DOCKER"): + return true + case name == "filter" && chain == "FORWARD" && strings.HasPrefix(line, "type "): + return true + case name == "raw" && chain == "PREROUTING": + return strings.Contains(line, "daddr") && strings.Contains(line, "iifname !=") + } + return false +} + +var verdict = regexp.MustCompile(`(^|\s)(drop|reject)(\s|$)`) + +func verdictRefuses(line string) bool { + return verdict.MatchString(line) +} + +// RefusingLegacy names the chains of an `iptables-legacy -S` that refuse traffic outside the +// container runtime's own. +func RefusingLegacy(rules string) []string { + var refusing []string + seen := map[string]bool{} + for _, line := range strings.Split(rules, "\n") { + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + chain := fields[1] + refuses := false + switch fields[0] { + case "-P": + refuses = fields[2] == "DROP" && chain != "FORWARD" + case "-A": + for i, f := range fields { + if f == "-j" && i+1 < len(fields) && (fields[i+1] == "DROP" || fields[i+1] == "REJECT") { + refuses = !strings.HasPrefix(chain, "DOCKER") + } + } + } + if refuses && !seen[chain] { + seen[chain] = true + refusing = append(refusing, "chain "+chain) + } + } + return refusing +} + +// --- ufw --------------------------------------------------------------------------------------- + +// Mark is the comment every rule the mesh adds carries: whose it is, which opening, and a digest +// of the rule itself, so a rule the opening no longer describes is recognised as stale without the +// host having to know how ufw prints a rule back. +func Mark(o *declaration.Opening) string { + sum := sha256.Sum256([]byte(strings.Join(Rule(o), " "))) + return marker(o.ID) + " " + hex.EncodeToString(sum[:])[:8] +} + +func marker(id string) string { return "mesh-host " + id } + +// markedFor is whether a comment is the mesh's, for this opening. +func markedFor(comment, id string) bool { + return comment == marker(id) || strings.HasPrefix(comment, marker(id)+" ") +} + +// Rule is the ufw rule an opening becomes, without its comment. +// +// incoming from everywhere allow proto tcp to any port P +// incoming from the mesh allow in on mesh0 proto tcp to any port P +// forwarded route allow [in on mesh0] proto tcp to any port +// +// A forwarded opening names the container's port because ufw's route rules are matched after the +// runtime's destination translation. +func Rule(o *declaration.Opening) []string { + var rule []string + port := o.Port + if o.Path == declaration.PathForwarded { + rule = append(rule, "route") + port = o.To + } + rule = append(rule, "allow") + if o.From == declaration.FromMesh { + rule = append(rule, "in", "on", MeshInterface) + } + return append(rule, "proto", o.Protocol, "to", "any", "port", strconv.Itoa(port)) +} + +var commentOf = regexp.MustCompile(`comment '([^']*)'`) + +// added is every rule `ufw show added` lists, each without its leading "ufw". +func added(ctx context.Context, run Runner) ([]string, error) { + out, err := run(ctx, "ufw", "show", "added") + if err != nil { + return nil, fmt.Errorf("reading ufw's rules: %w", err) + } + var rules []string + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "ufw ") { + rules = append(rules, strings.TrimPrefix(line, "ufw ")) + } + } + return rules, nil +} + +func comment(rule string) string { + m := commentOf.FindStringSubmatch(rule) + if m == nil { + return "" + } + return m[1] +} + +// words splits a rule as ufw printed it into arguments, keeping a quoted comment whole. +func words(rule string) []string { + var out []string + var cur strings.Builder + quoted, any := false, false + for _, r := range rule { + switch { + case r == '\'': + quoted = !quoted + any = true + case r == ' ' && !quoted: + if any { + out = append(out, cur.String()) + cur.Reset() + any = false + } + default: + cur.WriteRune(r) + any = true + } + } + if any { + out = append(out, cur.String()) + } + return out +} + +// Converge makes one opening true in ufw: its marked rule present, and any rule marked for it that +// no longer describes it deleted. Nothing unmarked is touched. The outcome is created, updated or +// unchanged, read back from ufw rather than assumed. +func Converge(ctx context.Context, run Runner, o *declaration.Opening) (string, error) { + rules, err := added(ctx, run) + if err != nil { + return "", err + } + mark := Mark(o) + present := false + var stale []string + for _, rule := range rules { + c := comment(rule) + switch { + case c == mark: + present = true + case markedFor(c, o.ID): + stale = append(stale, rule) + } + } + if present && len(stale) == 0 { + return "unchanged", nil + } + for _, rule := range stale { + if _, err := run(ctx, "ufw", append([]string{"delete"}, words(rule)...)...); err != nil { + return "", fmt.Errorf("deleting the mesh's stale ufw rule %q: %w", rule, err) + } + } + if !present { + args := append(Rule(o), "comment", mark) + if _, err := run(ctx, "ufw", args...); err != nil { + return "", fmt.Errorf("adding the ufw rule for %s: %w", o.Target(), err) + } + } + after, err := added(ctx, run) + if err != nil { + return "", err + } + found, leftover := false, 0 + for _, rule := range after { + c := comment(rule) + if c == mark { + found = true + } else if markedFor(c, o.ID) { + leftover++ + } + } + if !found { + return "", fmt.Errorf("ufw was asked for %s and does not list it afterwards", o.Target()) + } + if leftover > 0 { + return "", fmt.Errorf("ufw still lists %d stale rule(s) marked for %s after deleting them", leftover, o.ID) + } + if len(stale) > 0 { + return "updated", nil + } + return "created", nil +} + +// Remove deletes the rules marked for one opening, and nothing else. +func Remove(ctx context.Context, run Runner, id string) (int, error) { + rules, err := added(ctx, run) + if err != nil { + return 0, err + } + removed := 0 + for _, rule := range rules { + if !markedFor(comment(rule), id) { + continue + } + if _, err := run(ctx, "ufw", append([]string{"delete"}, words(rule)...)...); err != nil { + return removed, fmt.Errorf("deleting the mesh's ufw rule %q: %w", rule, err) + } + removed++ + } + after, err := added(ctx, run) + if err != nil { + return removed, err + } + for _, rule := range after { + if markedFor(comment(rule), id) { + return removed, fmt.Errorf("ufw still lists a rule marked for %s after deleting it", id) + } + } + return removed, nil +} + +// Enable turns ufw back on, as found, and reads back that it is. +func Enable(ctx context.Context, run Runner) error { + if _, err := run(ctx, "ufw", "--force", "enable"); err != nil { + return fmt.Errorf("enabling ufw again: %w", err) + } + return expectActive(ctx, run, true) +} + +// Disable retires ufw without flushing it: its configuration stays on disk, and the container +// runtime's rules are not its to remove. +func Disable(ctx context.Context, run Runner) error { + if _, err := run(ctx, "ufw", "disable"); err != nil { + return fmt.Errorf("disabling ufw: %w", err) + } + return expectActive(ctx, run, false) +} + +func expectActive(ctx context.Context, run Runner, want bool) error { + out, err := run(ctx, "ufw", "status") + if err != nil { + return fmt.Errorf("reading ufw's status back: %w", err) + } + if statusActive(out) != want { + state := "inactive" + if want { + state = "active" + } + return fmt.Errorf("ufw was asked to be %s and says: %s", state, strings.TrimSpace(out)) + } + return nil +} diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go new file mode 100644 index 0000000..67d8aa2 --- /dev/null +++ b/internal/firewall/firewall_test.go @@ -0,0 +1,335 @@ +package firewall + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force, the mesh opens +// what it needs through it in its own terms, and removes only what it marked. + +func dockerOnly(t *testing.T) string { + t.Helper() + // Captured from a real machine running the container runtime and nothing else that filters: + // its nat, filter and raw tables as iptables-nft writes them. + raw, err := os.ReadFile("testdata/docker-only.nft") + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +const aDroppingTable = ` +table inet filter { + chain input { + type filter hook input priority filter; policy drop; + ct state established,related accept + tcp dport 22 accept + } +} +` + +const ufwChains = ` +# Warning: table ip filter is managed by iptables-nft, do not touch! +table ip filter { + chain INPUT { + type filter hook input priority filter; policy drop; + counter packets 0 bytes 0 jump ufw-before-input + } + chain ufw-user-input { + tcp dport 22 counter packets 0 bytes 0 accept + } + chain ufw-reject-input { + counter packets 0 bytes 0 reject + } +} +` + +const theMeshsOwn = ` +table inet mesh { + chain input { + type filter hook input priority filter; policy drop; + iif lo accept + } +} +table inet mesh_guard { + chain prerouting { + type filter hook prerouting priority raw; policy accept; + iifname != "lo" tcp dport { 5432, 15672 } drop + } +} +` + +func TestTheContainerRuntimesOwnRulesAreNotAFirewall(t *testing.T) { + if got := Refusing(dockerOnly(t), false); len(got) != 0 { + t.Errorf("the runtime's own rules read as a firewall: %v", got) + } +} + +func TestTheMeshsOwnTablesAreNotAFirewall(t *testing.T) { + if got := Refusing(dockerOnly(t)+theMeshsOwn, false); len(got) != 0 { + t.Errorf("the mesh's own tables read as a found firewall: %v", got) + } +} + +func TestATableThatDropsIsAFirewall(t *testing.T) { + got := Refusing(dockerOnly(t)+aDroppingTable, false) + if len(got) != 1 || got[0] != "table inet filter" { + t.Errorf("a dropping table was not named: %v", got) + } +} + +func TestUfwsOwnChainsAreUfwsWhenItIsActive(t *testing.T) { + if got := Refusing(dockerOnly(t)+ufwChains, true); len(got) != 0 { + t.Errorf("ufw's own chains read as a second firewall: %v", got) + } + if got := Refusing(dockerOnly(t)+ufwChains, false); len(got) == 0 { + t.Error("iptables rules that refuse, with ufw not active, were not counted") + } +} + +func TestLegacyIptablesThatRefusesIsAFirewall(t *testing.T) { + docker := "-P INPUT ACCEPT\n-P FORWARD DROP\n-P OUTPUT ACCEPT\n-N DOCKER\n-A DOCKER -i docker0 -j DROP\n" + if got := RefusingLegacy(docker); len(got) != 0 { + t.Errorf("the runtime's legacy rules read as a firewall: %v", got) + } + if got := RefusingLegacy(docker + "-A INPUT -p tcp --dport 25 -j REJECT\n"); len(got) != 1 { + t.Errorf("a legacy reject was not counted: %v", got) + } +} + +// fakeUFW is ufw as far as the host can see it: a status, and user rules it prints back in its +// own canonical form — deliberately not the order the host wrote them in. +type fakeUFW struct { + active bool + installed bool + rules []string + ruleset string + firewalld bool + asked []string +} + +func canonical(args []string) string { + var route, in, port, proto, comment string + for i := 0; i < len(args); i++ { + switch args[i] { + case "route": + route = "route " + case "in": + in = "in on " + args[i+2] + " " + i += 2 + case "port": + port = args[i+1] + i++ + case "proto": + proto = args[i+1] + i++ + case "comment": + comment = args[i+1] + i++ + } + } + line := route + "allow " + in + port + "/" + proto + if comment != "" { + line += " comment '" + comment + "'" + } + return line +} + +func (f *fakeUFW) run(_ context.Context, name string, args ...string) (string, error) { + f.asked = append(f.asked, name+" "+strings.Join(args, " ")) + switch name { + case "firewall-cmd": + if f.firewalld { + return "running\n", nil + } + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + case "nft": + return f.ruleset, nil + case "iptables-legacy", "ip6tables-legacy": + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + case "ufw": + default: + return "", fmt.Errorf("unexpected %s", name) + } + if !f.installed { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + switch { + case args[0] == "status": + if f.active { + return "Status: active\n\nTo Action From\n", nil + } + return "Status: inactive\n", nil + case args[0] == "show": + out := "Added user rules (see 'ufw status' for running firewall):\n" + for _, r := range f.rules { + out += "ufw " + r + "\n" + } + return out, nil + case args[0] == "--force" && args[1] == "enable": + f.active = true + return "Firewall is active and enabled on system startup\n", nil + case args[0] == "disable": + f.active = false + return "Firewall stopped and disabled on system startup\n", nil + case args[0] == "delete": + for i, r := range f.rules { + if strings.Join(words(r), "\x00") == strings.Join(args[1:], "\x00") { + f.rules = append(f.rules[:i], f.rules[i+1:]...) + return "Rule deleted\n", nil + } + } + return "", errors.New("Could not delete non-existent rule") + default: + f.rules = append(f.rules, canonical(args)) + return "Rule added\n", nil + } +} + +func (f *fakeUFW) added() int { + n := 0 + for _, a := range f.asked { + if strings.HasPrefix(a, "ufw allow") || strings.HasPrefix(a, "ufw route") { + n++ + } + } + return n +} + +func opening(id string, port int, from, path string, to int) *declaration.Opening { + return &declaration.Opening{ID: id, Type: declaration.TypeOpening, Port: port, Protocol: "tcp", + From: from, Path: path, To: to} +} + +func TestAnOpeningBecomesTheUfwRuleForItsPathAndOrigin(t *testing.T) { + for _, c := range []struct { + o *declaration.Opening + want string + }{ + {opening("adoption.a", 5671, "everywhere", "incoming", 0), "allow proto tcp to any port 5671"}, + {opening("adoption.b", 5432, "mesh", "incoming", 0), "allow in on mesh0 proto tcp to any port 5432"}, + {opening("adoption.c", 20001, "everywhere", "forwarded", 8080), "route allow proto tcp to any port 8080"}, + {opening("adoption.d", 20001, "mesh", "forwarded", 8080), "route allow in on mesh0 proto tcp to any port 8080"}, + } { + if got := strings.Join(Rule(c.o), " "); got != c.want { + t.Errorf("%s: %q, want %q", c.o.ID, got, c.want) + } + } +} + +func TestAnOpeningIsAddedOnceAndMarkedAsTheMeshs(t *testing.T) { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp"}} + o := opening("adoption.opening-tcp-5671-incoming", 5671, "everywhere", "incoming", 0) + + action, err := Converge(context.Background(), f.run, o) + if err != nil || action != "created" { + t.Fatalf("first converge: %q %v", action, err) + } + if !strings.Contains(f.rules[1], "comment 'mesh-host adoption.opening-tcp-5671-incoming ") { + t.Errorf("the rule is not marked as the mesh's: %v", f.rules) + } + action, err = Converge(context.Background(), f.run, o) + if err != nil || action != "unchanged" { + t.Fatalf("second converge: %q %v", action, err) + } + if f.added() != 1 { + t.Errorf("re-converging added again: %v", f.asked) + } +} + +func TestAnOpeningLostToAReloadIsAddedAgain(t *testing.T) { + f := &fakeUFW{installed: true, active: true} + o := opening("adoption.x", 5671, "everywhere", "incoming", 0) + if _, err := Converge(context.Background(), f.run, o); err != nil { + t.Fatal(err) + } + f.rules = nil // what a reload that lost the rule leaves + action, err := Converge(context.Background(), f.run, o) + if err != nil || action != "created" || len(f.rules) != 1 { + t.Fatalf("a lost opening was not put back: %q %v %v", action, err, f.rules) + } +} + +func TestAChangedOpeningReplacesOnlyItsOwnRule(t *testing.T) { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 8080/tcp comment 'someone else'"}} + if _, err := Converge(context.Background(), f.run, opening("adoption.x", 5671, "everywhere", "incoming", 0)); err != nil { + t.Fatal(err) + } + action, err := Converge(context.Background(), f.run, opening("adoption.x", 5671, "mesh", "incoming", 0)) + if err != nil || action != "updated" { + t.Fatalf("%q %v", action, err) + } + if len(f.rules) != 3 || f.rules[0] != "allow 22/tcp" || f.rules[1] != "allow 8080/tcp comment 'someone else'" || + !strings.Contains(f.rules[2], "in on mesh0") { + t.Errorf("rules afterwards: %v", f.rules) + } +} + +func TestRemovingAnOpeningRemovesOnlyWhatWasMarkedForIt(t *testing.T) { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 8080/tcp"}} + for _, o := range []*declaration.Opening{ + opening("adoption.a", 5671, "everywhere", "incoming", 0), + opening("adoption.ab", 5000, "everywhere", "incoming", 0), + } { + if _, err := Converge(context.Background(), f.run, o); err != nil { + t.Fatal(err) + } + } + n, err := Remove(context.Background(), f.run, "adoption.a") + if err != nil || n != 1 { + t.Fatalf("removed %d: %v", n, err) + } + if len(f.rules) != 3 || f.rules[0] != "allow 22/tcp" || f.rules[1] != "allow 8080/tcp" || + !strings.Contains(f.rules[2], "adoption.ab") { + t.Errorf("more than the marked rule went: %v", f.rules) + } +} + +func TestEnableAndDisableReadBack(t *testing.T) { + f := &fakeUFW{installed: true, active: true} + if err := Disable(context.Background(), f.run); err != nil || f.active { + t.Fatalf("disable: %v", err) + } + if err := Enable(context.Background(), f.run); err != nil || !f.active { + t.Fatalf("enable: %v", err) + } + for _, a := range f.asked { + if strings.Contains(a, "reset") || strings.Contains(a, "flush") { + t.Errorf("the found firewall was reset: %s", a) + } + } +} + +func TestDetectingTheFoundFirewall(t *testing.T) { + for _, c := range []struct { + name string + f *fakeUFW + want Kind + }{ + {"nothing but the runtime", &fakeUFW{ruleset: dockerOnly(t)}, None}, + {"ufw active", &fakeUFW{installed: true, active: true, ruleset: dockerOnly(t) + ufwChains}, UFW}, + {"ufw installed and inactive", &fakeUFW{installed: true, ruleset: dockerOnly(t)}, None}, + {"firewalld", &fakeUFW{firewalld: true, ruleset: dockerOnly(t)}, Unsupported}, + {"an nftables table of its own", &fakeUFW{ruleset: dockerOnly(t) + aDroppingTable}, Unsupported}, + {"ufw beside an nftables table", &fakeUFW{installed: true, active: true, ruleset: dockerOnly(t) + ufwChains + aDroppingTable}, Unsupported}, + } { + got, name, err := Detect(context.Background(), c.f.run) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + if got != c.want { + t.Errorf("%s: detected %s (%s), want %s", c.name, got, name, c.want) + } + if got == Unsupported && name == "" { + t.Errorf("%s: an unsupported firewall was not named", c.name) + } + } +} diff --git a/internal/firewall/testdata/docker-only.nft b/internal/firewall/testdata/docker-only.nft new file mode 100644 index 0000000..7df77ba --- /dev/null +++ b/internal/firewall/testdata/docker-only.nft @@ -0,0 +1,297 @@ +# Warning: table ip nat is managed by iptables-nft, do not touch! +table ip nat { + chain DOCKER { + iifname != "br-c70303d221ee" tcp dport 5680 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-c70303d221ee" tcp dport 15673 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 59000 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 59001 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 55672 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 55673 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 55432 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "docker0" tcp dport 57732 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "docker0" tcp dport 57733 counter packets 0 bytes 0 xt target "DNAT" + iifname != "docker0" tcp dport 5314 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-613eb68ef5fb" tcp dport 4848 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-b3240c822bce" tcp dport 5679 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-b3240c822bce" tcp dport 15672 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-4b504efd6080" tcp dport 9000 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-4b504efd6080" tcp dport 9001 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-ef6df03f71a0" tcp dport 5432 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-ef6df03f71a0" tcp dport 8081 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-ec480f77ac34" tcp dport 6379 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-af4c9c2aa60a" tcp dport 8001 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-669fda75f1ac" tcp dport 28080 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-af4c9c2aa60a" tcp dport 6789 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-af4c9c2aa60a" tcp dport 8770 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-af4c9c2aa60a" tcp dport 1212 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-af4c9c2aa60a" tcp dport 80 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-af4c9c2aa60a" tcp dport 443 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "docker0" tcp dport 55541 counter packets 0 bytes 0 xt target "DNAT" + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 437947 bytes 71410534 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 5761 bytes 423317 jump DOCKER + } + + chain POSTROUTING { + type nat hook postrouting priority srcnat; policy accept; + ip saddr 172.22.0.0/16 oifname != "br-65f6dc782562" counter packets 124 bytes 16328 xt target "MASQUERADE" + ip saddr 172.28.0.0/16 oifname != "br-669fda75f1ac" counter packets 127 bytes 16928 xt target "MASQUERADE" + ip saddr 172.31.0.0/16 oifname != "br-ec480f77ac34" counter packets 127 bytes 16928 xt target "MASQUERADE" + ip saddr 192.168.48.0/20 oifname != "br-ef6df03f71a0" counter packets 127 bytes 16928 xt target "MASQUERADE" + ip saddr 172.25.0.0/16 oifname != "br-4b504efd6080" counter packets 129 bytes 17052 xt target "MASQUERADE" + ip saddr 192.168.32.0/20 oifname != "br-613eb68ef5fb" counter packets 1124 bytes 76748 xt target "MASQUERADE" + ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 1833 bytes 150990 xt target "MASQUERADE" + ip saddr 172.18.0.0/16 oifname != "br-07a5e2f2c42f" counter packets 504 bytes 65112 xt target "MASQUERADE" + ip saddr 172.27.0.0/16 oifname != "br-160f55da427c" counter packets 508 bytes 65784 xt target "MASQUERADE" + ip saddr 192.168.16.0/20 oifname != "br-dba077b9b543" counter packets 503 bytes 64784 xt target "MASQUERADE" + ip saddr 192.168.64.0/20 oifname != "br-d44fef8fd602" counter packets 1282 bytes 111308 xt target "MASQUERADE" + ip saddr 172.30.0.0/16 oifname != "br-af4c9c2aa60a" counter packets 2503 bytes 190324 xt target "MASQUERADE" + ip saddr 172.21.0.0/16 oifname != "br-9d6c95e8d80c" counter packets 1289 bytes 112644 xt target "MASQUERADE" + ip saddr 172.23.0.0/16 oifname != "br-679db9b21e00" counter packets 506 bytes 65384 xt target "MASQUERADE" + ip saddr 172.24.0.0/16 oifname != "br-40094534a5ee" counter packets 508 bytes 65784 xt target "MASQUERADE" + ip saddr 172.26.0.0/16 oifname != "br-3dcb6ef83ea1" counter packets 508 bytes 65784 xt target "MASQUERADE" + ip saddr 172.20.0.0/16 oifname != "br-3a760a74f4f6" counter packets 1153 bytes 104344 xt target "MASQUERADE" + ip saddr 192.168.80.0/20 oifname != "br-3636e05760a9" counter packets 546 bytes 70540 xt target "MASQUERADE" + ip saddr 172.29.0.0/16 oifname != "br-b3240c822bce" counter packets 551 bytes 71492 xt target "MASQUERADE" + ip saddr 172.19.0.0/16 oifname != "br-c70303d221ee" counter packets 1136 bytes 106592 xt target "MASQUERADE" + } +} +# Warning: table ip filter is managed by iptables-nft, do not touch! +table ip filter { + chain DOCKER { + ip daddr 172.17.0.5 iifname != "docker0" oifname "docker0" tcp dport 5432 counter packets 0 bytes 0 accept + ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 443 counter packets 0 bytes 0 accept + ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 80 counter packets 0 bytes 0 accept + ip daddr 172.30.0.10 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 3000 counter packets 0 bytes 0 accept + ip daddr 172.30.0.5 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 8000 counter packets 0 bytes 0 accept + ip daddr 172.30.0.3 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 6789 counter packets 0 bytes 0 accept + ip daddr 172.28.0.3 iifname != "br-669fda75f1ac" oifname "br-669fda75f1ac" tcp dport 8080 counter packets 0 bytes 0 accept + ip daddr 172.30.0.4 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 5540 counter packets 0 bytes 0 accept + ip daddr 172.31.0.2 iifname != "br-ec480f77ac34" oifname "br-ec480f77ac34" tcp dport 6379 counter packets 0 bytes 0 accept + ip daddr 192.168.48.3 iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" tcp dport 8081 counter packets 0 bytes 0 accept + ip daddr 192.168.48.2 iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" tcp dport 5432 counter packets 0 bytes 0 accept + ip daddr 172.25.0.2 iifname != "br-4b504efd6080" oifname "br-4b504efd6080" tcp dport 9001 counter packets 0 bytes 0 accept + ip daddr 172.25.0.2 iifname != "br-4b504efd6080" oifname "br-4b504efd6080" tcp dport 9000 counter packets 0 bytes 0 accept + ip daddr 172.29.0.2 iifname != "br-b3240c822bce" oifname "br-b3240c822bce" tcp dport 15672 counter packets 0 bytes 0 accept + ip daddr 172.29.0.2 iifname != "br-b3240c822bce" oifname "br-b3240c822bce" tcp dport 5672 counter packets 0 bytes 0 accept + ip daddr 192.168.32.2 iifname != "br-613eb68ef5fb" oifname "br-613eb68ef5fb" tcp dport 1433 counter packets 0 bytes 0 accept + ip daddr 172.17.0.3 iifname != "docker0" oifname "docker0" tcp dport 5000 counter packets 0 bytes 0 accept + ip daddr 172.17.0.2 iifname != "docker0" oifname "docker0" tcp dport 15672 counter packets 0 bytes 0 accept + ip daddr 172.17.0.2 iifname != "docker0" oifname "docker0" tcp dport 5672 counter packets 0 bytes 0 accept + ip daddr 192.168.80.4 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 5432 counter packets 0 bytes 0 accept + ip daddr 192.168.80.3 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 15672 counter packets 0 bytes 0 accept + ip daddr 192.168.80.3 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 5672 counter packets 0 bytes 0 accept + ip daddr 192.168.80.2 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 9001 counter packets 0 bytes 0 accept + ip daddr 192.168.80.2 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 9000 counter packets 0 bytes 0 accept + ip daddr 172.19.0.2 iifname != "br-c70303d221ee" oifname "br-c70303d221ee" tcp dport 15672 counter packets 0 bytes 0 accept + ip daddr 172.19.0.2 iifname != "br-c70303d221ee" oifname "br-c70303d221ee" tcp dport 5672 counter packets 0 bytes 0 accept + iifname != "br-c70303d221ee" oifname "br-c70303d221ee" counter packets 0 bytes 0 drop + iifname != "br-b3240c822bce" oifname "br-b3240c822bce" counter packets 0 bytes 0 drop + iifname != "br-3636e05760a9" oifname "br-3636e05760a9" counter packets 0 bytes 0 drop + iifname != "br-3a760a74f4f6" oifname "br-3a760a74f4f6" counter packets 0 bytes 0 drop + iifname != "br-3dcb6ef83ea1" oifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 drop + iifname != "br-40094534a5ee" oifname "br-40094534a5ee" counter packets 0 bytes 0 drop + iifname != "br-679db9b21e00" oifname "br-679db9b21e00" counter packets 0 bytes 0 drop + iifname != "br-9d6c95e8d80c" oifname "br-9d6c95e8d80c" counter packets 0 bytes 0 drop + iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + iifname != "br-d44fef8fd602" oifname "br-d44fef8fd602" counter packets 0 bytes 0 drop + iifname != "br-dba077b9b543" oifname "br-dba077b9b543" counter packets 0 bytes 0 drop + iifname != "br-160f55da427c" oifname "br-160f55da427c" counter packets 0 bytes 0 drop + iifname != "br-07a5e2f2c42f" oifname "br-07a5e2f2c42f" counter packets 0 bytes 0 drop + iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop + iifname != "br-613eb68ef5fb" oifname "br-613eb68ef5fb" counter packets 0 bytes 0 drop + iifname != "br-4b504efd6080" oifname "br-4b504efd6080" counter packets 0 bytes 0 drop + iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" counter packets 0 bytes 0 drop + iifname != "br-ec480f77ac34" oifname "br-ec480f77ac34" counter packets 0 bytes 0 drop + iifname != "br-669fda75f1ac" oifname "br-669fda75f1ac" counter packets 0 bytes 0 drop + iifname != "br-65f6dc782562" oifname "br-65f6dc782562" counter packets 0 bytes 0 drop + } + + chain DOCKER-FORWARD { + counter packets 6530319 bytes 11196484299 jump DOCKER-CT + counter packets 3312487 bytes 5001091084 jump DOCKER-INTERNAL + counter packets 3312487 bytes 5001091084 jump DOCKER-BRIDGE + iifname "br-c70303d221ee" counter packets 0 bytes 0 accept + iifname "br-b3240c822bce" counter packets 0 bytes 0 accept + iifname "br-3636e05760a9" counter packets 43 bytes 9355 accept + iifname "br-3a760a74f4f6" counter packets 0 bytes 0 accept + iifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 accept + iifname "br-40094534a5ee" counter packets 0 bytes 0 accept + iifname "br-679db9b21e00" counter packets 0 bytes 0 accept + iifname "br-9d6c95e8d80c" counter packets 0 bytes 0 accept + iifname "br-af4c9c2aa60a" counter packets 2761311 bytes 4959915711 accept + iifname "br-d44fef8fd602" counter packets 0 bytes 0 accept + iifname "br-dba077b9b543" counter packets 0 bytes 0 accept + iifname "br-160f55da427c" counter packets 0 bytes 0 accept + iifname "br-07a5e2f2c42f" counter packets 0 bytes 0 accept + iifname "docker0" counter packets 460394 bytes 25754554 accept + iifname "br-613eb68ef5fb" counter packets 10805 bytes 1792862 accept + iifname "br-4b504efd6080" counter packets 33 bytes 2892 accept + iifname "br-ef6df03f71a0" counter packets 0 bytes 0 accept + iifname "br-ec480f77ac34" counter packets 0 bytes 0 accept + iifname "br-669fda75f1ac" counter packets 0 bytes 0 accept + iifname "br-65f6dc782562" counter packets 0 bytes 0 accept + } + + chain DOCKER-BRIDGE { + oifname "br-c70303d221ee" counter packets 0 bytes 0 jump DOCKER + oifname "br-b3240c822bce" counter packets 0 bytes 0 jump DOCKER + oifname "br-3636e05760a9" counter packets 0 bytes 0 jump DOCKER + oifname "br-3a760a74f4f6" counter packets 0 bytes 0 jump DOCKER + oifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 jump DOCKER + oifname "br-40094534a5ee" counter packets 0 bytes 0 jump DOCKER + oifname "br-679db9b21e00" counter packets 0 bytes 0 jump DOCKER + oifname "br-9d6c95e8d80c" counter packets 0 bytes 0 jump DOCKER + oifname "br-af4c9c2aa60a" counter packets 118 bytes 8400 jump DOCKER + oifname "br-d44fef8fd602" counter packets 0 bytes 0 jump DOCKER + oifname "br-dba077b9b543" counter packets 0 bytes 0 jump DOCKER + oifname "br-160f55da427c" counter packets 0 bytes 0 jump DOCKER + oifname "br-07a5e2f2c42f" counter packets 0 bytes 0 jump DOCKER + oifname "docker0" counter packets 0 bytes 0 jump DOCKER + oifname "br-613eb68ef5fb" counter packets 0 bytes 0 jump DOCKER + oifname "br-4b504efd6080" counter packets 0 bytes 0 jump DOCKER + oifname "br-ef6df03f71a0" counter packets 0 bytes 0 jump DOCKER + oifname "br-ec480f77ac34" counter packets 0 bytes 0 jump DOCKER + oifname "br-669fda75f1ac" counter packets 0 bytes 0 jump DOCKER + oifname "br-65f6dc782562" counter packets 0 bytes 0 jump DOCKER + } + + chain DOCKER-CT { + oifname "br-c70303d221ee" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-b3240c822bce" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-3636e05760a9" xt match "conntrack" counter packets 35 bytes 23113 accept + oifname "br-3a760a74f4f6" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-3dcb6ef83ea1" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-40094534a5ee" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-679db9b21e00" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-9d6c95e8d80c" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-af4c9c2aa60a" xt match "conntrack" counter packets 2488066 bytes 1053784742 accept + oifname "br-d44fef8fd602" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-dba077b9b543" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-160f55da427c" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-07a5e2f2c42f" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "docker0" xt match "conntrack" counter packets 666252 bytes 5079577802 accept + oifname "br-613eb68ef5fb" xt match "conntrack" counter packets 7926 bytes 7636543 accept + oifname "br-4b504efd6080" xt match "conntrack" counter packets 29 bytes 10870 accept + oifname "br-ef6df03f71a0" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-ec480f77ac34" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-669fda75f1ac" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-65f6dc782562" xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy drop; + counter packets 33747166 bytes 176750349038 jump DOCKER-USER + counter packets 6530319 bytes 11196484299 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + oifname "mlab*" counter packets 15657582 bytes 162801189460 accept + iifname "mlab*" counter packets 10958315 bytes 687322055 accept + oifname "incusbr0" counter packets 388768 bytes 2053271282 accept + iifname "incusbr0" counter packets 212182 bytes 12081942 accept + } +} +# Warning: table ip6 nat is managed by iptables-nft, do not touch! +table ip6 nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 532 bytes 113834 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } +} +table ip6 filter { + chain DOCKER { + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + } + + chain DOCKER-BRIDGE { + } + + chain DOCKER-CT { + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy accept; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + } +} +table ip raw { + chain PREROUTING { + type filter hook prerouting priority raw; policy accept; + ip daddr 172.19.0.2 iifname != "br-c70303d221ee" counter packets 0 bytes 0 drop + ip daddr 192.168.80.2 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop + ip daddr 192.168.80.3 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 15673 counter packets 0 bytes 0 drop + ip daddr 192.168.80.4 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop + ip daddr 172.17.0.2 iifname != "docker0" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 57732 counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 57733 counter packets 0 bytes 0 drop + ip daddr 172.17.0.3 iifname != "docker0" counter packets 0 bytes 0 drop + ip daddr 172.17.0.4 iifname != "docker0" counter packets 0 bytes 0 drop + ip daddr 192.168.32.2 iifname != "br-613eb68ef5fb" counter packets 0 bytes 0 drop + ip daddr 172.30.0.7 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.29.0.2 iifname != "br-b3240c822bce" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 15672 counter packets 0 bytes 0 drop + ip daddr 172.25.0.2 iifname != "br-4b504efd6080" counter packets 0 bytes 0 drop + ip daddr 172.30.0.9 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 192.168.48.2 iifname != "br-ef6df03f71a0" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 5432 counter packets 0 bytes 0 drop + ip daddr 192.168.48.3 iifname != "br-ef6df03f71a0" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 8081 counter packets 0 bytes 0 drop + ip daddr 172.30.0.8 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.31.0.2 iifname != "br-ec480f77ac34" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 6379 counter packets 0 bytes 0 drop + ip daddr 172.30.0.4 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 8001 counter packets 0 bytes 0 drop + ip daddr 172.31.0.3 iifname != "br-ec480f77ac34" counter packets 0 bytes 0 drop + ip daddr 172.28.0.2 iifname != "br-669fda75f1ac" counter packets 0 bytes 0 drop + ip daddr 172.30.0.6 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.28.0.3 iifname != "br-669fda75f1ac" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 28080 counter packets 0 bytes 0 drop + ip daddr 172.30.0.3 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 6789 counter packets 0 bytes 0 drop + ip daddr 172.30.0.5 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.22.0.2 iifname != "br-65f6dc782562" counter packets 0 bytes 0 drop + ip daddr 172.30.0.10 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.22.0.3 iifname != "br-65f6dc782562" counter packets 0 bytes 0 drop + ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.17.0.5 iifname != "docker0" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 55541 counter packets 0 bytes 0 drop + } +} +table ip mangle { + chain FORWARD { + type filter hook forward priority mangle; policy accept; + tcp flags & (syn | rst) == syn counter packets 13760 bytes 825476 xt target "TCPMSS" + } +} diff --git a/internal/store/store.go b/internal/store/store.go index e778420..3e8a82a 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -83,6 +83,23 @@ type State struct { // nothing here is ever removed as an orphan — what is held is not the host's to remove, even // when its module is unassigned. Held []Held `json:"held,omitempty"` + + // Firewall is the firewall found on this machine when it was first adopted, and whether the + // mesh has since retired it (novox/hq ADR 0100). Nil on a node that was never adopted. + Firewall *FoundFirewall `json:"firewall,omitempty"` +} + +// FoundFirewall is what the host found filtering this machine, and what it did about it. +type FoundFirewall struct { + // Kind is ufw or none: an unsupported kind is refused adoption, never recorded. + Kind string `json:"kind"` + // WasActive is whether it was in force when found — which is what converging the node + // retires, and returning it to adopted restores. + WasActive bool `json:"was_active,omitempty"` + // DisabledByMesh is set when converging retired it, so returning to adopted enables it again + // and nothing else ever does. + DisabledByMesh bool `json:"disabled_by_mesh,omitempty"` + FoundAt time.Time `json:"found_at"` } // Held is one file or container found on an adopted node — present at a declared path or name, diff --git a/internal/system/system.go b/internal/system/system.go index 3f928e6..25ae307 100644 --- a/internal/system/system.go +++ b/internal/system/system.go @@ -178,6 +178,9 @@ func everyShape() []declaration.Type { // it: the mesh's own code runs as a process on the machine, and only software that // genuinely needs isolation asks for a container. declaration.TypeProcess, + // An opening is a rule in the firewall found on the machine, which a partial host neither + // has nor can manage (novox/hq ADR 0100). + declaration.TypeOpening, } }