Adoption mode: a node in use is adopted before it is converged (hq ADR 0100–0103) #20

Merged
jschoubben merged 52 commits from feat/adoption-mode into main 2026-09-22 19:01:47 +00:00
66 changed files with 9643 additions and 71 deletions
+80 -1
View File
@@ -18,6 +18,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
@@ -25,6 +26,7 @@ import (
"net/http"
"os"
"os/signal"
"strconv"
"syscall"
"time"
@@ -126,6 +128,19 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
--packet-filter which packet filter to run (nftables)
--extras catalogue modules beyond the floor, comma-separated
The foundation's ports are this machine's, each checked free before anything is
raised and kept as the node's setting for the module that binds it:
--store-port 5432 --bus-port 5671 --amqp-port 5672 --management-port 15672
--registry-port 5000 (follows --registry, and must agree with it)
--packages-port 3000 --hub-port 51820/udp
--overlay-range the private network's range (default 10.42.0.0/16); refused
if it overlaps an interface or route the machine already has
--adopted raise a machine in use as an adopted node: what it runs and its
firewall stay as they are, the foundation's filter is not loaded and
the mesh guards its own ports instead, and each module is taken on it
one at a time. Without it, a machine in use is refused
The installer carries a builder, not a control plane. What raises a mesh is therefore
the same thing that will maintain it, and the control plane a mesh ends up running is
one it built itself, from a repository and a commit it can name and build again.
@@ -176,7 +191,9 @@ func parseArgs(args []string) (string, bootstrap.Options, bool, error) {
HostService: defaultService,
// Longer than the host's 10s: these probes reach a container runtime that may be busy
// pulling, and a probe that times out on a working machine is a false refusal.
Timeout: 30 * time.Second,
Ports: bootstrap.DefaultPorts(),
OverlayRange: bootstrap.DefaultOverlayRange,
Timeout: 30 * time.Second,
// A socket-activated runtime queued behind the network, and a control plane running its
// first `initdb`-shaped wait, are both minutes rather than seconds.
Wait: 3 * time.Minute,
@@ -210,9 +227,50 @@ func parseArgs(args []string) (string, bootstrap.Options, bool, error) {
return "", opts, false, fmt.Errorf(
"unexpected argument %q — try `mesh-bootstrap help`", positionals[0])
}
if err := registryAgrees(set, &opts); err != nil {
return "", opts, false, err
}
return command, opts, jsonOut, nil
}
// registryAgrees makes --registry and --registry-port say one port (novox/hq ADR 0100): the
// registry is raised on the port the node gives it, and every node pulls from the address given.
// Either may be said alone and the other follows; said both ways, they must agree.
func registryAgrees(set *flag.FlagSet, opts *bootstrap.Options) error {
said := map[string]bool{}
set.Visit(func(f *flag.Flag) { said[f.Name] = true })
host, portText, err := net.SplitHostPort(opts.Registry)
var missing *net.AddrError
if errors.As(err, &missing) && missing.Err == "missing port in address" {
// A host alone, as --registry took before its port became the node's: the registry's
// port — the one given, or the catalogue's — completes it.
port := opts.Ports.Registry
if port == 0 {
port = bootstrap.DefaultPorts().Registry
}
opts.Ports.Registry = port
opts.Registry = net.JoinHostPort(strings.Trim(opts.Registry, "[]"), strconv.Itoa(port))
return nil
}
if err != nil {
return fmt.Errorf("--registry %q is not host:port: %w", opts.Registry, err)
}
port, err := strconv.Atoi(portText)
if err != nil {
return fmt.Errorf("--registry %q does not end in a port", opts.Registry)
}
switch {
case said["registry-port"] && said["registry"] && port != opts.Ports.Registry:
return fmt.Errorf("--registry %s and --registry-port %d name two ports for one registry",
opts.Registry, opts.Ports.Registry)
case said["registry-port"]:
opts.Registry = net.JoinHostPort(host, strconv.Itoa(opts.Ports.Registry))
case said["registry"]:
opts.Ports.Registry = port
}
return nil
}
func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
set := flag.NewFlagSet("mesh-bootstrap", flag.ContinueOnError)
set.SetOutput(os.Stderr)
@@ -255,6 +313,27 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
set.StringVar(&opts.SDKSource.Ref, "sdk-ref", opts.SDKSource.Ref,
"what of it to build (default main)")
set.StringVar(&opts.Site, "site", "main", "where this machine sits, for the private network")
// The foundation's ports are this node's (novox/hq ADR 0100): each is checked free before
// anything is raised, and becomes the node's setting for the module that binds it.
for _, p := range []struct {
name, what string
into *int
}{
{"store-port", "the store", &opts.Ports.Store},
{"bus-port", "the bus (amqps)", &opts.Ports.Bus},
{"amqp-port", "the broker's AMQP", &opts.Ports.AMQP},
{"management-port", "the broker's management, on loopback", &opts.Ports.Management},
{"registry-port", "the registry", &opts.Ports.Registry},
{"packages-port", "the package registry", &opts.Ports.Packages},
{"hub-port", "the private network's hub (udp)", &opts.Ports.Hub},
} {
set.IntVar(p.into, p.name, *p.into, "the machine's port for "+p.what)
}
set.BoolVar(&opts.Adopted, "adopted", false,
"raise this machine adopted: keep what it runs and its firewall until each module is taken")
set.StringVar(&opts.OverlayRange, "overlay-range", opts.OverlayRange,
"the private network's address range; must not overlap a tunnel the machine already runs")
if opts.Answers == nil {
opts.Answers = map[string]string{}
}
+40
View File
@@ -164,3 +164,43 @@ func TestTheNodeNameCanBeSaid(t *testing.T) {
t.Errorf("--catalog parsed as %q", opts.Catalogue)
}
}
// Defends novox/hq ADR 0100: the foundation's ports are inputs to genesis, and the registry's port
// and the address nodes pull from say one port.
func TestTheFoundationsPortsAreGiven(t *testing.T) {
_, opts, _, err := parseArgs([]string{"--store-port", "5433", "--hub-port", "51821", "--overlay-range", "10.77.0.0/16"})
if err != nil {
t.Fatal(err)
}
if opts.Ports.Store != 5433 || opts.Ports.Hub != 51821 || opts.Ports.Bus != 5671 || opts.OverlayRange != "10.77.0.0/16" {
t.Errorf("ports read as %+v, range %s", opts.Ports, opts.OverlayRange)
}
}
func TestTheRegistrysPortAndAddressAgree(t *testing.T) {
_, opts, _, err := parseArgs([]string{"--registry-port", "5100"})
if err != nil || opts.Registry != "127.0.0.1:5100" {
t.Errorf("--registry-port alone: %s %v", opts.Registry, err)
}
_, opts, _, err = parseArgs([]string{"--registry", "192.0.2.10:5100"})
if err != nil || opts.Ports.Registry != 5100 {
t.Errorf("--registry alone: %d %v", opts.Ports.Registry, err)
}
if _, _, _, err := parseArgs([]string{"--registry", "192.0.2.10:5000", "--registry-port", "5100"}); err == nil {
t.Error("two ports for one registry were accepted")
}
}
func TestARegistryGivenAsAHostAloneTakesTheRegistrysPort(t *testing.T) {
_, opts, _, err := parseArgs([]string{"--registry", "192.0.2.10"})
if err != nil || opts.Registry != "192.0.2.10:5000" || opts.Ports.Registry != 5000 {
t.Errorf("--registry host alone: %s %d %v", opts.Registry, opts.Ports.Registry, err)
}
_, opts, _, err = parseArgs([]string{"--registry", "192.0.2.10", "--registry-port", "5100"})
if err != nil || opts.Registry != "192.0.2.10:5100" {
t.Errorf("--registry host with --registry-port: %s %v", opts.Registry, err)
}
if _, _, _, err := parseArgs([]string{"--registry", "192.0.2.10:notaport"}); err == nil {
t.Error("a registry with a port that is not a number was accepted")
}
}
+149 -7
View File
@@ -17,8 +17,10 @@ import (
"fmt"
"os"
"os/signal"
"path/filepath"
"sort"
"strings"
"sync"
"syscall"
"text/tabwriter"
"time"
@@ -26,10 +28,12 @@ import (
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/bundle"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/firewall"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/inventory"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/profile"
"github.com/novox/mesh-host/internal/reachable"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
"github.com/novox/mesh-host/internal/upgrade"
@@ -436,6 +440,23 @@ func enrol(ctx context.Context, opts options) error {
return err
}
// An adopted node keeps the firewall it was found with (novox/hq ADR 0100), so a host that
// cannot speak that firewall must say so now — before the mesh records a node it could never
// open anything on.
if token.Adopted {
kind, name, err := firewall.Detect(ctx, apply.ExecRunner)
if err != nil {
return err
}
if kind == firewall.Unsupported {
return fmt.Errorf(
"this token joins this machine adopted, keeping the firewall found on it, and it is "+
"filtered by %s, which no host speaks yet. Nothing was enrolled", name)
}
fmt.Printf("joining adopted: what is on this machine is kept, and its firewall (%s) stays in force\n",
string(kind))
}
fmt.Printf("token for broker %s\n", token.Broker)
fmt.Printf(" pinned certificate %s\n", token.Fingerprint)
fmt.Printf(" signing key %s\n",
@@ -615,7 +636,28 @@ func runLink(ctx context.Context, opts options) error {
// new declarations; this holds the machine in the last one whether the link is up or not. A
// laptop shut for a week comes back and reconciles — it does not come back and ask what it is
// (novox/hq ADR 0004).
go holdTheMachine(ctx, opts, mine, say, sched)
// Reports a reconcile has to make unasked — what an adopted node holds changed, or its
// firewall did — go out over the link when it is up (novox/hq ADR 0100).
outbox := make(chan link.Unasked, 1)
watch := &adoptionWatch{}
applier = watch.noting(applier)
go holdTheMachine(ctx, opts, mine, say, sched, func(r link.Report) {
if !watch.differs(r) {
return
}
select {
case <-outbox:
// An older one nobody has published yet; this one says everything it did.
default:
}
// Counted as said only once the broker has taken it: queued and lost — the link down, the
// publish refused — the change would never be said again (novox/hq ADR 0100).
outbox <- link.Unasked{Report: r, Done: func(published bool) {
if published {
watch.said(r)
}
}}
})
return link.HoldRoused(ctx, link.Membership{
Node: mine.Node,
@@ -623,7 +665,64 @@ func runLink(ctx context.Context, opts options) error {
Fingerprint: mine.Membership.Fingerprint,
Password: mine.Membership.Password,
Signer: mine.Membership.Signer,
}, applier, say, opts.timeout, rousedBySignal(ctx))
}, applier, say, opts.timeout, rousedBySignal(ctx), outbox)
}
// adoptionWatch remembers what the node last said about what it holds and its firewall, so a
// reconcile speaks unasked only when that changed.
type adoptionWatch struct {
mu sync.Mutex
last string
}
// fingerprint is what a report says about adoption: each hold and whether it changed, the
// firewall, and what is reachable on the machine — which only an adopted node reports, and which
// is what the controller previews a flip from, so a port that opens or closes between deliveries
// must reach it too (novox/hq ADR 0100).
func adoptionFingerprint(r link.Report) string {
parts := []string{"firewall=" + r.Firewall}
for _, h := range r.Held {
parts = append(parts, "held "+h.ID+"="+h.Changed)
}
for _, reach := range r.Reachable {
parts = append(parts, fmt.Sprintf("reach %s %s:%d %s %v %d", reach.Protocol, reach.Address,
reach.Port, reach.By, reach.Published, reach.ContainerPort))
}
sort.Strings(parts[1:])
return strings.Join(parts, "\n")
}
// differs says whether a report says anything the last one that went out did not. It records
// nothing: what was said is what reached the mesh, not what was written down to send.
func (w *adoptionWatch) differs(r link.Report) bool {
w.mu.Lock()
defer w.mu.Unlock()
return adoptionFingerprint(r) != w.last
}
// said records a report the mesh has actually been told.
func (w *adoptionWatch) said(r link.Report) {
w.mu.Lock()
defer w.mu.Unlock()
w.last = adoptionFingerprint(r)
}
// changed is differs and said together, for a report published as it is made.
func (w *adoptionWatch) changed(r link.Report) bool {
if !w.differs(r) {
return false
}
w.said(r)
return true
}
// noting wraps the applier, so a report the link publishes after a delivery counts as said.
func (w *adoptionWatch) noting(apply link.Applier) link.Applier {
return func(ctx context.Context, raw, signature []byte) link.Report {
r := apply(ctx, raw, signature)
w.changed(r)
return r
}
}
// rousedBySignal is the machine telling this process that its link is probably stale.
@@ -671,7 +770,7 @@ func rousedBySignal(ctx context.Context) link.Roused {
const ReconcileEvery = 5 * time.Minute
func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, say link.Announce,
sched *apply.Scheduler) {
sched *apply.Scheduler, publish func(link.Report)) {
ticker := time.NewTicker(ReconcileEvery)
defer ticker.Stop()
@@ -694,6 +793,12 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
}
report := applyDeclared(ctx, opts, declared, sched)
// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
// its firewall changed, because that is how a predecessor still writing is caught
// (novox/hq ADR 0100); publish decides whether anything did.
if publish != nil && report.Refused == "" && (len(report.Held) > 0 || report.Firewall != "") {
publish(report)
}
switch {
case report.Refused != "":
say("what this node was last told no longer applies: " + report.Refused)
@@ -712,10 +817,22 @@ func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.S
return applyAndKeep(ctx, opts, raw, nil, sched)
}
// applying serialises applies on this node.
//
// **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
// acts on the machine, and writes the state back. Run at the same time they interleave, and the
// one that saves last writes a state read before the other acted — losing what the first recorded:
// a hold, the firewall found here, a resource just applied. The machine would then be one thing
// and its record another, which is the fault every read-back in this package exists to prevent.
var applying sync.Mutex
// applyAndKeep applies a declaration and, when it came from the mesh, keeps it so this node can
// go on obeying it while disconnected.
// go on obeying it while disconnected. One at a time, whoever asks.
func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared,
sched *apply.Scheduler) link.Report {
applying.Lock()
defer applying.Unlock()
declared, err := declaration.Parse(raw)
if err != nil {
return link.Report{Refused: err.Error()}
@@ -740,8 +857,8 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
// Declared, not carried. A declaration from the mesh removes only what the mesh previously
// declared — never what this machine raised for itself from its bundle (04-ISSUES/010).
outcome, updated, applyErr := apply.Apply(ctx, built, declared, known, store.OriginDeclared,
apply.ExecRunner, nil, sealOpener(opts.state))
outcome, updated, applyErr := apply.ApplyKeeping(ctx, built, declared, known, store.OriginDeclared,
apply.ExecRunner, nil, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
// Saved whichever way it went. Recording only on success would lose the footprint of a
// failed apply, and that footprint is on the machine either way.
@@ -756,11 +873,36 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
// declared is forgotten — and after a host restart the first apply rebuilds them all. A nil
// scheduler is the one-shot CLI path, which exits rather than staying up to fire anything.
if sched != nil {
sched.Sync(declared)
held := map[string]bool{}
for _, h := range updated.Held {
held[h.ID] = true
}
sched.Sync(declared, held)
}
report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw)}
// What this node found and holds, its firewall, and what is reachable on it — so an adopted
// node never reads as converged (novox/hq ADR 0100).
for _, h := range updated.Held {
report.Held = append(report.Held, link.Held{ID: h.ID, Module: h.Module, Kind: h.Kind,
Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept})
}
if declared.Adoption != nil {
if updated.Firewall != nil {
report.Firewall = updated.Firewall.Kind
}
reached, err := reachable.Collect(ctx, apply.ExecRunner)
if err != nil {
fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read what is reachable here: %v\n", err)
}
report.Reachable = reached
}
for _, change := range outcome.Outcomes {
// What is held is not what this machine owns: it was found, and is kept as it was until
// its module is taken (novox/hq ADR 0100).
if change.Action == "held" {
continue
}
report.Applied = append(report.Applied, change.ID)
}
// Kept whichever way it went, so a node that is disconnected next minute still knows what it
+128 -1
View File
@@ -1,9 +1,17 @@
package main
import (
"github.com/novox/mesh-host/internal/store"
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// Argument handling gets tests because it already failed silently once: `mesh-host inventory
@@ -135,3 +143,122 @@ func TestAFlagAfterAPositionalIsRead(t *testing.T) {
}
}
}
// Defends novox/hq ADR 0100: a reconcile on an adopted node speaks unasked only when what it holds
// or its firewall changed — which is how a predecessor still writing is caught, without a report
// every five minutes saying nothing new.
func TestAReconcileSpeaksOnlyWhenWhatIsHeldChanged(t *testing.T) {
w := &adoptionWatch{}
held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page"}, {ID: "hello-web.server"}}}
if !w.changed(held) {
t.Fatal("the first report of a hold was not said")
}
again := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.server"}, {ID: "hello-web.page"}}}
if w.changed(again) {
t.Error("the same holds in another order were said again")
}
rewritten := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}, {ID: "hello-web.server"}}}
if !w.changed(rewritten) {
t.Error("a held file rewritten by something else was not said")
}
if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) {
t.Error("a changed firewall was not said")
}
}
func TestWhatTheLinkPublishedCountsAsSaid(t *testing.T) {
w := &adoptionWatch{}
report := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "a"}}}
applier := w.noting(func(context.Context, []byte, []byte) link.Report { return report })
applier(context.Background(), nil, nil)
if w.changed(report) {
t.Error("a reconcile repeated what the link had just published")
}
}
func TestAReconcileSpeaksWhenWhatIsReachableChanged(t *testing.T) {
// The controller previews a flip from what the node last said is reachable; a port that opened
// since must reach it without waiting for the next delivery (novox/hq ADR 0100).
w := &adoptionWatch{}
before := link.Report{Firewall: "ufw", Reachable: []link.Reach{
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}}
if !w.changed(before) {
t.Fatal("the first report was not said")
}
reordered := link.Report{Firewall: "ufw", Reachable: []link.Reach{
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}}
if w.changed(reordered) {
t.Error("the same reachable set was said again")
}
opened := link.Report{Firewall: "ufw", Reachable: append(before.Reachable,
link.Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, ContainerPort: 80})}
if !w.changed(opened) {
t.Error("a newly published port was not said")
}
}
// Defends the node's own record: the link and the reconcile loop both apply, and each reads the
// state, acts, and writes it back — so they must not run at the same time, or the last save loses
// what the other recorded.
func TestOnlyOneApplyRunsAtATime(t *testing.T) {
// A host is built for one system at link time, and a test binary has no link time: this asks
// the machine it runs on, and stands aside where the answer is no.
built, err := system.For("arch")
if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil {
t.Skip("this machine is not one these tests can apply on")
}
was := builtFor
builtFor = "arch"
t.Cleanup(func() { builtFor = was })
dir := t.TempDir()
opts := options{state: filepath.Join(dir, "state.json")}
raw := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` +
filepath.Join(dir, "a.conf") + `","content":"x\n"}]}`)
// Whatever else is applying — the link, while this is the reconcile — this waits for it.
applying.Lock()
done := make(chan link.Report, 1)
go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil) }()
select {
case report := <-done:
applying.Unlock()
t.Fatalf("an apply ran while another held the node: %+v", report)
case <-time.After(50 * time.Millisecond):
}
if _, err := os.Stat(filepath.Join(dir, "a.conf")); !errors.Is(err, os.ErrNotExist) {
applying.Unlock()
t.Fatal("the waiting apply had already touched the machine")
}
applying.Unlock()
select {
case report := <-done:
if report.Refused != "" {
t.Fatalf("refused: %s", report.Refused)
}
case <-time.After(10 * time.Second):
t.Fatal("the apply never ran once the node was free")
}
known, loadErr := store.Load(opts.state)
if loadErr != nil || len(known.Resources) != 1 {
t.Errorf("the apply recorded %d resource(s): %v", len(known.Resources), loadErr)
}
}
// Defends novox/hq ADR 0100: a change is counted as said only once the mesh has been told. Queued
// and lost — the link down when the reconcile spoke — it must be said again.
func TestAChangeThatNeverReachedTheMeshIsSaidAgain(t *testing.T) {
w := &adoptionWatch{}
held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}}}
if !w.differs(held) {
t.Fatal("the first report of a change was not new")
}
// The link was down: nothing published it, so nothing says it was said.
if !w.differs(held) {
t.Error("a change that never reached the mesh was counted as said")
}
w.said(held)
if w.differs(held) {
t.Error("a change the mesh was told was said again")
}
}
+259 -10
View File
@@ -50,6 +50,8 @@ type Outcome struct {
// wrote is a digest of what this apply put there, kept so the next one can tell a machine
// that drifted from one the mesh changed its mind about. Not reported: it is bookkeeping.
wrote string
// into is what a file written into held before the mesh's keys (novox/hq ADR 0102).
into *store.Into
}
// Report is what an apply did, in the order it did it.
@@ -61,7 +63,8 @@ type Report struct {
// nothing is the ordinary steady state, and saying so is not the same as saying it failed.
func (r Report) Changed() bool {
for _, o := range r.Outcomes {
if o.Action != "unchanged" {
// Holding is keeping the machine as it was found, which is not moving it.
if o.Action != "unchanged" && o.Action != "held" {
return true
}
}
@@ -111,7 +114,9 @@ func (e *Error) Unwrap() error { return e.Err }
// Removal happens FIRST, and the order is not arbitrary. A resource that leaves a declaration
// while another arrives at the same path is an ordinary rename: removing afterwards would
// delete the file that had just been written. Removing first risks losing the old state if the
// apply then fails — a recovery concern, where the other is a correctness one.
// apply then fails — a recovery concern, where the other is a correctness one. The one exception
// is what protects an adopted node, the openings and the guard: that goes last, and only when
// everything else applied (novox/hq ADR 0103).
func Apply(
ctx context.Context,
sys system.System,
@@ -121,6 +126,23 @@ func Apply(
run Runner,
log func(string),
unseal Unseal,
) (Report, store.State, error) {
return ApplyKeeping(ctx, sys, d, known, origin, run, log, unseal, nil)
}
// ApplyKeeping is Apply on a node that may be adopted: keep is where the original of a file found
// there is recorded before anything else happens to it (novox/hq ADR 0100). Nil is a caller that
// can never be handed an adopted declaration — the carried bundle, which may not say it.
func ApplyKeeping(
ctx context.Context,
sys system.System,
d *declaration.Declaration,
known store.State,
origin string,
run Runner,
log func(string),
unseal Unseal,
keep Keep,
) (Report, store.State, error) {
if log == nil {
log = func(string) {}
@@ -132,10 +154,27 @@ func Apply(
declared[r.Identity()] = true
}
for _, orphan := range known.Orphans(declared, origin) {
action, detail, err := remove(ctx, sys, orphan, run)
// Which firewall is found here, before anything else, since an unsupported one refuses the
// whole declaration (novox/hq ADR 0100). Nothing for a converged node.
fw, err := foundFirewall(ctx, d, &known, run, log)
if err != nil {
return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report}
}
// What an adopted node's untaken modules find on the machine, looked at before anything in
// this apply — a removal included — could change it or its records (novox/hq ADR 0103).
before := lookBefore(ctx, sys, d, known, run)
removeOrphan := func(orphan store.Applied) error {
var action, detail string
var err error
if declaration.Type(orphan.Type) == declaration.TypeOpening {
action, detail, err = removeOpening(ctx, orphan, run, known.Firewall)
} else {
action, detail, err = remove(ctx, sys, orphan, run)
}
if err != nil {
return report, known, &Error{Resource: orphan.ID, Err: err, Done: report}
return &Error{Resource: orphan.ID, Err: err, Done: report}
}
known.Forget(orphan.ID)
report.Outcomes = append(report.Outcomes, Outcome{
@@ -143,6 +182,74 @@ func Apply(
Action: action, Detail: detail,
})
log(fmt.Sprintf(" %s %s (%s)", action, orphan.ID, orphan.Target))
return nil
}
// **What protects an adopted node goes last on the flip, and first on the way back** (novox/hq
// ADR 0103). The openings and the guard are what keep the mesh reachable through the found
// firewall and the store unreachable from outside.
//
// When a node is converged they leave the declaration, and removing them first would leave the
// store open from the moment the guard stops until the derived filter loads — and for ever, if
// the filter then fails. So on a converged declaration they are removed only once everything
// else applied and the found firewall is retired; if anything failed, they stay, recorded, for
// the next try.
//
// Returned to adopted, it is the mirror image: removing the derived filter first would leave
// the store open until the guard loads. So the guard's own resources are applied before any
// orphan is removed, and if a removal then fails the guard is already up. A stale opening on an
// adopted node is removed as any orphan is.
var protecting, orphans []store.Applied
for _, orphan := range known.Orphans(declared, origin) {
if d.Adoption == nil && strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) {
protecting = append(protecting, orphan)
continue
}
orphans = append(orphans, orphan)
}
ordered := d.Resources
guardFirst := 0
if d.Adoption != nil {
ordered = nil
for _, r := range d.Resources {
if strings.HasPrefix(r.Identity(), guardPrefix) {
ordered = append(ordered, r)
}
}
guardFirst = len(ordered)
for _, r := range d.Resources {
if !strings.HasPrefix(r.Identity(), guardPrefix) {
ordered = append(ordered, r)
}
}
}
orphansRemoved := false
removeOrphans := func() error {
orphansRemoved = true
for _, orphan := range orphans {
if err := removeOrphan(orphan); err != nil {
return err
}
}
return nil
}
// **A hold whose resource is no longer declared is let go, and nothing on disk is touched.**
// What was found stays as it was; only the host's note that it holds it for a module goes, so
// the node stops reporting a hold for a module no longer assigned. Should the module come back,
// what is there is present with no record and is found, and held, again — its first kept
// original is never overwritten (novox/hq ADR 0100). Only a declaration from the mesh says
// what is assigned: a carried bundle's silence is not an unassignment.
if origin == store.OriginDeclared {
for _, h := range append([]store.Held{}, known.Held...) {
if declared[h.ID] {
continue
}
known.Release(h.ID)
report.Outcomes = append(report.Outcomes, Outcome{ID: h.ID, Type: h.Kind, Target: h.Target,
Action: "forgotten", Detail: "no longer declared; left as found"})
log(fmt.Sprintf(" forgotten %s (%s): no longer declared; left as found", h.ID, h.Target))
}
}
// What moved in this apply, so a service that must reflect a file can be told the file
@@ -181,9 +288,60 @@ func Apply(
// is a different thing — one is "this machine could not do it", the other is "this was never
// a declaration", and they are fixed in different places.
var failures []*Error
for _, resource := range d.Resources {
for i, resource := range ordered {
if !orphansRemoved && i == guardFirst {
// **Only a guard that is up may let the filter go.** Removing the derived filter's
// resources stops its unit, whose stop deletes the mesh's table; if a guard resource
// failed, doing that would leave the node with neither, and the store open until some
// later reconcile gets the guard up (novox/hq ADR 0103).
if len(failures) > 0 {
first := failures[0]
first.Done = report
first.Others = len(failures) - 1
log(" kept " + guardPrefix + "*: the guard is not up, so what it replaces was left in force")
return report, known, first
}
if err := removeOrphans(); err != nil {
return report, known, err
}
}
// **On an adopted node, what is found is kept until its module is taken** (novox/hq ADR
// 0100, ADR 0103). Before anything is applied: whatever of a module not yet taken is
// present with no record of this host making it — or would reach what is — is held as it
// is and reported. Once held it stays held until its module is taken, and it is never
// recorded as applied, so it is never removed as an orphan either.
if d.Adoption != nil {
isHeld, news, outcome, err := holdOnAdopted(ctx, sys, resource, d, &known, before, run, keep,
changed, time.Now().UTC())
if err != nil {
failures = append(failures, &Error{Resource: resource.Identity(), Err: err, Done: report})
log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), resource.Target(), err))
continue
}
if isHeld {
report.Outcomes = append(report.Outcomes, outcome)
if news {
log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail))
}
continue
}
}
was, _ := known.Find(resource.Identity())
outcome, err := applyOne(ctx, sys, resource, run, changed, declares, was, unseal)
var outcome Outcome
var err error
if o, isOpening := resource.(*declaration.Opening); isOpening {
outcome, err = applyOpening(ctx, o, run, fw)
} else {
// A file this host has no record of, under any id, is the machine's until the mesh
// writes over it — on any node, adopted or not: its original is kept first.
var keepFound Keep
if f, isFile := resource.(*declaration.File); isFile && was.ID == "" &&
!known.Recorded(string(declaration.TypeFile), f.Path) {
keepFound = keep
}
outcome, err = applyOne(ctx, sys, resource, run, changed, declares, was, unseal, keepFound)
}
if err != nil {
failed := &Error{Resource: resource.Identity(), Err: err, Done: report}
failures = append(failures, failed)
@@ -227,8 +385,14 @@ func Apply(
ID: resource.Identity(), Type: string(resource.Kind()),
Target: outcome.Target, AppliedAt: time.Now().UTC(),
Wrote: outcome.wrote,
Into: outcome.into,
Holds: holds(resource),
})
// Its module has been taken, and what was held for it is now the mesh's.
if held, wasHeld := known.HeldAt(resource.Identity()); wasHeld {
known.Release(held.ID)
outcome.Detail = takenDetail(held)
}
report.Outcomes = append(report.Outcomes, outcome)
if outcome.Action != "unchanged" {
changed[resource.Identity()] = true
@@ -236,6 +400,25 @@ func Apply(
}
}
if !orphansRemoved {
if err := removeOrphans(); err != nil {
return report, known, err
}
}
// A converged node whose found firewall was in force retires it only now, once everything —
// the mesh's derived filter among it — applied cleanly (novox/hq ADR 0100).
if len(failures) == 0 {
if err := retireFirewall(ctx, d, origin, &known, run, log); err != nil {
return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report}
}
for _, orphan := range protecting {
if err := removeOrphan(orphan); err != nil {
return report, known, err
}
}
}
if len(failures) > 0 {
// The first, carrying everything that did happen. One error is what the caller reports
// and what a person reads first; the rest are in the report, which is what the mesh
@@ -248,18 +431,22 @@ func Apply(
return report, known, nil
}
// guardPrefix is the ids of the mesh's guard on an adopted node: its package, table, unit and
// service (novox/hq ADR 0100).
const guardPrefix = declaration.AdoptionPrefix + "guard"
// Unseal opens a value the mesh sealed to this node. Nil when the node has no sealing key, which
// makes every sealed file an error rather than a silently skipped one.
type Unseal func(sealed string) ([]byte, error)
func applyOne(ctx context.Context, sys system.System, r declaration.Resource, run Runner,
changed map[string]bool, declares map[string]string, previous store.Applied,
unseal Unseal) (Outcome, error) {
unseal Unseal, keepFound Keep) (Outcome, error) {
switch res := r.(type) {
case *declaration.Directory:
return applyDirectory(res)
case *declaration.File:
return applyFile(res, previous, unseal)
return applyFile(res, previous, unseal, keepFound)
case *declaration.Service:
return applyService(ctx, sys, res, run, changed)
case *declaration.Package:
@@ -404,7 +591,12 @@ func applyAccess(r *declaration.Access) (Outcome, error) {
return out, nil
}
func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outcome, error) {
// keepFound, when not nil, is where the original of a file this host has no record of is kept
// before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome.
func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) {
if r.Into != "" {
return applyInto(r, previous)
}
out := begin(r)
// What actually goes on disk. For a sealed file the mesh never had this, and neither did
@@ -498,7 +690,15 @@ func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outc
drifted := existed && previous.Wrote != "" && digestOf(string(existing)) != previous.Wrote
modeSame := existed && beforeMode == mode.Perm()
kept := ""
if !contentSame {
if existed && keepFound != nil {
// Before anything is written: a keep that fails stops the write, since the
// original could not be had back otherwise.
if kept, err = keepFound(r.Path, existing, beforeMode); err != nil {
return out, fmt.Errorf("keeping the original of %s before writing over it: %w", r.Path, err)
}
}
if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil {
return out, err
}
@@ -562,6 +762,12 @@ func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outc
default:
out.Action = "unchanged"
}
if kept != "" {
if out.Detail != "" {
out.Detail += "; "
}
out.Detail += "the file found here, which the mesh had no record of, was kept at " + kept
}
return out, nil
}
@@ -604,11 +810,32 @@ func reflected(r *declaration.Service, changed map[string]bool) []string {
return restartedBy(r.RestartOn, changed)
}
// serviceReloader is a service manager that can tell a running unit to read its configuration again.
type serviceReloader interface {
ReloadService(ctx context.Context, run system.Runner, unit string) error
}
// unitReloader is a service manager that caches unit files and must be told to read them again.
type unitReloader interface {
ReloadUnits(ctx context.Context, run system.Runner) error
}
func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
changed map[string]bool) (Outcome, error) {
out := begin(r)
var changes []string
// A file the service reflects changed, and it may be the unit's own file or a drop-in: the
// service manager reads those again only when told to, and a restart without it runs the unit
// it had already loaded.
if reflects(r, changed) {
if u, ok := sys.(unitReloader); ok {
if err := u.ReloadUnits(ctx, run); err != nil {
return out, fmt.Errorf("reloading the service manager's units for %s: %w", r.Unit, err)
}
}
}
// Boot first. A unit asked to be running and enabled should survive this apply failing
// half way in the more useful direction: enabled-and-stopped comes back at the next boot,
// where running-and-disabled does not.
@@ -674,6 +901,25 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
"%s was restarted to pick up a change and is %s", r.Unit, after)
}
changes = append(changes, "restarted for "+strings.Join(reflected(r, changed), ", "))
} else if r.State == "running" && len(restartedBy(r.ReloadOn, changed)) > 0 {
// Told to read its configuration again, not stopped: for a service whose restart would
// stop what it runs — every container, for the container runtime (novox/hq ADR 0102).
reloader, ok := sys.(serviceReloader)
if !ok {
return out, fmt.Errorf("%s must be reloaded for %s and this machine's service manager "+
"cannot reload a unit", r.Unit, strings.Join(restartedBy(r.ReloadOn, changed), ", "))
}
if err := reloader.ReloadService(ctx, run, r.Unit); err != nil {
return out, fmt.Errorf("reloading %s: %w", r.Unit, err)
}
after, err := sys.ServiceState(ctx, run, r.Unit)
if err != nil {
return out, err
}
if after != "running" {
return out, fmt.Errorf("%s was reloaded to pick up a change and is %s", r.Unit, after)
}
changes = append(changes, "reloaded for "+strings.Join(restartedBy(r.ReloadOn, changed), ", "))
}
if len(changes) == 0 {
@@ -728,6 +974,9 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner)
return "removed", "no longer declared, and empty", nil
case declaration.TypeFile:
if a.Into != nil {
return removeInto(a)
}
if err := os.RemoveAll(a.Target); err != nil {
return "", "", err
}
+12
View File
@@ -1114,6 +1114,18 @@ func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) {
if !stopped || !started {
t.Errorf("the file changed and the service was not restarted; commands were %v", commands)
}
reloaded, stop := -1, -1
for i, c := range commands {
if strings.Contains(c, "daemon-reload") && reloaded < 0 {
reloaded = i
}
if strings.Contains(c, "stop thing.service") && stop < 0 {
stop = i
}
}
if reloaded < 0 || reloaded > stop {
t.Errorf("the service was restarted without the unit files being read again first; commands were %v", commands)
}
}
}
+637
View File
@@ -0,0 +1,637 @@
package apply
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// Keep records the original of a file found on an adopted node, before anything else happens to
// it, and says where (novox/hq ADR 0100). It never overwrites an original it already kept: the
// first copy is the one that was there before the mesh.
type Keep func(path string, content []byte, mode os.FileMode) (string, error)
// KeepIn keeps originals under dir/kept, each named for the path it came from AND for what was in
// it, readable by root alone — a predecessor's configuration may carry its credentials.
//
// **By content as well as path, because a path has more than one original.** A file held, let go
// when its module was unassigned, rewritten by the predecessor and found again is a second
// original; named by path alone the second copy was silently discarded while the report said it
// was kept (novox/hq ADR 0100). The same content at the same path is kept once.
func KeepIn(dir string) Keep {
return func(path string, content []byte, _ os.FileMode) (string, error) {
where := sha256.Sum256([]byte(path))
what := sha256.Sum256(content)
kept := filepath.Join(dir, "kept", hex.EncodeToString(where[:])[:12]+"-"+
hex.EncodeToString(what[:])[:12]+"-"+filepath.Base(path))
if _, err := os.Lstat(kept); err == nil {
return kept, nil
}
if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil {
return "", err
}
if err := writeAtomically(kept, content, 0o600); err != nil {
return "", err
}
back, err := os.ReadFile(kept)
if err != nil || string(back) != string(content) {
return "", fmt.Errorf("kept the original of %s at %s and cannot read it back", path, kept)
}
return kept, nil
}
}
// foundBefore is what an adopted apply finds on the machine before it changes anything: each
// directory, service unit and container mount source of an untaken module that is present with no
// record (novox/hq ADR 0103). Looked at first, because the apply itself makes such things — a
// file's parent directory, a unit file a module writes, a package that brings its unit — and what
// the mesh made in this apply was not found.
type foundBefore struct {
is map[string]bool
// trouble is what could not be asked about, by the same key, so the resource that would need
// the answer fails rather than proceeding as if the machine had nothing there.
trouble map[string]string
}
func (f foundBefore) has(key string) bool { return f.is[key] }
// why is the reason a key could not be settled, and empty when there was none.
func (f foundBefore) why(key string) string { return f.trouble[key] }
func lookBefore(ctx context.Context, sys system.System, d *declaration.Declaration, known store.State,
run Runner) foundBefore {
seen := foundBefore{is: map[string]bool{}, trouble: map[string]string{}}
if d.Adoption == nil {
return seen
}
cri, asked := "", false
for _, r := range d.Resources {
if _, untaken := d.Adoption.UntakenModuleOf(r.Identity()); !untaken {
continue
}
if _, held := known.HeldAt(r.Identity()); held {
continue
}
switch res := r.(type) {
case *declaration.Directory:
if present(res.Path) && !recordedPath(known, res.Path) {
seen.is["path:"+res.Path] = true
}
case *declaration.Archive:
// Unpacking over it, and re-owning it recursively, would change the predecessor's
// files.
if present(res.Path) && !recordedPath(known, res.Path) {
seen.is["path:"+res.Path] = true
}
case *declaration.Process:
// Its unit would be written over and restarted.
if !known.Recorded(string(declaration.TypeProcess), res.Name) &&
present(filepath.Join(unitDir, res.Name+".service")) {
seen.is["unit-file:"+res.Name] = true
}
case *declaration.User:
// Its shell and groups would be changed.
if !known.Recorded(string(declaration.TypeUser), res.Name) {
if _, exists, err := system.LookUpUser(ctx, run, res.Name); err == nil && exists {
seen.is["user:"+res.Name] = true
}
}
case *declaration.Service:
if known.Recorded(string(declaration.TypeService), res.Unit) {
continue
}
// **Found is a unit somebody put on this machine, or one the machine uses.**
//
// Where it comes from first: a unit the service manager loads from outside /usr —
// /etc/systemd/system or /run/systemd/system — was installed by an administrator, so
// it is a predecessor's whatever state it is in, and one deliberately stopped and
// disabled must stay that way (novox/hq ADR 0103).
//
// A unit a package ships, under /usr, is not held by its mere presence: the private
// network's own wg-quick@mesh0 is an instance of a template the tunnel package ships,
// nothing had ever run it, and holding it kept the private network from ever coming up
// (found by the adoption bed). Such a unit is held only if the machine actually uses
// it — running, or started at boot.
state, err := sys.ServiceState(ctx, run, res.Unit)
if err != nil {
continue
}
if from, ok := sys.(unitFiles); ok {
if path, err := from.ServiceUnitFile(ctx, run, res.Unit); err == nil && installedByHand(path) {
seen.is["unit:"+res.Unit] = true
continue
}
}
boot, _ := sys.ServiceBoot(ctx, run, res.Unit)
if state == "running" || boot == "enabled" {
seen.is["unit:"+res.Unit] = true
}
case *declaration.Container:
if known.Recorded(string(declaration.TypeContainer), res.Name) {
continue
}
for _, v := range res.Volumes {
src := mountSource(v)
switch {
case src == "":
case strings.HasPrefix(src, "/"):
if !systemPath(src) && present(src) && !recordedPath(known, src) {
seen.is["path:"+src] = true
}
default:
if !asked {
cri, _ = containerRuntime(ctx, run)
asked = true
}
if cri == "" {
continue
}
if _, err := run(ctx, cri, "volume", "inspect", src); err == nil {
seen.is["volume:"+src] = true
} else if !absent(err) {
seen.trouble["volume:"+src] = fmt.Sprintf(
"the container runtime could not say whether the volume %s is here: %v", src, err)
}
}
}
}
}
return seen
}
// unitFiles is a service manager that can say where it loads a unit from.
type unitFiles interface {
ServiceUnitFile(ctx context.Context, run Runner, unit string) (string, error)
}
// installedByHand is whether a unit file is one somebody put on this machine rather than one a
// package ships: anywhere but /usr, where distributions keep what they install.
func installedByHand(path string) bool {
if path == "" {
return false
}
return !strings.HasPrefix(filepath.Clean(path), "/usr/")
}
func present(path string) bool {
_, err := os.Lstat(path)
return err == nil
}
// recordedPath is whether this host has a record of MAKING something at a path — a directory it
// created, a file it wrote, an archive it unpacked. An access record is not one of those: it says
// the mesh set permissions on a path it does not own, which is exactly what it does to a path
// somebody else's software made, so a path it only has access for is still found (novox/hq ADR 0103).
func recordedPath(known store.State, path string) bool {
for _, kind := range []declaration.Type{declaration.TypeDirectory, declaration.TypeFile,
declaration.TypeArchive} {
if known.Recorded(string(kind), path) {
return true
}
}
return false
}
// systemPath is whether a bind-mount source is the machine's own plumbing — the runtime's socket,
// the kernel's filesystems, the devices, the clock — which every machine has and no predecessor's
// data lives in. Mounting it shares nothing that was found.
func systemPath(src string) bool {
clean := filepath.Clean(src)
for _, exact := range []string{"/etc/localtime", "/etc/timezone", "/etc/hosts", "/etc/resolv.conf",
"/etc/machine-id", "/etc/passwd", "/etc/group"} {
if clean == exact {
return true
}
}
for _, under := range []string{"/run", "/var/run", "/sys", "/proc", "/dev", "/usr/share/zoneinfo",
"/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/lib/modules", "/usr/lib/modules"} {
if clean == under || strings.HasPrefix(clean, under+"/") {
return true
}
}
return false
}
// mountSource is what a volume mapping mounts: a path on the machine, or a named volume. Empty for
// an anonymous volume, which mounts nothing that could already be there.
func mountSource(mapping string) string {
src, _, ok := strings.Cut(mapping, ":")
if !ok {
return ""
}
return src
}
// runsIn is the container a resource runs inside, if any: an action's `in`, or a run-once step
// sharing a container's namespace. An action with no `in` runs on the machine itself and is not
// held for a container: it reaches nothing a predecessor holds by running there, and holding every
// action of an untaken module would stop a module preparing itself before its cutover.
func runsIn(r declaration.Resource) string {
switch res := r.(type) {
case *declaration.Action:
return res.In
case *declaration.Container:
if res.RunOnce {
if name, ok := strings.CutPrefix(res.Network, "container:"); ok {
return name
}
}
}
return ""
}
// heldContainer is what is held under a container's name.
func heldContainer(known store.State, name string) (store.Held, bool) {
for _, h := range known.Held {
if h.Kind == string(declaration.TypeContainer) && h.Target == name {
return h, true
}
}
return store.Held{}, false
}
// holdOnAdopted decides whether a resource of an adopted node is held rather than applied, and
// holds it (novox/hq ADR 0100, ADR 0103). For a module not yet taken, what is present with no
// record is kept as it is: a file or a container under its name, a directory, a service's unit,
// and a container that would mount a path or a volume found there. An action or a run-once step
// run inside a held container is held with it. Once held, a resource stays held — changed or gone
// — until its module is taken, and it is never recorded as applied, so never removed as an orphan.
//
// Held is false for a resource to apply as usual. News is whether the hold is new or changed,
// which is what is worth a line in the log.
func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resource, d *declaration.Declaration,
known *store.State, before foundBefore, run Runner, keep Keep, changed map[string]bool,
now time.Time) (held, news bool, out Outcome, err error) {
was, already := known.HeldAt(r.Identity())
if in := runsIn(r); in != "" {
if container, isHeld := heldContainer(*known, in); isHeld {
module, untaken := d.Adoption.UntakenModuleOf(r.Identity())
if !untaken {
module = container.Module
}
h := was
if !already {
h = store.Held{ID: r.Identity(), Kind: string(r.Kind()), Target: r.Target(), Since: now}
}
h.Module, h.Why = module, "runs in "+in
known.RecordHeld(h)
out = begin(r)
out.Action = "held"
out.Detail = fmt.Sprintf("runs in %s, which is held as found; not run until %s is taken", in, module)
return true, !already, out, nil
}
}
// A file written into replaces nothing that was found, so it is never held (novox/hq ADR
// 0102) — and a hold from when it was declared whole must not keep the mesh's keys out.
if f, ok := r.(*declaration.File); ok && f.Into != "" {
if already {
known.Release(r.Identity())
}
return false, false, out, nil
}
module, untaken := d.Adoption.UntakenModuleOf(r.Identity())
if !untaken {
return false, false, out, nil
}
why := was.Why
isFound := already
if !already {
switch res := r.(type) {
case *declaration.File:
if res.Into == "" {
if isFound, err = found(ctx, r, run, *known); err != nil {
return false, false, begin(r), err
}
}
case *declaration.Container:
if known.Recorded(string(declaration.TypeContainer), res.Name) {
break
}
_, exists, err := inspectFound(ctx, res.Name, run)
if err != nil {
return false, false, begin(r), err
}
if exists {
if isFound, err = found(ctx, r, run, *known); err != nil {
return false, false, begin(r), err
}
break
}
// Not there under its name, and still it would share what was found: created, it
// would mount the predecessor's data beside the predecessor's own container.
for _, v := range res.Volumes {
src := mountSource(v)
key := "volume:" + src
if strings.HasPrefix(src, "/") {
key = "path:" + src
}
if src == "" {
continue
}
if trouble := before.why(key); trouble != "" {
return false, false, begin(r), fmt.Errorf(
"%s, so it is not safe to create a container that would mount it", trouble)
}
if before.has(key) {
isFound, why = true, "would mount "+src+", found on the machine"
break
}
}
case *declaration.Directory:
isFound = before.has("path:" + res.Path)
case *declaration.Archive:
isFound = before.has("path:" + res.Path)
case *declaration.Process:
isFound = before.has("unit-file:" + res.Name)
case *declaration.User:
isFound = before.has("user:" + res.Name)
case *declaration.Service:
isFound = before.has("unit:" + res.Unit)
}
}
if !isFound {
return false, false, out, nil
}
out, h, err := hold(ctx, sys, r, module, was, already, why, run, keep, now)
if err != nil {
return true, false, out, err
}
// A held service is not started, stopped, enabled or restarted — but a reload stops nothing,
// so one the module names still happens (novox/hq ADR 0102, ADR 0103).
if svc, ok := r.(*declaration.Service); ok && svc.State == "running" {
if which := restartedBy(svc.ReloadOn, changed); len(which) > 0 {
if state, err := sys.ServiceState(ctx, run, svc.Unit); err == nil && state == "running" {
reloader, can := sys.(serviceReloader)
if !can {
return true, false, out, fmt.Errorf("%s must be reloaded for %s and this machine's "+
"service manager cannot reload a unit", svc.Unit, strings.Join(which, ", "))
}
if err := reloader.ReloadService(ctx, run, svc.Unit); err != nil {
return true, false, out, fmt.Errorf("reloading the held %s: %w", svc.Unit, err)
}
out.Detail += "; reloaded for " + strings.Join(which, ", ") + ", which stops nothing"
}
}
}
known.RecordHeld(h)
return true, !already || h.Changed != was.Changed, out, nil
}
// found is whether a declared file or container is present on the machine with no record of this
// host making it (novox/hq ADR 0100). A container carrying the host's own spec label was made by
// a host, whatever this store says, so it is never found.
func found(ctx context.Context, r declaration.Resource, run Runner, known store.State) (bool, error) {
if known.Recorded(string(r.Kind()), r.Target()) {
return false, nil
}
switch res := r.(type) {
case *declaration.File:
_, err := os.Lstat(res.Path)
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
return err == nil, err
case *declaration.Container:
seen, exists, err := inspectFound(ctx, res.Name, run)
if err != nil || !exists {
return false, err
}
return seen.spec == "", nil
}
return false, nil
}
type foundContainer struct {
id string
running bool
spec string
}
// inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and
// whether a host made it.
func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, bool, error) {
cri, err := containerRuntime(ctx, run)
if err != nil {
return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name)
}
out, err := run(ctx, cri, "inspect", "--format",
"{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
if err != nil {
if absent(err) {
return foundContainer{}, false, nil
}
// **A runtime that could not answer is not a machine with nothing there.** Read as
// absence, a daemon that is down or a permission denied would let the mesh create its own
// container over a predecessor's — the one thing an adopted node must never do
// (novox/hq ADR 0100).
return foundContainer{}, false, fmt.Errorf(
"the container runtime could not say whether %s is here, so it is not safe to make one: %w",
name, err)
}
parts := strings.Split(strings.TrimSpace(out), "\t")
for len(parts) < 3 {
parts = append(parts, "")
}
spec := strings.TrimSpace(parts[2])
if spec == "<no value>" {
spec = ""
}
return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil
}
// absent is whether a runtime said the thing is not there, rather than failing to answer. Its own
// words: docker and podman both say "No such object", "No such container" or "No such volume".
func absent(err error) bool {
said := strings.ToLower(err.Error())
for _, missing := range []string{"no such object", "no such container", "no such volume",
"no such image"} {
if strings.Contains(said, missing) {
return true
}
}
return false
}
// hold keeps a found file or container as it is, and reports it — the first time by recording
// what was found, every time after by comparing against that. Nothing is reverted, restarted or
// created: a held target that disappears stays held and gone until its module is taken.
func hold(ctx context.Context, sys system.System, r declaration.Resource, module string, was store.Held,
already bool, why string, run Runner, keep Keep, now time.Time) (Outcome, store.Held, error) {
out := begin(r)
h := was
if !already {
h = store.Held{ID: r.Identity(), Module: module, Kind: string(r.Kind()),
Target: r.Target(), Since: now, Why: why}
}
h.Module = module
detail := "found on the machine; kept until " + module + " is taken"
var changed string
switch res := r.(type) {
case *declaration.File:
info, err := os.Lstat(res.Path)
switch {
case errors.Is(err, os.ErrNotExist):
if !already {
return out, h, fmt.Errorf("%s was found and is gone before it could be kept", res.Path)
}
changed = "gone"
case err != nil:
return out, h, err
default:
content, err := os.ReadFile(res.Path)
if err != nil {
return out, h, fmt.Errorf("%s was found and cannot be read to keep it: %w", res.Path, err)
}
if !already {
// The original first, before anything is recorded: a hold with no kept copy
// would be a promise the host cannot keep.
if keep == nil {
return out, h, fmt.Errorf(
"%s was found on this adopted node and this host has nowhere to keep its original", res.Path)
}
kept, err := keep(res.Path, content, info.Mode().Perm())
if err != nil {
return out, h, fmt.Errorf("keeping the original of %s: %w", res.Path, err)
}
h.Kept = kept
h.Digest = digestOf(string(content))
h.Mode = fmt.Sprintf("%04o", info.Mode().Perm())
if st, ok := info.Sys().(*syscall.Stat_t); ok {
h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid)
}
} else if digestOf(string(content)) != h.Digest {
changed = "rewritten"
}
}
case *declaration.Directory:
info, err := os.Lstat(res.Path)
switch {
case errors.Is(err, os.ErrNotExist):
if !already {
return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path)
}
changed = "gone"
case err != nil:
return out, h, err
case !already:
// Its mode and owner as found, which the mesh leaves: a database refuses to start
// on a data directory whose mode changed.
h.Mode = fmt.Sprintf("%04o", info.Mode().Perm())
if st, ok := info.Sys().(*syscall.Stat_t); ok {
h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid)
}
}
detail = "found on the machine; its mode, owner and contents kept until " + module + " is taken"
case *declaration.Archive:
if _, err := os.Lstat(res.Path); errors.Is(err, os.ErrNotExist) {
if !already {
return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path)
}
changed = "gone"
} else if err != nil {
return out, h, err
}
detail = "something is already at " + res.Path + "; nothing unpacked over it or re-owned until " +
module + " is taken"
case *declaration.Process:
unit := filepath.Join(unitDir, res.Name+".service")
if !present(unit) {
if !already {
return out, h, fmt.Errorf("%s was found and is gone before it could be held", unit)
}
changed = "gone"
}
detail = "its unit " + unit + " was found on the machine; not written over or restarted until " +
module + " is taken"
case *declaration.User:
_, exists, err := system.LookUpUser(ctx, run, res.Name)
switch {
case err != nil:
return out, h, err
case !exists && !already:
return out, h, fmt.Errorf("the user %s was found and is gone before it could be held", res.Name)
case !exists:
changed = "gone"
}
detail = "the user was found on the machine; its shell and groups are kept until " + module + " is taken"
case *declaration.Service:
state, err := sys.ServiceState(ctx, run, res.Unit)
switch {
case err != nil && !already:
return out, h, fmt.Errorf("the unit %s was found and cannot be read to hold it: %w", res.Unit, err)
case err != nil:
changed = "gone"
case !already:
h.Running = state == "running"
case h.Running && state != "running":
changed = "stopped"
}
detail = "its unit was found on the machine; its state and whether it starts at boot are " +
"kept until " + module + " is taken"
case *declaration.Container:
if h.Why != "" && h.Container == "" {
// Held for what it would mount, never created: there is nothing of it to compare.
detail = "not created: it " + h.Why + "; kept until " + module + " is taken"
break
}
seen, exists, err := inspectFound(ctx, res.Name, run)
if err != nil {
return out, h, err
}
switch {
case !exists && !already:
return out, h, fmt.Errorf("container %s was found and is gone before it could be held", res.Name)
case !already:
h.Container, h.Running = seen.id, seen.running
case !exists:
changed = "gone"
case seen.id != h.Container:
changed = "replaced"
case h.Running && !seen.running:
changed = "stopped"
}
default:
return out, h, fmt.Errorf("a %s cannot be held", r.Kind())
}
if changed != h.Changed {
h.Changed = changed
h.ChangedAt = now
if changed == "" {
h.ChangedAt = time.Time{}
}
}
out.Action = "held"
out.Detail = detail
if h.Changed != "" {
out.Detail += "; " + h.Changed + " by something other than the mesh since it was found, and not reverted"
}
return out, h, nil
}
// takenDetail is what an outcome says when a module's cutover replaced what was held for it.
func takenDetail(h store.Held) string {
if h.Kind == string(declaration.TypeAction) || (h.Why != "" && h.Container == "") {
return "taken: no longer held (" + h.Why + ")"
}
if h.Kept != "" {
return "taken: replaced what was found; original kept at " + h.Kept
}
return "taken: replaced what was found"
}
File diff suppressed because it is too large Load Diff
+388
View File
@@ -0,0 +1,388 @@
package apply
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"sort"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// A file written into, never over (novox/hq ADR 0102).
//
// **The file is the machine's; the mesh owns keys in it.** The container runtime's configuration
// is the case that needed it: the mesh states one fact there — its registry is trusted over the
// private network — and writing the file whole replaced everything the machine had set, down to
// where the runtime keeps its data. So the host reads what is there, sets only the declared keys,
// keeps every other key as it found it, and records what each of its keys held before. Undeclared,
// each key goes back, and a file the mesh created goes only if nothing but its keys is left.
// applyInto writes a file's declared keys into the object already at its path.
func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) {
out := begin(r)
if r.Into != declaration.IntoJSON {
return out, fmt.Errorf("%s: into %q is not a format this host writes into", r.Path, r.Into)
}
var declared map[string]json.RawMessage
if err := json.Unmarshal([]byte(r.Content), &declared); err != nil {
return out, fmt.Errorf("%s: the keys to write are not a JSON object: %w", r.Path, err)
}
existing, err := os.ReadFile(r.Path)
existed := err == nil
if err != nil && !errors.Is(err, os.ErrNotExist) {
return out, err
}
object := map[string]json.RawMessage{}
if existed && len(bytes.TrimSpace(existing)) > 0 {
if err := json.Unmarshal(existing, &object); err != nil || object == nil {
// Refused, never replaced: a file the host cannot read as an object is a file it
// cannot write into without losing whatever it is.
return out, fmt.Errorf("%s is not a JSON object, so the mesh cannot write its keys into it "+
"without replacing what is there; it was left as it is", r.Path)
}
}
rec := store.Into{Format: declaration.IntoJSON, Before: map[string]json.RawMessage{},
Added: map[string][]json.RawMessage{}}
if previous.Into != nil {
rec.Created = previous.Into.Created
for k, v := range previous.Into.Before {
rec.Before[k] = v
}
rec.Absent = slices.Clone(previous.Into.Absent)
for k, v := range previous.Into.Added {
rec.Added[k] = slices.Clone(v)
}
} else {
rec.Created = !existed
}
tracked := func(k string) bool {
_, before := rec.Before[k]
_, added := rec.Added[k]
return before || added || slices.Contains(rec.Absent, k)
}
// Drift: the machine no longer holds what this host last set in its keys.
drifted := previous.Wrote != "" && existed && digestOf(viewOf(object, rec, keysTracked(rec))) != previous.Wrote
// Keys the mesh set before and no longer declares go back to what they held.
for _, k := range keysTracked(rec) {
if _, still := declared[k]; still {
continue
}
giveBack(object, &rec, k)
}
// Declared keys: remember what each held the first time, then set it. A list is the
// machine's too — a predecessor's own trusted registries, say — so the mesh adds its members
// to it rather than replacing it, and remembers exactly which it added.
for _, k := range keysIn(declared) {
_, scalar := rec.Before[k]
if isList(declared[k]) && !scalar {
current, had := object[k]
if had && !isList(current) {
return out, fmt.Errorf("%s: the mesh adds to the list %q, and the machine holds something "+
"other than a list there; it was left as it is", r.Path, k)
}
if !tracked(k) && !had {
rec.Absent = append(rec.Absent, k)
}
merged, added, err := addMembers(current, declared[k], rec.Added[k], previous.Into == nil)
if err != nil {
return out, fmt.Errorf("%s: %q: %w", r.Path, k, err)
}
rec.Added[k] = added
object[k] = merged
continue
}
if !tracked(k) {
v, had := object[k]
// **What the host may have written itself is not the machine's.** With no record of
// this file — the first apply, or a host that wrote and died before saving its state —
// a key already holding exactly what the mesh declares cannot be told from one the
// mesh set a moment ago. Remembered as the machine's, it would never be given back:
// undeclaring would leave the mesh's own value behind for ever. So it is the mesh's,
// and undeclaring takes it out (novox/hq ADR 0102).
if had && !(previous.Into == nil && canonical(v) == canonical(declared[k])) {
rec.Before[k] = v
} else {
rec.Absent = append(rec.Absent, k)
}
}
object[k] = declared[k]
}
want, err := render(object)
if err != nil {
return out, err
}
same := existed && canonical(existing) == canonical(want)
if !same {
mode := os.FileMode(0o644)
if info, err := os.Stat(r.Path); err == nil {
mode = info.Mode().Perm() // the machine's file keeps the machine's mode
} else if r.Mode != "" {
if m, err := modeOf(r.Mode, mode); err == nil {
mode = m
}
}
if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil {
return out, err
}
if err := writeAtomically(r.Path, want, mode); err != nil {
return out, err
}
}
// Read back: every declared key holds what was declared.
written, err := os.ReadFile(r.Path)
if err != nil {
return out, fmt.Errorf("wrote into %s and cannot read it back: %w", r.Path, err)
}
var check map[string]json.RawMessage
if err := json.Unmarshal(written, &check); err != nil {
return out, fmt.Errorf("%s is not a JSON object after writing into it: %w", r.Path, err)
}
for k, v := range declared {
if _, list := rec.Added[k]; list {
members, _ := membersOf(v)
have, err := membersOf(check[k])
if err != nil {
return out, fmt.Errorf("%s does not hold a list at %q after writing into it", r.Path, k)
}
for _, m := range members {
if !hasMember(have, m) {
return out, fmt.Errorf("%s does not hold the declared %s in %q after writing into it", r.Path, m, k)
}
}
continue
}
if canonical(check[k]) != canonical(v) {
return out, fmt.Errorf("%s does not hold the declared %q after writing into it", r.Path, k)
}
}
if len(rec.Before) == 0 {
rec.Before = nil
}
if len(rec.Added) == 0 {
rec.Added = nil
}
out.into = &rec
out.wrote = digestOf(viewOf(check, rec, keysIn(declared)))
switch {
case !existed:
out.Action = "created"
out.Detail = "written into; the file was not there"
case same:
out.Action = "unchanged"
case drifted:
out.Action = "corrected"
out.Detail = "the mesh's keys had been changed on the machine; the rest of the file was kept"
default:
out.Action = "updated"
out.Detail = "the mesh's keys written in; every other key kept as it was"
}
return out, nil
}
// removeInto gives back what a file written into held before the mesh's keys.
func removeInto(a store.Applied) (string, string, error) {
existing, err := os.ReadFile(a.Target)
if errors.Is(err, os.ErrNotExist) {
return "forgotten", "no longer there", nil
}
if err != nil {
return "", "", err
}
object := map[string]json.RawMessage{}
if len(bytes.TrimSpace(existing)) > 0 {
if err := json.Unmarshal(existing, &object); err != nil || object == nil {
return "kept", "no longer a JSON object, so the mesh's keys were left in it; " +
"remove them by hand", nil
}
}
rec := *a.Into
for _, k := range keysTracked(rec) {
giveBack(object, &rec, k)
}
if a.Into.Created && len(object) == 0 {
if err := os.Remove(a.Target); err != nil {
return "", "", err
}
return "removed", "no longer declared; the mesh had created it and nothing else was in it", nil
}
want, err := render(object)
if err != nil {
return "", "", err
}
info, err := os.Stat(a.Target)
if err != nil {
return "", "", err
}
if err := writeAtomically(a.Target, want, info.Mode().Perm()); err != nil {
return "", "", err
}
return "restored", "no longer declared; the mesh's keys were given back what they held", nil
}
func giveBack(object map[string]json.RawMessage, rec *store.Into, k string) {
if added, list := rec.Added[k]; list {
// Only the members the mesh added go; the list and everything else in it stay, unless
// the mesh made the key and nothing is left in it.
wasAbsent := slices.Contains(rec.Absent, k)
if current, had := object[k]; had && isList(current) {
have, _ := membersOf(current)
have = slices.DeleteFunc(have, func(m json.RawMessage) bool { return hasMember(added, m) })
if len(have) == 0 && wasAbsent {
delete(object, k)
} else {
object[k] = listOf(have)
}
}
delete(rec.Added, k)
rec.Absent = slices.DeleteFunc(rec.Absent, func(a string) bool { return a == k })
return
}
if v, had := rec.Before[k]; had {
object[k] = v
delete(rec.Before, k)
return
}
delete(object, k)
rec.Absent = slices.DeleteFunc(rec.Absent, func(a string) bool { return a == k })
}
func keysTracked(rec store.Into) []string {
var keys []string
for k := range rec.Before {
keys = append(keys, k)
}
for k := range rec.Added {
keys = append(keys, k)
}
keys = append(keys, rec.Absent...)
sort.Strings(keys)
return slices.Compact(keys)
}
// viewOf is what the mesh holds itself to in a file written into: each scalar key's value, and for
// a list only whether each member the mesh added is still there — what the machine keeps beside
// them is not the mesh's to judge.
func viewOf(object map[string]json.RawMessage, rec store.Into, keys []string) string {
var b bytes.Buffer
for _, k := range keys {
if added, list := rec.Added[k]; list {
have, _ := membersOf(object[k])
b.WriteString(k + " holds")
for _, m := range added {
fmt.Fprintf(&b, " %s=%v", canonical(m), hasMember(have, m))
}
b.WriteString("\n")
continue
}
b.WriteString(k + "=" + canonical(object[k]) + "\n")
}
return b.String()
}
func isList(raw json.RawMessage) bool {
t := bytes.TrimSpace(raw)
return len(t) > 0 && t[0] == '['
}
func membersOf(raw json.RawMessage) ([]json.RawMessage, error) {
if len(bytes.TrimSpace(raw)) == 0 {
return nil, nil
}
var members []json.RawMessage
if err := json.Unmarshal(raw, &members); err != nil {
return nil, err
}
return members, nil
}
func hasMember(list []json.RawMessage, m json.RawMessage) bool {
for _, have := range list {
if canonical(have) == canonical(m) {
return true
}
}
return false
}
func listOf(members []json.RawMessage) json.RawMessage {
if members == nil {
members = []json.RawMessage{}
}
raw, _ := json.Marshal(members)
return raw
}
// addMembers adds the declared members to the machine's list, dropping only members the mesh
// added before and no longer declares. It returns the list and exactly which members the mesh
// added — a declared member the machine already had is the machine's, and is never recorded.
// unrecorded says there is no record of this file yet, in which case a declared member already in
// the list may be one the host itself wrote before it could save its state, and is taken as the
// mesh's.
func addMembers(current, declared json.RawMessage, addedBefore []json.RawMessage,
unrecorded bool) (json.RawMessage, []json.RawMessage, error) {
have, err := membersOf(current)
if err != nil {
return nil, nil, err
}
want, err := membersOf(declared)
if err != nil {
return nil, nil, err
}
added := []json.RawMessage{}
for _, a := range addedBefore {
if hasMember(want, a) {
added = append(added, a)
continue
}
have = slices.DeleteFunc(have, func(m json.RawMessage) bool { return canonical(m) == canonical(a) })
}
for _, m := range want {
if !hasMember(have, m) {
have = append(have, m)
} else if !unrecorded {
continue // the machine's own, and never the mesh's to take out
}
if !hasMember(added, m) {
added = append(added, m)
}
}
return listOf(have), added, nil
}
func keysIn(m map[string]json.RawMessage) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// canonical is a JSON value compacted, so formatting is not mistaken for a change.
func canonical(raw []byte) string {
var b bytes.Buffer
if err := json.Compact(&b, raw); err != nil {
return string(raw)
}
return b.String()
}
func render(object map[string]json.RawMessage) ([]byte, error) {
b, err := json.MarshalIndent(object, "", " ")
if err != nil {
return nil, err
}
return append(b, '\n'), nil
}
+342
View File
@@ -0,0 +1,342 @@
package apply
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0102: a file the mesh shares with software it did not install is written
// into, never over, and a service that re-reads its configuration is reloaded, not restarted.
func intoDecl(t *testing.T, path, keys string) string {
t.Helper()
return fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q}
]}`, path, keys)
}
func readObject(t *testing.T, path string) map[string]any {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var o map[string]any
if err := json.Unmarshal(raw, &o); err != nil {
t.Fatalf("%s is not a JSON object: %v\n%s", path, err, raw)
}
return o
}
// The machine's own runtime settings, the way a predecessor leaves them.
const machinesOwn = `{"data-root":"/srv/docker","log-opts":{"max-size":"10m"},"insecure-registries":["192.0.2.7:5000"]}`
func TestWritingIntoKeepsEveryKeyTheMachineHad(t *testing.T) {
path := filepath.Join(t.TempDir(), "daemon.json")
if err := os.WriteFile(path, []byte(machinesOwn), 0o600); err != nil {
t.Fatal(err)
}
d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
o := readObject(t, path)
if o["data-root"] != "/srv/docker" {
t.Errorf("the machine's data directory was not kept: %v", o)
}
if fmt.Sprint(o["log-opts"]) != "map[max-size:10m]" {
t.Errorf("the machine's logging settings were not kept: %v", o)
}
if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" {
t.Errorf("the mesh's member was not added beside the machine's own: %v", o)
}
if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 {
t.Errorf("the machine's file mode was changed to %o", info.Mode().Perm())
}
if got := report.Outcomes[0].Action; got != "updated" {
t.Errorf("writing into was reported as %q", got)
}
// Again, with nothing changed: nothing to do.
report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if got := report.Outcomes[0].Action; got != "unchanged" {
t.Errorf("a second apply was %q", got)
}
// Undeclared: the key goes back to what the machine had, and the file stays.
empty := somethingElse(t)
report, _, err = Apply(context.Background(), archHost(t), empty, state, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
o = readObject(t, path)
if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" || o["data-root"] != "/srv/docker" {
t.Errorf("undeclaring did not give the machine back what it had: %v", o)
}
if got := report.Outcomes[0].Action; got != "restored" {
t.Errorf("undeclaring was reported as %q", got)
}
}
func TestAFileWrittenIntoThatWasNotThereIsRemovedWhenOnlyTheMeshsKeysAreLeft(t *testing.T) {
path := filepath.Join(t.TempDir(), "daemon.json")
d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if got := report.Outcomes[0].Action; got != "created" {
t.Errorf("writing into a file that was not there was %q", got)
}
// Somebody else adds a key of their own: the file is no longer only the mesh's.
o := readObject(t, path)
o["debug"] = true
raw, _ := json.Marshal(o)
_ = os.WriteFile(path, raw, 0o644)
empty := somethingElse(t)
if _, _, err := Apply(context.Background(), archHost(t), empty, state, store.OriginDeclared, nil, nil, nil); err != nil {
t.Fatal(err)
}
o = readObject(t, path)
if _, still := o["insecure-registries"]; still || o["debug"] != true {
t.Errorf("undeclaring should remove the mesh's key and keep the other: %v", o)
}
// Without the other key, the file the mesh created goes.
path2 := filepath.Join(t.TempDir(), "daemon.json")
d2 := parse(t, intoDecl(t, path2, `{"insecure-registries":["10.42.0.1:5000"]}`))
_, state2, err := Apply(context.Background(), archHost(t), d2, store.State{}, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if _, _, err := Apply(context.Background(), archHost(t), empty, state2, store.OriginDeclared, nil, nil, nil); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(path2); !os.IsNotExist(err) {
t.Errorf("a file the mesh created, holding only its keys, was left behind")
}
}
func TestAKeyNoLongerDeclaredGoesBackAndANewOneIsRemembered(t *testing.T) {
path := filepath.Join(t.TempDir(), "daemon.json")
_ = os.WriteFile(path, []byte(machinesOwn), 0o644)
first := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
_, state, err := Apply(context.Background(), archHost(t), first, store.State{}, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
second := parse(t, intoDecl(t, path, `{"registry-mirrors":["http://10.42.0.1:5000"]}`))
if _, _, err := Apply(context.Background(), archHost(t), second, state, store.OriginDeclared, nil, nil, nil); err != nil {
t.Fatal(err)
}
o := readObject(t, path)
if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" {
t.Errorf("a key the mesh stopped declaring was not given back: %v", o)
}
if fmt.Sprint(o["registry-mirrors"]) != "[http://10.42.0.1:5000]" {
t.Errorf("the newly declared key was not written: %v", o)
}
}
func TestAFileThatIsNotAnObjectIsRefusedAndLeftAlone(t *testing.T) {
path := filepath.Join(t.TempDir(), "daemon.json")
_ = os.WriteFile(path, []byte("# not json at all\n"), 0o644)
d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil); err == nil {
t.Fatal("writing into a file that is not a JSON object was not refused")
}
raw, _ := os.ReadFile(path)
if string(raw) != "# not json at all\n" {
t.Errorf("a file the mesh could not write into was changed: %q", raw)
}
}
func TestAFileWrittenIntoIsNeverHeldOnAnAdoptedNode(t *testing.T) {
path := filepath.Join(t.TempDir(), "daemon.json")
_ = os.WriteFile(path, []byte(machinesOwn), 0o644)
d := adopted(t, `{"taken":[],"untaken":{"networking":["networking.registry-trust"]}}`,
fmt.Sprintf(`{"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q}`,
path, `{"insecure-registries":["10.42.0.1:5000"]}`))
m := &machine{}
report, state := applyAdopted(t, d, store.State{}, m, t.TempDir())
if got := outcomeOf(report, "networking.registry-trust").Action; got == "held" {
t.Fatal("a file written into was held, though it replaces nothing that was found")
}
if len(state.Held) != 0 {
t.Errorf("something was held: %+v", state.Held)
}
o := readObject(t, path)
if o["data-root"] != "/srv/docker" || fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" {
t.Errorf("the adopted node's file was not written into: %v", o)
}
}
func TestAServiceIsReloadedNotRestartedForWhatItReloadsOn(t *testing.T) {
path := filepath.Join(t.TempDir(), "daemon.json")
d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"trust","type":"file","path":%q,"into":"json","content":%q},
{"id":"runtime","type":"service","unit":"docker.service","state":"running","reload-on":["trust"]}
]}`, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
var commands []string
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
recordingServices(&commands), nil, nil); err != nil {
t.Fatal(err)
}
joined := strings.Join(commands, "\n")
if !strings.Contains(joined, "systemctl reload docker.service") {
t.Errorf("the runtime was not reloaded; commands were %v", commands)
}
if strings.Contains(joined, "stop docker.service") || strings.Contains(joined, "restart docker.service") {
t.Errorf("the runtime was stopped, which stops every container on the machine; commands were %v", commands)
}
}
// somethingElse is a declaration that no longer holds the file: only an unrelated directory.
func somethingElse(t *testing.T) *declaration.Declaration {
t.Helper()
return parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"other","type":"directory","path":%q}
]}`, filepath.Join(t.TempDir(), "other")))
}
func TestAListIsAddedToNeverReplaced(t *testing.T) {
// The predecessor's own trusted registries are kept; the mesh adds its own and, undeclared,
// takes back only what it added (novox/hq ADR 0102).
path := filepath.Join(t.TempDir(), "daemon.json")
_ = os.WriteFile(path, []byte(`{"insecure-registries":["192.0.2.7:5000","10.42.0.9:5000"]}`), 0o644)
// First the mesh's own member alone, so there is a record of this file.
first := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
_, state, err := Apply(context.Background(), archHost(t), first, store.State{}, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
// Now 10.42.0.9 is declared too, and was already the machine's: it is never the mesh's to remove.
d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000","10.42.0.9:5000"]}`))
_, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if got := fmt.Sprint(readObject(t, path)["insecure-registries"]); got != "[192.0.2.7:5000 10.42.0.9:5000 10.42.0.1:5000]" {
t.Fatalf("the list after writing into it: %s", got)
}
rec, _ := state.Find("networking.registry-trust")
if added := rec.Into.Added["insecure-registries"]; len(added) != 1 || canonical(added[0]) != `"10.42.0.1:5000"` {
t.Errorf("recorded as added: %s", added)
}
// The predecessor adds a member of its own: not the mesh's drift.
o := readObject(t, path)
o["insecure-registries"] = append(o["insecure-registries"].([]any), "198.51.100.3:5000")
raw, _ := json.Marshal(o)
_ = os.WriteFile(path, raw, 0o644)
report, state, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if got := report.Outcomes[0].Action; got != "unchanged" {
t.Errorf("a member the machine added was taken for drift: %q", got)
}
// Somebody takes the mesh's member out: that is drift, and it is put back.
o = readObject(t, path)
o["insecure-registries"] = []any{"192.0.2.7:5000", "10.42.0.9:5000", "198.51.100.3:5000"}
raw, _ = json.Marshal(o)
_ = os.WriteFile(path, raw, 0o644)
report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if got := report.Outcomes[0].Action; got != "corrected" {
t.Errorf("the mesh's member removed by hand was %q", got)
}
// Undeclared: only the member the mesh added goes.
if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil {
t.Fatal(err)
}
if got := fmt.Sprint(readObject(t, path)["insecure-registries"]); got != "[192.0.2.7:5000 10.42.0.9:5000 198.51.100.3:5000]" {
t.Errorf("undeclaring took more than the mesh added: %s", got)
}
}
func TestAListTheMeshCreatedGoesWhenEmptied(t *testing.T) {
path := filepath.Join(t.TempDir(), "daemon.json")
_ = os.WriteFile(path, []byte(`{"data-root":"/srv/docker"}`), 0o644)
d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`))
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil {
t.Fatal(err)
}
if o := readObject(t, path); fmt.Sprint(o) != "map[data-root:/srv/docker]" {
t.Errorf("the key the mesh created was not removed: %v", o)
}
}
func TestAHoldFromAWholeFileDoesNotKeepOutAnIntoWrite(t *testing.T) {
// Declared whole before, the runtime's file was held; declared into now, it is written into.
path := filepath.Join(t.TempDir(), "daemon.json")
_ = os.WriteFile(path, []byte(machinesOwn), 0o644)
known := store.State{Held: []store.Held{{ID: "networking.registry-trust", Module: "networking",
Kind: "file", Target: path}}}
d := adopted(t, `{"taken":[],"untaken":{"networking":["networking.registry-trust"]}}`,
fmt.Sprintf(`{"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q}`,
path, `{"insecure-registries":["10.42.0.1:5000"]}`))
report, state := applyAdopted(t, d, known, &machine{}, t.TempDir())
if got := outcomeOf(report, "networking.registry-trust").Action; got != "updated" {
t.Errorf("the file was %q, not written into", got)
}
if len(state.Held) != 0 {
t.Errorf("the old hold outlived the into declaration: %+v", state.Held)
}
if fmt.Sprint(readObject(t, path)["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" {
t.Errorf("the mesh's member was not written in: %v", readObject(t, path))
}
}
func TestAWriteWithNoRecordOfItIsTheMeshsOwn(t *testing.T) {
// A host that wrote into the file and died before saving its state comes back with no record
// of it. What is there is then exactly what the mesh declares — and remembered as the
// machine's it would never be given back (novox/hq ADR 0102).
path := filepath.Join(t.TempDir(), "daemon.json")
_ = os.WriteFile(path, []byte(`{"data-root":"/srv/docker","insecure-registries":["192.0.2.7:5000"]}`), 0o644)
d := parse(t, intoDecl(t, path, `{"live-restore":true,"insecure-registries":["10.42.0.1:5000"]}`))
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil); err != nil {
t.Fatal(err)
}
// The crash: the state was never saved, so the next apply knows nothing of this file.
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
rec, _ := state.Find("networking.registry-trust")
if _, asTheMachines := rec.Into.Before["live-restore"]; asTheMachines {
t.Error("the mesh's own key was remembered as the machine's")
}
if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil {
t.Fatal(err)
}
o := readObject(t, path)
if _, still := o["live-restore"]; still {
t.Errorf("undeclaring left the mesh's key behind: %v", o)
}
if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" || o["data-root"] != "/srv/docker" {
t.Errorf("the machine did not get its file back: %v", o)
}
}
+138
View File
@@ -0,0 +1,138 @@
package apply
import (
"context"
"fmt"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/firewall"
"github.com/novox/mesh-host/internal/store"
)
// foundFirewall settles, before anything else in an apply, which firewall this node has — and on
// an adopted node that the mesh had converged, puts it back in force first (novox/hq ADR 0100).
//
// Only an adopted node asks. It is detected on every apply rather than remembered, so a firewall
// switched on after adoption is spoken to from the next reconcile; what is remembered is what was
// found first, and whether the mesh retired it. An unsupported firewall refuses the whole
// declaration: the mesh could neither open what it needs through it nor say what it would close.
func foundFirewall(ctx context.Context, d *declaration.Declaration, known *store.State, run Runner,
log func(string)) (firewall.Kind, error) {
if d.Adoption == nil {
return "", nil
}
rec := known.Firewall
if rec != nil && rec.DisabledByMesh && rec.Kind == string(firewall.UFW) {
// Returned to adopted: the found firewall is enabled again before the openings are
// converged through it, and the derived filter is gone with this declaration.
if err := firewall.Enable(ctx, run); err != nil {
return "", err
}
rec.DisabledByMesh = false
rec.Forward = nil
log(" enabled ufw again: this node is adopted, and the firewall found on it is in force")
}
kind, name, err := firewall.Detect(ctx, run)
if err != nil {
return "", err
}
if kind == firewall.Unsupported {
return "", fmt.Errorf(
"this machine is filtered by %s, and no host speaks that firewall yet. An adopted node "+
"keeps the firewall it was found with, so the mesh could neither open what it needs "+
"through it nor say what it would close; this declaration is refused whole", name)
}
if rec == nil {
rec = &store.FoundFirewall{Kind: string(kind), WasActive: kind == firewall.UFW,
FoundAt: time.Now().UTC()}
} else {
rec.Kind = string(kind)
rec.WasActive = rec.WasActive || kind == firewall.UFW
}
known.Firewall = rec
return kind, nil
}
// retireFirewall disables the found firewall once a converged declaration has applied cleanly,
// which is when the mesh's derived filter has taken its place. Disabled, never flushed: its
// configuration stays on disk for a return to adopted, and the container runtime's rules are not
// its to take.
//
// Only a declaration from the mesh converges a node. A carried bundle never says a node is adopted
// — it cannot — so its silence is not the controller's word that the node was converged, and an
// adopted node re-applying its bundle keeps the firewall it was found with.
func retireFirewall(ctx context.Context, d *declaration.Declaration, origin string, known *store.State,
run Runner, log func(string)) error {
rec := known.Firewall
if origin != store.OriginDeclared || d.Adoption != nil || rec == nil || rec.Kind != string(firewall.UFW) || !rec.WasActive ||
rec.DisabledByMesh {
return nil
}
// **Nothing is retired until what replaces it is in force** (novox/hq ADR 0100). The flip
// loads the mesh's derived filter in ufw's place; disabling ufw before that table is actually
// loaded — a filter module not assigned, or a unit that did not load — leaves the machine with
// no filter at all.
loaded, err := firewall.MeshTableLoaded(ctx, run)
if err != nil {
return err
}
if !loaded {
return fmt.Errorf("this node is converged and the mesh's own filter (table %s) is not loaded on "+
"this machine, so ufw was left in force: retiring it would leave the machine filtering "+
"nothing. Assign a filter module to this node, or return it to adopted", firewall.MeshTable)
}
if rec.Forward == nil {
// Recorded before ufw is touched: disabling it opens the forward policy, and a retry
// must know what it was (novox/hq ADR 0100).
rec.Forward = firewall.ForwardPolicies(ctx, run)
}
if err := firewall.Disable(ctx, run, rec.Forward); err != nil {
return err
}
rec.DisabledByMesh = true
log(" disabled ufw: this node is converged and filtered by the mesh; ufw's configuration is left on disk")
return nil
}
// applyOpening makes one opening true through the firewall found here.
func applyOpening(ctx context.Context, o *declaration.Opening, run Runner, kind firewall.Kind) (Outcome, error) {
out := begin(o)
switch kind {
case firewall.None:
out.Action = "unchanged"
out.Detail = "no firewall found; nothing filters this port"
return out, nil
case firewall.UFW:
done, err := firewall.Converge(ctx, run, o)
if err != nil {
return out, err
}
out.Action = done.Action
out.Detail = "through ufw, marked " + firewall.Mark(o)
if done.SatisfiedBy != "" {
// ufw would take a rule differing only in its comment for the same one, so the
// mesh's is not added beside it (novox/hq ADR 0103).
out.Detail = "satisfied by a rule found in ufw (" + done.SatisfiedBy +
"); the mesh added nothing and will remove nothing"
}
return out, nil
}
return out, fmt.Errorf("no firewall is known for this node, so %s cannot be opened", o.Target())
}
// removeOpening deletes the rules the mesh marked for an opening no longer declared, and nothing
// the machine had before.
func removeOpening(ctx context.Context, a store.Applied, run Runner, rec *store.FoundFirewall) (string, string, error) {
if rec == nil || rec.Kind != string(firewall.UFW) {
return "forgotten", "no firewall held a rule for it", nil
}
n, err := firewall.Remove(ctx, run, a.ID)
if err != nil {
return "", "", err
}
if n == 0 {
return "forgotten", "ufw held no rule marked for it", nil
}
return "removed", fmt.Sprintf("%d ufw rule(s) marked as the mesh's deleted", n), nil
}
+492
View File
@@ -0,0 +1,492 @@
package apply
import (
"context"
"errors"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force; converging the
// node retires it by disabling it, and returning the node to adopted enables it again.
type ufwMachine struct {
installed, active bool
rules []string
ruleset string
asked []string
// forward is iptables' forward policy when set; empty is a machine without iptables. failP
// is how many -P calls fail before one succeeds.
forward string
failP int
}
func (u *ufwMachine) iptables(args []string) (string, error) {
if len(args) == 3 && args[0] == "-P" {
if u.failP > 0 {
u.failP--
return "", errors.New("iptables: resource temporarily unavailable")
}
u.forward = args[2]
return "", nil
}
return "-P FORWARD " + u.forward + "\n-A FORWARD -j DOCKER-USER\n", nil
}
func (u *ufwMachine) run(_ context.Context, name string, args ...string) (string, error) {
u.asked = append(u.asked, name+" "+strings.Join(args, " "))
switch name {
case "nft":
return u.ruleset, nil
case "iptables":
if u.forward == "" {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
return u.iptables(args)
case "ufw":
if !u.installed {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
default:
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
switch args[0] {
case "status":
if u.active {
return "Status: active\n", nil
}
return "Status: inactive\n", nil
case "show":
out := "Added user rules (see 'ufw status' for running firewall):\n"
for _, r := range u.rules {
out += "ufw " + r + "\n"
}
return out, nil
case "--force":
u.active = true
return "", nil
case "disable":
u.active = false
if u.forward != "" {
u.forward = "ACCEPT" // as measured: ufw disable opens the forward policy
}
return "", nil
case "delete":
want := strings.Join(args[1:], " ")
for i, r := range u.rules {
if strings.ReplaceAll(r, "'", "") == want {
u.rules = append(u.rules[:i], u.rules[i+1:]...)
return "", nil
}
}
return "", errors.New("Could not delete non-existent rule")
default:
// Printed back the way it was given, with the comment quoted as ufw does.
line := strings.Join(args[:len(args)-1], " ") + " '" + args[len(args)-1] + "'"
u.rules = append(u.rules, line)
return "", nil
}
}
func (u *ufwMachine) index(prefix string) int {
for i, a := range u.asked {
if strings.HasPrefix(a, prefix) {
return i
}
}
return -1
}
const busOpening = `{"id":"adoption.opening-tcp-5671-incoming","type":"opening","port":5671,"protocol":"tcp","from":"everywhere","path":"incoming"}`
func withConf(dir string) string {
return `{"id":"x.conf","type":"file","path":"` + filepath.Join(dir, "x.conf") + `","content":"x\n"}`
}
func applyWith(t *testing.T, d *declaration.Declaration, known store.State, run Runner) (Report, store.State, error) {
t.Helper()
return ApplyKeeping(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil,
KeepIn(t.TempDir()))
}
func TestAnOpeningOnAMachineWithNoFirewallChangesNothing(t *testing.T) {
dir := t.TempDir()
u := &ufwMachine{}
report, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
o := outcomeOf(report, "adoption.opening-tcp-5671-incoming")
if o.Action != "unchanged" || !strings.Contains(o.Detail, "nothing filters this port") {
t.Errorf("an opening with no firewall: %+v", o)
}
if state.Firewall == nil || state.Firewall.Kind != "none" {
t.Errorf("the firewall found was not recorded: %+v", state.Firewall)
}
}
func TestAnUnsupportedFirewallRefusesTheWholeDeclaration(t *testing.T) {
dir := t.TempDir()
u := &ufwMachine{ruleset: "table inet filter {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t}\n}\n"}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err == nil || !strings.Contains(err.Error(), "no host speaks that firewall") {
t.Fatalf("an unsupported firewall was not refused: %v", err)
}
if _, statErr := os.Stat(filepath.Join(dir, "x.conf")); !errors.Is(statErr, os.ErrNotExist) {
t.Error("part of a refused declaration was applied")
}
if state.Firewall != nil {
t.Errorf("an unsupported firewall was recorded: %+v", state.Firewall)
}
}
func TestConvergingRetiresTheFoundFirewallAndReturningRestoresIt(t *testing.T) {
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
// Adopted: the opening goes through ufw.
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
if len(u.rules) != 2 || !u.active {
t.Fatalf("adopted: rules %v, active %v", u.rules, u.active)
}
if state.Firewall == nil || state.Firewall.Kind != "ufw" || !state.Firewall.WasActive {
t.Fatalf("adopted: firewall recorded as %+v", state.Firewall)
}
// Converged: the derived filter is loaded, the opening's rule goes, and only then is ufw
// disabled — never reset.
u.asked = nil
u.ruleset = "table inet mesh\n"
converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
_, state, err = applyWith(t, converged, state, u.run)
if err != nil {
t.Fatal(err)
}
if u.active || !state.Firewall.DisabledByMesh {
t.Fatalf("converged: ufw still active (%v) or not recorded as retired (%+v)", u.active, state.Firewall)
}
if len(u.rules) != 1 || u.rules[0] != "allow 22/tcp" {
t.Errorf("converged: the operator's rules were touched, or the mesh's left: %v", u.rules)
}
// What protected the adopted node goes last: after the derived filter applied and ufw was
// retired (novox/hq ADR 0103).
if del, dis := u.index("ufw delete"), u.index("ufw disable"); dis < 0 || del < dis {
t.Errorf("converged: the opening was removed before ufw was retired: %v", u.asked)
}
for _, a := range u.asked {
if strings.Contains(a, "reset") {
t.Errorf("converged: ufw was reset: %s", a)
}
}
// Converged again: nothing more to retire.
u.asked = nil
if _, state, err = applyWith(t, converged, state, u.run); err != nil {
t.Fatal(err)
}
if u.index("ufw") >= 0 {
t.Errorf("a converged node kept talking to a retired ufw: %v", u.asked)
}
// Returned to adopted: ufw is enabled before the opening is converged through it.
u.asked = nil
_, state, err = applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), state, u.run)
if err != nil {
t.Fatal(err)
}
if !u.active || state.Firewall.DisabledByMesh {
t.Fatalf("returned: ufw active %v, record %+v", u.active, state.Firewall)
}
if en, add := u.index("ufw --force enable"), u.index("ufw allow"); en < 0 || add < en {
t.Errorf("returned: ufw was not enabled before the opening was added: %v", u.asked)
}
if len(u.rules) != 2 {
t.Errorf("returned: the opening was not converged again: %v", u.rules)
}
}
func TestAConvergedNodeThatWasNeverAdoptedNeverAsksAboutAFirewall(t *testing.T) {
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true}
if _, _, err := applyWith(t, parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`), store.State{}, u.run); err != nil {
t.Fatal(err)
}
if len(u.asked) != 0 {
t.Errorf("a converged apply asked the machine about its firewall: %v", u.asked)
}
}
func TestAnOpeningOnAConvergedNodeIsRefused(t *testing.T) {
if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` + busOpening + `]}`)); err == nil {
t.Error("an opening was accepted on a node the declaration does not say is adopted")
}
}
func TestACarriedApplyOnAnAdoptedNodeLeavesItsFirewallInForce(t *testing.T) {
// The bundle, re-applied by the installer or the one-shot CLI, never says a node is adopted.
// That is not the controller converging it, so ufw must stay enabled (novox/hq ADR 0100).
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
u.asked = nil
carried := parse(t, `{"declaration":1,"resources":[`+withConf(filepath.Join(dir, "bundle"))+`]}`)
_, state, err = ApplyKeeping(context.Background(), archHost(t), carried, state, store.OriginCarried,
u.run, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if !u.active || state.Firewall.DisabledByMesh || u.index("ufw disable") >= 0 {
t.Fatalf("a carried apply retired the found firewall: active %v, record %+v, asked %v",
u.active, state.Firewall, u.asked)
}
}
func TestAFlipThatFailsKeepsTheGuardAndTheOpenings(t *testing.T) {
// Converging a node removes its openings and its guard only once everything else applied and
// the found firewall is retired. A flip that fails part-way keeps them, so the store is never
// left unguarded behind a filter that did not load (novox/hq ADR 0103).
dir := t.TempDir()
guard := filepath.Join(dir, "guard.nft")
guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}`
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+guardFile+","+withConf(dir)),
store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
// The derived filter cannot be written: its path is under a file.
blocked := filepath.Join(dir, "not-a-directory")
if err := os.WriteFile(blocked, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
filter := `{"id":"nftables.config","type":"file","path":"` + filepath.Join(blocked, "nftables.conf") + `","content":"table inet mesh {}\n"}`
converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`,`+filter+`]}`)
u.asked = nil
u.ruleset = "table inet mesh\n" // what the filter's unit loads once the file is written
_, state, err = applyWith(t, converged, state, u.run)
if err == nil {
t.Fatal("the failing flip reported success")
}
if !u.active || u.index("ufw disable") >= 0 {
t.Errorf("the found firewall was retired by a flip that failed: %v", u.asked)
}
if len(u.rules) != 2 || u.index("ufw delete") >= 0 {
t.Errorf("the opening was removed by a flip that failed: %v", u.rules)
}
if _, statErr := os.Stat(guard); statErr != nil {
t.Errorf("the guard was removed by a flip that failed: %v", statErr)
}
for _, id := range []string{"adoption.guard", "adoption.opening-tcp-5671-incoming"} {
if _, ok := state.Find(id); !ok {
t.Errorf("%s was forgotten, so the next flip would never remove it", id)
}
}
// Fixed, the next flip completes: filter, retire, and only then the guard and the openings.
if err := os.Remove(blocked); err != nil {
t.Fatal(err)
}
u.asked = nil
_, state, err = applyWith(t, converged, state, u.run)
if err != nil {
t.Fatal(err)
}
if u.active || len(u.rules) != 1 {
t.Errorf("the completed flip left ufw active %v, rules %v", u.active, u.rules)
}
if _, statErr := os.Stat(guard); !errors.Is(statErr, os.ErrNotExist) {
t.Errorf("the guard outlived the completed flip: %v", statErr)
}
if _, ok := state.Find("adoption.guard"); ok {
t.Error("the guard is still recorded after the completed flip")
}
}
func TestAnOpeningAFoundRuleAnswersIsReportedSatisfied(t *testing.T) {
// novox/hq ADR 0103: the mesh adds nothing beside a rule ufw would take for the same one.
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 5671/tcp"}}
report, _, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
o := outcomeOf(report, "adoption.opening-tcp-5671-incoming")
if o.Action != "unchanged" || !strings.Contains(o.Detail, "satisfied by a rule found in ufw (allow 5671/tcp)") {
t.Errorf("the opening was not reported satisfied: %+v", o)
}
if len(u.rules) != 2 || u.index("ufw allow") >= 0 {
t.Errorf("a rule was added beside the found one: %v", u.rules)
}
}
// Defends novox/hq ADR 0103: returned to adopted, the guard is up before the derived filter's
// orphans go, and stays up if removing them fails.
func TestReturningToAdoptedLoadsTheGuardBeforeRemovingTheFilter(t *testing.T) {
dir := t.TempDir()
guard := filepath.Join(dir, "guard.nft")
guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}`
for _, stopFails := range []bool{false, true} {
_ = os.Remove(guard)
guardUpAtStop := false
run := func(_ context.Context, name string, args ...string) (string, error) {
if name != "systemctl" {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
switch args[0] {
case "show":
return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\n", nil
case "stop":
_, err := os.Stat(guard)
guardUpAtStop = err == nil
if stopFails {
return "", errors.New("the filter would not stop")
}
}
return "", nil
}
converged := store.State{Resources: []store.Applied{
{ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run)
if !guardUpAtStop {
t.Errorf("stop fails %v: the derived filter was stopped before the guard was written", stopFails)
}
if stopFails {
if err == nil {
t.Error("a failed removal was not reported")
}
if _, statErr := os.Stat(guard); statErr != nil {
t.Error("the guard is not up after the filter's removal failed")
}
if _, ok := state.Find("adoption.guard"); !ok {
t.Error("the guard applied before the failure was not recorded")
}
if _, still := state.Find("nftables.load"); !still {
t.Error("the filter that would not stop was forgotten, so nothing would stop it later")
}
} else if err != nil {
t.Fatal(err)
}
}
}
func TestAStaleOpeningOnAnAdoptedNodeIsRemovedAsAnyOrphan(t *testing.T) {
// Only the flip defers the adoption's own orphans; an adopted node drops a stale opening at
// once, before what replaces it is applied.
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
u.asked = nil
other := `{"id":"adoption.opening-tcp-5000-incoming","type":"opening","port":5000,"protocol":"tcp","from":"everywhere","path":"incoming"}`
if _, _, err = applyWith(t, adopted(t, `{"taken":[]}`, other+","+withConf(dir)), state, u.run); err != nil {
t.Fatal(err)
}
if del, add := u.index("ufw delete"), u.index("ufw allow"); del < 0 || add < 0 || del > add {
t.Errorf("the stale opening was not removed before the new one was added: %v", u.asked)
}
}
func TestARetiredFirewallRetriedStillPutsBackTheForwardPolicy(t *testing.T) {
// The forward policy is recorded before ufw is disabled, so a retirement that failed after
// the disable restores what the machine had, not what the disable left (novox/hq ADR 0100).
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, forward: "DROP", failP: 1, ruleset: "table inet mesh\n"}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
if _, state, err = applyWith(t, converged, state, u.run); err == nil {
t.Fatal("the failed restore was not reported")
}
if u.active || u.forward != "ACCEPT" || state.Firewall.Forward["iptables"] != "DROP" {
t.Fatalf("after the failed attempt: active %v, forward %s, recorded %+v", u.active, u.forward, state.Firewall)
}
if _, state, err = applyWith(t, converged, state, u.run); err != nil {
t.Fatal(err)
}
if u.forward != "DROP" || !state.Firewall.DisabledByMesh {
t.Errorf("the retry did not put the forward policy back: %s, %+v", u.forward, state.Firewall)
}
}
func TestAGuardThatFailsLeavesTheDerivedFilterInForce(t *testing.T) {
// Returning to adopted: if the guard cannot be raised, the filter it replaces must not be
// stopped — its stop deletes the mesh's table, and the node would have neither (novox/hq ADR 0103).
dir := t.TempDir()
blocked := filepath.Join(dir, "not-a-directory")
if err := os.WriteFile(blocked, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
guardFile := `{"id":"adoption.guard","type":"file","path":"` + filepath.Join(blocked, "guard.nft") +
`","content":"table inet mesh_guard {}\n"}`
stopped := false
run := func(_ context.Context, name string, args ...string) (string, error) {
if name != "systemctl" {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
if args[0] == "show" {
return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\n", nil
}
if args[0] == "stop" {
stopped = true
}
return "", nil
}
converged := store.State{Resources: []store.Applied{
{ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run)
if err == nil {
t.Fatal("a guard that could not be written reported success")
}
if stopped {
t.Error("the derived filter was stopped though the guard is not up")
}
if _, gone := state.Find("nftables.load"); !gone {
t.Error("the filter was forgotten, so nothing would ever stop it")
}
}
func TestUfwIsNotRetiredUntilTheMeshsOwnFilterIsLoaded(t *testing.T) {
// The flip retires the found firewall because the mesh's derived filter takes its place. If
// that table is not loaded, retiring would leave the machine filtering nothing (novox/hq ADR 0100).
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
_, state, err = applyWith(t, converged, state, u.run)
if err == nil || !strings.Contains(err.Error(), "table inet mesh") {
t.Fatalf("ufw was retired with nothing in its place: %v", err)
}
if !u.active || state.Firewall.DisabledByMesh {
t.Errorf("ufw was disabled: active %v, %+v", u.active, state.Firewall)
}
// Once the table is loaded, the same declaration retires it.
u.ruleset = "table inet mesh\n"
if _, state, err = applyWith(t, converged, state, u.run); err != nil {
t.Fatal(err)
}
if u.active || !state.Firewall.DisabledByMesh {
t.Errorf("ufw was not retired once the mesh's filter was loaded: active %v, %+v", u.active, state.Firewall)
}
}
+3 -2
View File
@@ -32,8 +32,9 @@ import (
// owners end up disagreeing about one path.
const daemonRoot = "/var/lib/mesh/daemons"
// unitDir is where the mesh writes the units it owns.
const unitDir = "/etc/systemd/system"
// unitDir is where the mesh writes the units it owns. A variable only so a test can point it at a
// directory of its own.
var unitDir = "/etc/systemd/system"
func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
changed map[string]bool, previous store.Applied) (Outcome, error) {
+13 -1
View File
@@ -86,7 +86,11 @@ func NewScheduler(clock Clock, run Runner, log func(string)) *Scheduler {
// A job whose declaration is unchanged keeps its place in the cadence — its next due time and
// whether a run is in flight — so an ordinary reconcile every few minutes does not keep resetting
// the clock out from under a schedule and prevent it ever firing.
func (s *Scheduler) Sync(d *declaration.Declaration) {
// held is the ids this node holds as found — what an adopted node keeps until its module is taken
// (novox/hq ADR 0100). A step of a module not yet taken is not armed: run on its cadence it would
// work on the predecessor's data, under the predecessor's service, which is the one thing an
// adopted node must not do. Nil on a converged node, where nothing is held.
func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) {
s.mu.Lock()
defer s.mu.Unlock()
@@ -96,6 +100,14 @@ func (s *Scheduler) Sync(d *declaration.Declaration) {
if !ok || c.Schedule == "" {
continue
}
if module, untaken := d.Adoption.UntakenModuleOf(c.Identity()); untaken || held[c.Identity()] {
if module == "" {
module = "its module"
}
s.log(fmt.Sprintf("scheduled step %s: not armed while %s is held as found on this node",
c.Identity(), module))
continue
}
cron, err := declaration.ParseCron(c.Schedule)
if err != nil {
// The declaration parser already refused a malformed cron before this runs, so a
+50 -5
View File
@@ -244,7 +244,7 @@ func TestAScheduledStepRunsWhenDueAndNotBefore(t *testing.T) {
d := &declaration.Declaration{Version: 1, Resources: []declaration.Resource{
scheduledContainer(t, "* * * * *"), // every minute; next due 12:01:00
}}
s.Sync(d)
s.Sync(d, nil)
// Not yet due: 12:00:45 is before 12:01:00, so nothing runs.
s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 0, 45, 0, time.UTC))
@@ -306,7 +306,7 @@ func TestAFailedRunIsRecordedAndDoesNotFailAnything(t *testing.T) {
s := NewScheduler(clock, run, log)
s.Sync(&declaration.Declaration{Version: 1, Resources: []declaration.Resource{
scheduledContainer(t, "* * * * *"),
}})
}}, nil)
// Fire a run that exits non-zero. Advance returns nothing — there is no error to fail an apply,
// because the run happens outside any apply and outside the store.
@@ -371,7 +371,7 @@ func TestASlowRunSkipsTheNextDueRunRatherThanStacking(t *testing.T) {
s := NewScheduler(clock, run, log)
s.Sync(&declaration.Declaration{Version: 1, Resources: []declaration.Resource{
scheduledContainer(t, "* * * * *"), // every minute
}})
}}, nil)
// First occurrence: 12:01 is due — starts a run that blocks in the runner.
s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 1, 5, 0, time.UTC))
@@ -414,7 +414,7 @@ func TestSyncForgetsAScheduleTheDeclarationNoLongerNames(t *testing.T) {
s.Sync(&declaration.Declaration{Version: 1, Resources: []declaration.Resource{
scheduledContainer(t, "* * * * *"),
}})
}}, nil)
s.mu.Lock()
have := len(s.jobs)
s.mu.Unlock()
@@ -427,10 +427,55 @@ func TestSyncForgetsAScheduleTheDeclarationNoLongerNames(t *testing.T) {
parseTrusted(t, `{"declaration":1,"resources":[
{"id":"web","type":"container","name":"web","image":"`+pinned+`"}
]}`).Resources[0],
}})
}}, nil)
s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 5, 5, 0, time.UTC))
s.Wait()
if n := rec.fireCount(); n != 0 {
t.Errorf("a schedule the declaration no longer names still fired (%d run(s))", n)
}
}
// Defends novox/hq ADR 0103: a scheduled step of a module not yet taken is not armed — run on its
// cadence it would work on the predecessor's data.
func TestAScheduledStepOfAnUntakenModuleIsNotArmed(t *testing.T) {
clock := &fixedClock{now: time.Date(2026, 9, 7, 12, 0, 30, 0, time.UTC)}
rec := &recordingRun{}
var said []string
s := NewScheduler(clock, rec.run, func(line string) { said = append(said, line) })
d := &declaration.Declaration{Version: 1,
Adoption: &declaration.Adoption{Untaken: map[string][]string{"backups": {"sync"}}},
Resources: []declaration.Resource{
scheduledContainer(t, "* * * * *"),
}}
s.Sync(d, nil)
s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 1, 5, 0, time.UTC))
s.Wait()
if rec.fireCount() != 0 {
t.Errorf("a held module's scheduled step ran %d time(s)", rec.fireCount())
}
if len(said) == 0 || !strings.Contains(said[0], "held as found") {
t.Errorf("nothing said why the step was not armed: %v", said)
}
// Held by id — a step whose own container was found on the machine.
s2 := NewScheduler(clock, rec.run, nil)
s2.Sync(&declaration.Declaration{Version: 1, Resources: []declaration.Resource{
scheduledContainer(t, "* * * * *"),
}}, map[string]bool{"sync": true})
s2.Advance(context.Background(), time.Date(2026, 9, 7, 12, 1, 5, 0, time.UTC))
s2.Wait()
if rec.fireCount() != 0 {
t.Errorf("a held scheduled step ran %d time(s)", rec.fireCount())
}
// Taken: the same step is armed and runs.
taken := &declaration.Declaration{Version: 1, Adoption: &declaration.Adoption{Taken: []string{"backups"}},
Resources: []declaration.Resource{scheduledContainer(t, "* * * * *")}}
s.Sync(taken, nil)
s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 2, 5, 0, time.UTC))
s.Wait()
if rec.fireCount() == 0 {
t.Error("a taken module's scheduled step never ran")
}
}
+202
View File
@@ -0,0 +1,202 @@
package bootstrap
import (
"bytes"
"context"
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"github.com/novox/mesh-host/internal/declaration"
)
// What an adopted genesis changes about the foundation (novox/hq ADR 0100).
//
// **The firewall found on the machine stays in force.** The foundation's own filter drops by
// default, and every base chain at a hook runs; an accept ends only its own chain and a drop in any
// is final — so loading it would close whatever the machine serves. On an adopted machine it is
// not loaded. Its duty, the store never reachable from outside, passes to the mesh's guard: a table
// of the mesh's own that only refuses, and only the foundation's own ports, which genesis has just
// checked free — so it cannot close anything the machine serves.
// The guard, as the controller declares it: the same ids, paths and text, so the first push
// finds it already there and takes it over unchanged.
const (
guardID = declaration.AdoptionPrefix + "guard"
guardUnitID = declaration.AdoptionPrefix + "guard-unit"
guardRunningID = declaration.AdoptionPrefix + "guard-running"
guardPath = "/etc/mesh/guard.nft"
guardUnit = "mesh-guard.service"
guardUnitPath = "/etc/systemd/system/" + guardUnit
)
// AsGuard renders the mesh's refusal-only table for the given machine ports. It passes everything
// by default; it refuses the ports except from the machine itself — its loopback and the container
// runtime's own networks — and from the private network, known by the interface a packet arrives
// on and never by its source address; at prerouting, ahead of the runtime's destination
// translation, in the inet family so both address families. It matches only packets addressed to
// this machine: what the machine routes for others is never its business (novox/hq ADR 0103).
//
// Character for character the controller's (mesh-controller internal/catalogue AsGuard); a test
// on each side holds its copy to the same golden text.
func AsGuard(ports []int) string {
sorted := append([]int{}, ports...)
sort.Ints(sorted)
listed := make([]string, len(sorted))
for i, p := range sorted {
listed[i] = strconv.Itoa(p)
}
var b strings.Builder
b.WriteString("table inet mesh_guard {}\n")
b.WriteString("delete table inet mesh_guard\n")
b.WriteString("table inet mesh_guard {\n")
b.WriteString("\tchain prerouting {\n")
b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n")
fmt.Fprintf(&b, "\t\tfib daddr type local iifname != \"lo\" iifname != \"docker0\" iifname != \"br-*\" "+
"iifname != \"mesh0\" tcp dport { %s } drop\n", strings.Join(listed, ", "))
b.WriteString("\t}\n")
b.WriteString("}\n")
return b.String()
}
// guardUnitText is the unit that loads the guard. Stopping it deletes only its own table — never a
// flush, which would take the container runtime's rules and the found firewall with it.
func guardUnitText() string {
return "[Unit]\n" +
"Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" +
"DefaultDependencies=no\n" +
"Wants=network-pre.target\n" +
"Before=network-pre.target shutdown.target\n" +
"Conflicts=shutdown.target\n" +
"\n" +
"[Service]\n" +
"Type=oneshot\n" +
"RemainAfterExit=yes\n" +
"ExecStart=nft -f " + guardPath + "\n" +
"ExecReload=nft -f " + guardPath + "\n" +
"ExecStop=nft delete table inet mesh_guard\n" +
"\n" +
"[Install]\n" +
"WantedBy=multi-user.target\n"
}
// guardResources are the guard as three resources of kinds the host already has.
func guardResources(ports []int) []map[string]any {
return []map[string]any{
{"id": guardID, "type": "file", "path": guardPath, "content": AsGuard(ports), "mode": "0644"},
{"id": guardUnitID, "type": "file", "path": guardUnitPath, "content": guardUnitText(), "mode": "0644"},
// A changed table is reloaded — the unit's ExecReload loads it in one transaction, so the
// ports are never unguarded — and only a changed unit restarts it. As the controller
// declares it, so the first push finds nothing different.
{"id": guardRunningID, "type": "service", "unit": guardUnit, "state": "running",
"boot": "enabled", "reload-on": []any{guardID}, "restart-on": []any{guardUnitID}},
}
}
// guardAfter is where the guard goes: once the container runtime runs, before anything publishes
// a port.
const guardAfter = "container-runtime-running"
// AdoptedRewrite says what RewriteAdopted did.
type AdoptedRewrite struct {
Removed []string
Guarded []int
}
// RewriteAdopted makes the produced bundle one for an adopted machine: the foundation's own filter
// taken out, and the mesh's guard put in its place, guarding on this node the store's port, the
// broker's management port and the broker's plaintext port. The last is published on every
// interface and the foundation's filter admits it from the private network only, so a found
// firewall that filters only incoming traffic would leave it reachable from anywhere (novox/hq ADR
// 0103). Every one is a port of a module genesis takes. The nftables package stays: the guard is loaded with it, and
// installing a package loads no table. Openings are not the bundle's — the first push declares
// them, once there is a controller to derive them.
func RewriteAdopted(r *Rewritten, p FoundationPorts) (AdoptedRewrite, error) {
var out AdoptedRewrite
p = p.orDefaults()
bundle := r.Bundle
var err error
for _, id := range []string{"base-filter-loaded", "base-filter"} {
if !r.declares(id) {
continue
}
if bundle, err = removeResource(bundle, id); err != nil {
return out, err
}
out.Removed = append(out.Removed, id)
}
out.Guarded = []int{p.Store, p.Management, p.AMQP}
var text bytes.Buffer
text.WriteString(",\n // The mesh's guard (novox/hq ADR 0100): this machine is adopted, so its own firewall\n" +
" // stays in force and the foundation's filter is not loaded. The guard only refuses: the\n" +
" // store's port and the broker's management and plaintext ports, except from the machine and\n" +
" // the private network.")
for _, res := range guardResources(out.Guarded) {
var one bytes.Buffer
enc := json.NewEncoder(&one)
enc.SetEscapeHTML(false)
if err := enc.Encode(res); err != nil {
return out, err
}
text.WriteString("\n ")
text.Write(bytes.TrimSpace(one.Bytes()))
text.WriteString(",")
}
insert := bytes.TrimSuffix(text.Bytes(), []byte(","))
_, _, to, err := resourceAt(bundle, guardAfter)
if err != nil {
return out, fmt.Errorf("the guard goes after %q, and %w", guardAfter, err)
}
rest := bundle[to:]
joined := make([]byte, 0, len(bundle)+len(insert))
joined = append(joined, bundle[:to]...)
joined = append(joined, insert...)
// What followed the resource — its own comma, or the end of the list — now follows the guard.
if trimmed := bytes.TrimLeft(rest, " \t\r\n"); len(trimmed) > 0 && trimmed[0] != ',' && trimmed[0] != ']' {
return out, fmt.Errorf("the bundle does not separate %q from what follows it the way a list does", guardAfter)
}
joined = append(joined, rest...)
parsed, err := declaration.ParseFileTrusted(joined)
if err != nil {
return out, fmt.Errorf("the bundle stopped being a declaration once it was made an adopted one, which is this installer's fault: %w", err)
}
r.Bundle, r.Declaration, r.Resources = joined, parsed, len(parsed.Resources)
return out, nil
}
// declares is whether the produced bundle names a resource.
func (r Rewritten) declares(id string) bool {
for _, res := range r.Declaration.Resources {
if res.Identity() == id {
return true
}
}
return false
}
// genesisTakes are the modules an adopted genesis takes as it installs them: the foundation's and
// the mesh's own, whose names genesis checked free, so taking them replaces nothing a predecessor
// ran. The private network is not among them — it rewrites the machine's hosts file and the
// container runtime's configuration whole — and neither is anything the operator installs later.
var genesisTakes = map[string]bool{
RegistryModule: true, ControlPlaneModule: true, BuilderModule: true,
"postgres": true, "lavinmq": true, "mesh-vault": true, "mesh-catalog": true,
}
// takeIfAdopted takes one of genesis's own modules on an adopted node, once it is assigned and
// before the push that raises it.
func takeIfAdopted(ctx context.Context, o Options, control controlPlane, module string, say func(string)) error {
if !o.Adopted || !genesisTakes[module] {
return nil
}
if _, err := control.tell(ctx, "take", o.Node, module); err != nil {
return err
}
say(" taken " + module + " on " + o.Node + " — the mesh's own, its name checked free")
return nil
}
+233
View File
@@ -0,0 +1,233 @@
package bootstrap
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/declaration"
)
// Defends novox/hq ADR 0100: an adopted genesis loads no table that drops by default or holds an
// accept; the mesh guards its own ports in a table that only refuses; and genesis takes the mesh's
// own modules as it installs them, and nothing else.
// The same golden text the controller's test holds its AsGuard to.
const goldenGuard = `table inet mesh_guard {}
delete table inet mesh_guard
table inet mesh_guard {
chain prerouting {
type filter hook prerouting priority raw; policy accept;
fib daddr type local iifname != "lo" iifname != "docker0" iifname != "br-*" iifname != "mesh0" tcp dport { 5432, 15672 } drop
}
}
`
func TestTheGuardIsExactlyThisTable(t *testing.T) {
if got := AsGuard([]int{15672, 5432}); got != goldenGuard {
t.Fatalf("the guard changed:\n%s", got)
}
}
// The same golden unit the controller's test holds its guard unit to. It is loaded before the
// network is up, so it carries no default dependencies, and it is stopped only at shutdown.
const goldenGuardUnit = `[Unit]
Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)
DefaultDependencies=no
Wants=network-pre.target
Before=network-pre.target shutdown.target
Conflicts=shutdown.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=nft -f /etc/mesh/guard.nft
ExecReload=nft -f /etc/mesh/guard.nft
ExecStop=nft delete table inet mesh_guard
[Install]
WantedBy=multi-user.target
`
func TestTheGuardUnitIsExactlyThisUnit(t *testing.T) {
if got := guardUnitText(); got != goldenGuardUnit {
t.Fatalf("the guard's unit changed:\n%s", got)
}
}
func TestTheGuardRefusesOnlyWhatIsAddressedToThisMachine(t *testing.T) {
// A machine that routes for others — a predecessor's private-network hub — must not have a
// packet for another machine's database port refused (novox/hq ADR 0103).
for _, line := range strings.Split(AsGuard([]int{5432}), "\n") {
if strings.Contains(line, " drop") && !strings.HasPrefix(strings.TrimSpace(line), "fib daddr type local ") {
t.Errorf("a refusal matches packets not addressed to this machine: %q", line)
}
}
}
func TestAnAdoptedBundleLoadsNoDroppingTableAndExactlyTheGuard(t *testing.T) {
r := producedBundle(t)
p := FoundationPorts{Store: 5433, Management: 15673, AMQP: 5773}
if _, err := RewritePorts(&r, p, ""); err != nil {
t.Fatal(err)
}
got, err := RewriteAdopted(&r, p)
if err != nil {
t.Fatal(err)
}
if strings.Join(got.Removed, ",") != "base-filter-loaded,base-filter" {
t.Errorf("removed %v", got.Removed)
}
at := map[string]int{}
var guards []*declaration.File
for i, res := range r.Declaration.Resources {
at[res.Identity()] = i
if f, ok := res.(*declaration.File); ok {
if strings.Contains(f.Content, "policy drop") || strings.Contains(f.Content, " accept\n") &&
!strings.Contains(f.Content, "policy accept") {
t.Errorf("%s loads a table that drops or accepts: %q", f.ID, f.Content)
}
if f.Path == guardPath {
guards = append(guards, f)
}
}
if s, ok := res.(*declaration.Service); ok && s.Unit == "nftables.service" {
t.Errorf("the foundation's filter is still loaded by %s", s.ID)
}
}
if len(guards) != 1 {
t.Fatalf("%d guard table(s)", len(guards))
}
// The store's, the broker's plaintext and its management port: each one the filter admits
// from the private network only (novox/hq ADR 0103).
if !strings.Contains(guards[0].Content, "tcp dport { 5433, 5773, 15673 } drop") {
t.Errorf("the guard does not refuse this node's ports: %s", guards[0].Content)
}
if strings.Count(guards[0].Content, "accept") != 1 || !strings.Contains(guards[0].Content, "policy accept") {
t.Errorf("the guard holds an accept of its own: %s", guards[0].Content)
}
for _, id := range []string{guardID, guardUnitID, guardRunningID} {
if _, ok := at[id]; !ok {
t.Errorf("the bundle has no %s", id)
}
}
if !(at["container-runtime-running"] < at[guardID] && at[guardRunningID] < at["store"]) {
t.Errorf("the guard is not between the runtime and the store: %v", at)
}
if _, kept := at["base-filter-package"]; !kept {
t.Error("nft, which loads the guard, is no longer installed")
}
unit := r.Declaration.Resources[at[guardRunningID]].(*declaration.Service)
// A changed table is reloaded, never restarted: a restart deletes the table before loading
// it again, leaving the ports unguarded in between.
if unit.Unit != guardUnit || unit.State != "running" || strings.Join(unit.RestartOn, ",") != guardUnitID ||
strings.Join(unit.ReloadOn, ",") != guardID {
t.Errorf("the guard's service: %+v", unit)
}
stop := r.Declaration.Resources[at[guardUnitID]].(*declaration.File).Content
if !strings.Contains(stop, "ExecStop=nft delete table inet mesh_guard") || strings.Contains(stop, "flush") {
t.Errorf("stopping the guard does not delete only its own table: %s", stop)
}
}
func TestAConvergedBundleIsNotMadeAnAdoptedOne(t *testing.T) {
// The converged genesis keeps the foundation's filter, byte for byte (novox/hq ADR 0088).
r := producedBundle(t)
for _, res := range r.Declaration.Resources {
if strings.HasPrefix(res.Identity(), declaration.AdoptionPrefix) {
t.Errorf("a converged bundle carries %s", res.Identity())
}
}
if !r.declares("base-filter-loaded") {
t.Error("a converged bundle lost its filter")
}
}
func TestAnAdoptedGenesisTakesTheMeshsOwnModulesBeforePushingThem(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
for _, c := range []struct {
module string
takes bool
}{{RegistryModule, true}, {ControlPlaneModule, true}, {BuilderModule, true}, {"gitea", false}} {
rec := &controlRecorder{settings: map[string]string{}}
control := controlPlane{container: "temp-mesh-controller", run: rec.run, timeout: time.Second}
o := Options{Node: "anchor", Adopted: true, Wait: time.Second}
if _, err := installModule(context.Background(), o, control, c.module, []byte(`{}`), quietly); err != nil {
t.Fatal(err)
}
assign, take, push := rec.index("assign anchor "+c.module), rec.index("take anchor "+c.module), rec.index("push anchor")
if !c.takes {
if take >= 0 {
t.Errorf("%s was taken at genesis", c.module)
}
continue
}
if !(assign >= 0 && assign < take && take < push) {
t.Errorf("%s: assign %d, take %d, push %d: %v", c.module, assign, take, push, rec.told)
}
}
}
func TestAConvergedGenesisTakesNothing(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
rec := &controlRecorder{settings: map[string]string{}}
control := controlPlane{container: "temp-mesh-controller", run: rec.run, timeout: time.Second}
if _, err := installModule(context.Background(), Options{Node: "anchor", Wait: time.Second}, control,
RegistryModule, []byte(`{}`), quietly); err != nil {
t.Fatal(err)
}
if rec.index("take") >= 0 {
t.Errorf("a converged genesis took a module: %v", rec.told)
}
}
func TestAnAdoptedGenesisOpensTheRegistryFromAnywhere(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
rec := &controlRecorder{settings: map[string]string{}}
control := controlPlane{container: "temp-mesh-controller", run: rec.run, timeout: time.Second}
o := Options{Node: "anchor", Adopted: true, Ports: FoundationPorts{Registry: 5100}, Wait: time.Second}
if _, err := installModule(context.Background(), o, control, RegistryModule, []byte(`{}`), quietly); err != nil {
t.Fatal(err)
}
if got := rec.settings["distribution-settings.json"]; got != `{"expose":{"5000":"anywhere"},"ports":{"5000":5100}}` {
t.Errorf("the registry was told %s", got)
}
}
func TestAnAdoptedGenesisChoosesTheFilterAndLoadsNone(t *testing.T) {
rec := &controlRecorder{settings: map[string]string{}}
control := controlPlane{container: "mesh-controller", run: rec.run, timeout: time.Second}
o := Options{Node: "anchor", Adopted: true, Answers: map[string]string{"packet-filter": "nftables"}}
filter, err := ChooseAndInstallFilter(context.Background(), o, control, quietly)
if err != nil || filter != "nftables" {
t.Fatalf("%q %v", filter, err)
}
if len(rec.told) != 0 {
t.Errorf("an adopted genesis installed a filter: %v", rec.told)
}
}
func TestAnAdoptedNodeIsRecordedAdopted(t *testing.T) {
stop := errors.New("stop here")
runtime := &asked{answer: func(name string, args []string) (string, error) {
joined := strings.Join(args, " ")
switch {
case strings.Contains(joined, "node list"):
return "", nil
case strings.Contains(joined, "node add"):
return "", nil
}
return "", fmt.Errorf("%w: %s %v", stop, name, args)
}}
_, _ = Enrol(context.Background(), Options{
Node: "anchor", Adopted: true, State: t.TempDir() + "/state.json", Timeout: time.Second,
Host: "/usr/local/bin/mesh-host", HostInBackground: true,
}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
func(string) {})
if !runtime.ran("node add anchor --adopted") {
t.Errorf("the node was not added adopted: %v", runtime.commands)
}
}
+8 -2
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"github.com/novox/mesh-host/internal/apply"
@@ -46,8 +47,13 @@ func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declarati
return apply.Report{}, err
}
report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried, run,
func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed)
// The bundle writes over whatever the machine has at the paths the foundation needs — a
// distribution's own /etc/nftables.conf among them — so it keeps the original of each file it
// has no record of, beside the node's state, exactly as a declaration from the mesh does
// (novox/hq ADR 0100).
report, updated, applyErr := apply.ApplyKeeping(ctx, sys, d, known, store.OriginCarried, run,
func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed,
apply.KeepIn(filepath.Dir(o.State)))
// Saved whichever way it went, for the reason `mesh-host` gives: what was applied before a
// failure is on the machine either way, and a host that did not record it would believe it
+38
View File
@@ -3,8 +3,13 @@ package bootstrap
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/system"
)
// `mesh-host` is built for one operating system and pins it at link time. An installer run by hand
@@ -89,3 +94,36 @@ func TestASealedFileInAFoundationIsRefusedWithAReason(t *testing.T) {
t.Errorf("the refusal does not say why there is no key: %v", err)
}
}
// Defends novox/hq ADR 0100: the carried bundle keeps the original of a file it writes over that
// the host has no record of — the distribution's own ruleset, say.
func TestTheBundleKeepsTheOriginalOfWhatItWritesOver(t *testing.T) {
dir := t.TempDir()
conf := filepath.Join(dir, "nftables.conf")
if err := os.WriteFile(conf, []byte("# the distribution's own\n"), 0o644); err != nil {
t.Fatal(err)
}
d, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[
{"id":"base-filter","type":"file","path":"` + conf + `","content":"table inet mesh {}\n"}]}`))
if err != nil {
t.Fatal(err)
}
sys, err := system.For("arch")
if err != nil {
t.Fatal(err)
}
o := Options{State: filepath.Join(dir, "state.json")}
report, err := ApplyBundle(context.Background(), o, sys, d, nil, quietly)
if err != nil {
t.Fatal(err)
}
detail := report.Outcomes[0].Detail
at := strings.Index(detail, "kept at ")
if at < 0 {
t.Fatalf("the bundle wrote over a file it had no record of and kept nothing: %q", detail)
}
if got, err := os.ReadFile(detail[at+len("kept at "):]); err != nil ||
string(got) != "# the distribution's own\n" {
t.Errorf("the kept original is %q (%v)", got, err)
}
}
+76 -2
View File
@@ -34,6 +34,8 @@ import (
"fmt"
"strings"
"time"
"github.com/novox/mesh-host/internal/firewall"
)
// Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence
@@ -185,6 +187,20 @@ type Options struct {
Prompt func(Choice) (string, error)
// Extras are catalogue modules beyond the floor, asked for by name.
Extras []string
// Ports are the ports the foundation binds on this machine (novox/hq ADR 0100). Inputs to
// genesis, each checked free before anything is raised, and then the node's settings for the
// foundation's modules — so adopting the foundation as modules leaves it where it was raised.
// Zero means the catalogue's defaults.
Ports FoundationPorts
// OverlayRange is the private network's address range, checked against every interface and
// route the machine already has. Empty means the mesh's default.
OverlayRange string
// Adopted raises this machine as an adopted node (novox/hq ADR 0100): what is on it is kept
// until each module is taken, its firewall stays in force, and the mesh guards its own ports
// in a table that only refuses. Without it, a machine in use is refused.
Adopted bool
}
// pivots reports whether this run goes past the foundation.
@@ -287,6 +303,14 @@ type Result struct {
// Stopped names why a run went no further. Empty on a run that pivoted.
Stopped string `json:"stopped,omitempty"`
// Adopted, the firewall found, and the ports the foundation was raised on (novox/hq ADR 0100).
Adopted bool `json:"adopted,omitempty"`
Firewall string `json:"firewall,omitempty"`
Ports FoundationPorts `json:"ports"`
// Filter is the packet filter chosen for when the node converges; an adopted genesis loads
// none, and the flip assigns this one.
Filter string `json:"filter-on-converge,omitempty"`
}
// Run performs the bootstrap, saying what it is doing as it goes.
@@ -339,7 +363,12 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
if say == nil {
say = func(string) {}
}
result := Result{DryRun: o.DryRun}
result := Result{DryRun: o.DryRun, Adopted: o.Adopted}
o.Ports = o.Ports.orDefaults()
result.Ports = o.Ports
if err := o.Ports.Check(); err != nil {
return result, failed(StepPreflight, err)
}
// ---- 1. preflight -------------------------------------------------------------------
say("preflight — what has to be true before anything is changed")
@@ -350,6 +379,23 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
// Which half of the host applies things here. Asked of the machine and proved, because
// `mesh-host` pins this at link time and an installer run by hand has no link time.
// An adopted machine keeps the firewall it was found with, so the mesh must speak it; one no
// host speaks is refused here, before anything changes (novox/hq ADR 0100).
if o.Adopted {
kind, name, err := firewall.Detect(ctx, d.Run)
if err != nil {
return result, failed(StepPreflight, err)
}
if kind == firewall.Unsupported {
return result, failed(StepPreflight, fmt.Errorf(
"this machine is filtered by %s, and no host speaks that firewall yet. An adopted "+
"machine keeps its firewall in force, so the mesh could neither open what it needs "+
"through it nor say what it would close. Nothing was changed", name))
}
result.Firewall = string(kind)
say(" adopted what is on this machine is kept; its firewall (" + string(kind) + ") stays in force")
}
sys, err := WorkOutSystem(ctx, d.Run, o.System)
if err != nil {
return result, failed(StepPreflight, err)
@@ -399,12 +445,32 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
if err := RefuseExistingServers(ctx, d.Run, creds); err != nil {
return result, failed(StepBundle, err)
}
// The foundation's ports, its private network's range and its containers' names are checked
// free before anything is raised (novox/hq ADR 0100), each refusal naming what holds it.
if err := CheckTheMachine(ctx, o, d.Run, rewritten.Declaration, say); err != nil {
return result, failed(StepBundle, err)
}
// From here on nothing this installer says contains the values it just made.
say = Masking(say, creds)
root, err := RewriteRoot(&rewritten, creds)
if err != nil {
return result, failed(StepBundle, err)
}
moved, err := RewritePorts(&rewritten, o.Ports, o.OverlayRange)
if err != nil {
return result, failed(StepBundle, err)
}
if moved.Places > 0 {
say(fmt.Sprintf(" ports %d place(s) rewritten to this node's foundation ports", moved.Places))
}
if o.Adopted {
adopted, err := RewriteAdopted(&rewritten, o.Ports)
if err != nil {
return result, failed(StepBundle, err)
}
say(fmt.Sprintf(" adopted bundle the foundation's filter is not loaded (%s); the mesh's guard refuses %v from outside",
strings.Join(adopted.Removed, ", "), adopted.Guarded))
}
for _, c := range []struct {
what, path string
made bool
@@ -679,7 +745,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
// ---- 17. filter -----------------------------------------------------------------------
say("filter — required, so the question is which, not whether")
if err := ChooseAndInstallFilter(ctx, o, permanentControl, say); err != nil {
filter, err := ChooseAndInstallFilter(ctx, o, permanentControl, say)
result.Filter = filter
if err != nil {
return result, failed(StepFilter, err)
}
@@ -697,6 +765,12 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
return result, failed(StepExport, err)
}
if o.Adopted {
say("\nthis machine is a mesh of one adopted node: it builds its own software, holds its graph " +
"and sits on its private network, and what it ran before is kept as it was, behind the firewall " +
"it was found with. Take each module on it once its data has moved; converge it when done.")
return result, nil
}
say("\nthis machine is a mesh of one node: it builds its own software, holds its graph, " +
"sits on its private network, and filters what modules declared.")
say("what remains is somebody else's: adding nodes, and assigning what they should run.")
+27
View File
@@ -1,8 +1,10 @@
package bootstrap
import (
"bytes"
"context"
"fmt"
"strconv"
"strings"
)
@@ -53,6 +55,9 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane
"This is the manifest that makes the builder an ordinary module. Without it the mesh "+
"has the image and no way to run it, so nothing can be built here", err)
}
if manifest, err = followPackagesPort(manifest, o.Ports.orDefaults().Packages); err != nil {
return out, err
}
pinned, places, err := pinPlaceholder(manifest, published.Reference, BuilderModule)
if err != nil {
return out, err
@@ -84,3 +89,25 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane
out.Installed.Pushed, err = pushNode(ctx, o, control, say)
return out, err
}
// packagesPortInBinding is the package registry's port as the builder's manifest names it, in the
// binding file it carries — JSON inside a JSON string, so its quotes are escaped.
const packagesPortInBinding = `\"port\": 3000`
// followPackagesPort points the builder's package binding at the port the node gave the package
// registry (novox/hq ADR 0100). Genesis raises the registry by hand before gitea is a module, so no
// binding the controller resolves can say where it is; the builder carries the address in its own
// manifest, and a port given at genesis must reach it there or the base build dials a port nothing
// answers on. At the default it is left byte for byte as the catalogue has it.
func followPackagesPort(manifest []byte, port int) ([]byte, error) {
if port == defaultGiteaPort {
return manifest, nil
}
if n := bytes.Count(manifest, []byte(packagesPortInBinding)); n != 1 {
return nil, fmt.Errorf("the builder's manifest names the package registry's port %d time(s) where "+
"this installer looks for it once (%s), so the port given with --packages-port cannot reach "+
"it; nothing was changed", n, packagesPortInBinding)
}
return bytes.Replace(manifest, []byte(packagesPortInBinding),
[]byte(`\"port\": `+strconv.Itoa(port)), 1), nil
}
+69
View File
@@ -0,0 +1,69 @@
package bootstrap
import (
"encoding/json"
"strings"
"testing"
)
// Defends novox/hq ADR 0100: a port given for the package registry at genesis reaches the one
// thing that dials it by a fixed number, the builder's package binding.
// The builder's package binding exactly as the catalogue's manifest carries it.
const builderManifest = `{
"module": "builder",
"resources": [
{
"id": "package-binding",
"type": "file",
"path": "/var/lib/mesh/builder/package-registry.json",
"mode": "0600",
"content": "{\"provision\": \"package-registry\", \"from\": \"gitea\", \"at\": \"127.0.0.1\", \"as\": \"mesh-builder\", \"serves\": {\"scheme\": \"http\", \"port\": 3000, \"npm-path\": \"/api/packages/novox/npm/\"}}\n"
}
]
}`
func bindingPort(t *testing.T, manifest []byte) float64 {
t.Helper()
var m struct {
Resources []struct {
Content string `json:"content"`
} `json:"resources"`
}
if err := json.Unmarshal(manifest, &m); err != nil {
t.Fatal(err)
}
var binding struct {
Serves struct {
Port float64 `json:"port"`
} `json:"serves"`
}
if err := json.Unmarshal([]byte(m.Resources[0].Content), &binding); err != nil {
t.Fatal(err)
}
return binding.Serves.Port
}
func TestTheBuilderFollowsThePackageRegistrysGivenPort(t *testing.T) {
got, err := followPackagesPort([]byte(builderManifest), 3100)
if err != nil {
t.Fatal(err)
}
if p := bindingPort(t, got); p != 3100 {
t.Errorf("the builder's binding dials %v, not the port given", p)
}
}
func TestTheBuilderOnTheDefaultPortIsUnchanged(t *testing.T) {
got, err := followPackagesPort([]byte(builderManifest), 3000)
if err != nil || string(got) != builderManifest {
t.Errorf("the default port changed the manifest: %v", err)
}
}
func TestABuilderManifestThatNoLongerNamesThePortIsRefused(t *testing.T) {
moved := strings.Replace(builderManifest, `\"port\": 3000`, `\"port\": 3001`, 1)
if _, err := followPackagesPort([]byte(moved), 3100); err == nil || !strings.Contains(err.Error(), "--packages-port") {
t.Errorf("a manifest the port cannot reach was accepted: %v", err)
}
}
+7 -1
View File
@@ -72,7 +72,13 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
if mentions(nodes, o.Node) {
say(" already a node " + o.Node)
} else {
if _, err := control.tell(ctx, "node", "add", o.Node); err != nil {
add := []string{"node", "add", o.Node}
if o.Adopted {
// The controller records the node's mode; an adopted one keeps what it was found with
// until each module is taken (novox/hq ADR 0100).
add = append(add, "--adopted")
}
if _, err := control.tell(ctx, add...); err != nil {
return out, err
}
out.Added = true
+146
View File
@@ -0,0 +1,146 @@
package bootstrap
import (
"context"
"fmt"
"net"
"strings"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/reachable"
"github.com/novox/mesh-host/internal/store"
)
// quiet are the processes every fresh machine runs that serve nobody: name resolution (whose
// link-local resolver listens on TCP as well as UDP, on every address) and the network manager's
// address configuration. ss names a process by its first fifteen characters, so both spellings are
// here.
//
// **Only what the measurement found** (novox/hq ADR 0101): these two hold every listener on a
// freshly installed lab machine (testdata/fresh-machine-listeners.txt) and nothing else does. A
// daemon joins this list with a measurement of a fresh machine that holds it, never by guess — a
// time client or an address-configuration client listening on a machine that does not run one as
// standard is something somebody installed, and that is a machine in use.
var quiet = map[string]bool{
"systemd-resolved": true, "systemd-resolve": true,
"systemd-networkd": true, "systemd-network": true,
}
// InUse says what makes this machine a machine in use (novox/hq ADR 0100): every running container
// no host made, and every socket listening on an address other than loopback that is neither ssh's
// nor held by what every fresh machine runs. ours names
// what the mesh itself runs, which a re-run of genesis finds and does not count.
func InUse(ctx context.Context, run Runner, ours func(name string) bool) ([]string, []reachable.Reach, error) {
var containers []string
out, err := run(ctx, "docker", "ps", "--format", "{{.Names}}\t{{.Label \"mesh-host.spec\"}}")
if err != nil {
return nil, nil, fmt.Errorf("cannot ask the container runtime what is running here: %w", err)
}
for _, line := range strings.Split(out, "\n") {
name, label, _ := strings.Cut(strings.TrimSpace(line), "\t")
label = strings.TrimSpace(label)
if name == "" || (label != "" && label != "<no value>") || ours(name) {
continue
}
containers = append(containers, name)
}
listening, err := run(ctx, "ss", "-Hltunp")
if err != nil {
return nil, nil, fmt.Errorf("cannot read what listens on this machine: %w", err)
}
var listeners []reachable.Reach
for _, r := range reachable.Sockets(listening) {
if counts(r) && !ours(r.By) {
listeners = append(listeners, r)
}
}
return containers, listeners, nil
}
func counts(r reachable.Reach) bool {
if ip := net.ParseIP(r.Address); ip != nil && ip.IsLoopback() {
return false
}
switch r.Protocol {
case "tcp":
return r.By != "sshd" && !(r.By == "" && r.Port == 22) && !quiet[r.By]
case "udp":
return !quiet[r.By]
}
return false
}
// RefuseAMachineInUse is the check a converged genesis makes before changing anything: a machine
// in use is refused, naming every container and listener counted, because raising the foundation's
// filter there would close what it serves — a forgotten --adopted must not close a working machine.
// An adopted genesis is told what it found, and goes on.
func RefuseAMachineInUse(ctx context.Context, o Options, run Runner, say func(string)) error {
known, err := store.Load(o.State)
if err != nil {
return err
}
if len(known.Resources) > 0 {
// **The machine says how it was raised** (novox/hq ADR 0103). A re-run must not change
// the node's mode by a flag forgotten or added: without --adopted the bundle would load
// the foundation's dropping filter over the found firewall, and with it on a converged
// machine the filter the node relies on would be removed as no longer carried.
switch adopted := RecordsAdoption(known); {
case adopted && !o.Adopted:
return fmt.Errorf("this machine was raised adopted, and genesis was run again without --adopted. " +
"Run it again the way it was raised: pass --adopted. Returning it to converged is the " +
"controller's act (converge), never genesis's; nothing was changed")
case !adopted && o.Adopted:
return fmt.Errorf("this machine was raised converged, and genesis was run again with --adopted, " +
"which would remove the foundation's filter it relies on. Run it again without --adopted; " +
"returning a node to adopted is the controller's act (adopt); nothing was changed")
}
// What genesis raised on an earlier run is the mesh's, and it is what the machine now
// serves; the question was answered the first time.
say(" in use not asked: this machine carries what an earlier genesis raised")
return nil
}
containers, listeners, err := InUse(ctx, run, func(string) bool { return false })
if err != nil {
return err
}
if len(containers) == 0 && len(listeners) == 0 {
say(" in use no: no container runs and nothing listens beyond ssh")
return nil
}
var named []string
for _, c := range containers {
named = append(named, "container "+c)
}
for _, l := range listeners {
by := l.By
if by == "" {
by = "an unnamed process"
}
named = append(named, fmt.Sprintf("%s %s:%d by %s", l.Protocol, l.Address, l.Port, by))
}
if o.Adopted {
say(fmt.Sprintf(" in use yes, and adopted: %d thing(s) found are kept", len(named)))
return nil
}
return fmt.Errorf("this machine is in use, and a converged genesis would close what it serves:\n - %s\n"+
"If it is meant to join the mesh keeping what it runs, pass --adopted: its firewall stays in "+
"force and every module is taken on it one at a time. Nothing was changed",
strings.Join(named, "\n - "))
}
// RecordsAdoption is whether this machine's state says it is an adopted node: it holds something
// of the mesh's that only an adopted node has — the guard or an opening, under the adoption prefix
// — or something it found and holds. A node the controller converged has neither any more; the
// record of the firewall it found outlives the flip, so it is not read as the mode.
func RecordsAdoption(known store.State) bool {
if len(known.Held) > 0 {
return true
}
for _, r := range known.Resources {
if strings.HasPrefix(r.ID, declaration.AdoptionPrefix) {
return true
}
}
return false
}
+179
View File
@@ -0,0 +1,179 @@
package bootstrap
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0100: a converged genesis refuses a machine in use, naming every container
// and listener it counted.
// Lines as `ss -Hltunp` prints them. The ssh, samba, loopback and proxy lines are captured from a
// real machine; the resolver and network-manager lines are written in the same shape. What a fresh
// machine actually runs is measured in testdata/fresh-machine-listeners.txt.
const inUseSockets = `tcp LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=1188536,fd=6))
tcp LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=1188536,fd=7))
tcp LISTEN 0 32 127.0.0.1:53 0.0.0.0:* users:(("dnsmasq",pid=1189392,fd=7))
tcp LISTEN 0 4096 127.0.0.1:5432 0.0.0.0:* users:(("docker-proxy",pid=1854543,fd=7))
udp UNCONN 0 0 0.0.0.0:5355 0.0.0.0:* users:(("systemd-resolve",pid=301,fd=11))
udp UNCONN 0 0 192.0.2.10%eth0:68 0.0.0.0:* users:(("systemd-network",pid=280,fd=19))
`
const servingSockets = `tcp LISTEN 0 50 0.0.0.0:445 0.0.0.0:* users:(("smbd",pid=1248,fd=29))
tcp LISTEN 0 4096 0.0.0.0:8080 0.0.0.0:* users:(("docker-proxy",pid=1920035,fd=7))
udp UNCONN 0 0 0.0.0.0:123 0.0.0.0:* users:(("ntpd",pid=1070791,fd=17))
`
type inUseRunner struct{ ps, ss string }
func (m inUseRunner) run(_ context.Context, name string, args ...string) (string, error) {
if name == "docker" {
return m.ps, nil
}
return m.ss, nil
}
func TestAFreshMachineIsNotInUse(t *testing.T) {
containers, listeners, err := InUse(context.Background(), inUseRunner{ss: inUseSockets}.run,
func(string) bool { return false })
if err != nil {
t.Fatal(err)
}
if len(containers) != 0 || len(listeners) != 0 {
t.Errorf("ssh, loopback and the daemons a fresh machine runs were counted: %v %v", containers, listeners)
}
}
func TestAMachineServingIsInUse(t *testing.T) {
m := inUseRunner{ps: "hello-web\t\nmesh-store\tabc123\n", ss: inUseSockets + servingSockets}
containers, listeners, err := InUse(context.Background(), m.run, func(string) bool { return false })
if err != nil {
t.Fatal(err)
}
if len(containers) != 1 || containers[0] != "hello-web" {
t.Errorf("containers counted: %v (one a host made is not a predecessor's)", containers)
}
var by []string
for _, l := range listeners {
by = append(by, l.By)
}
if strings.Join(by, " ") != "smbd docker-proxy ntpd" {
t.Errorf("listeners counted: %v", listeners)
}
}
func TestAConvergedGenesisRefusesAMachineInUseNamingEverything(t *testing.T) {
o := Options{State: filepath.Join(t.TempDir(), "state.json")}
m := inUseRunner{ps: "hello-web\t\n", ss: inUseSockets + servingSockets}
err := RefuseAMachineInUse(context.Background(), o, m.run, quietly)
if err == nil {
t.Fatal("a machine in use was not refused")
}
for _, want := range []string{"container hello-web", "tcp 0.0.0.0:445 by smbd", "tcp 0.0.0.0:8080 by docker-proxy",
"udp 0.0.0.0:123 by ntpd", "--adopted"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not name %q: %v", want, err)
}
}
o.Adopted = true
if err := RefuseAMachineInUse(context.Background(), o, m.run, quietly); err != nil {
t.Errorf("an adopted genesis was refused a machine in use: %v", err)
}
}
func TestARerunOfGenesisIsNotAMachineInUse(t *testing.T) {
o := Options{State: filepath.Join(t.TempDir(), "state.json")}
if err := store.Save(o.State, store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}}); err != nil {
t.Fatal(err)
}
m := inUseRunner{ps: "mesh-gitea-server\t\n", ss: servingSockets}
if err := RefuseAMachineInUse(context.Background(), o, m.run, quietly); err != nil {
t.Errorf("what an earlier genesis raised was counted as a machine in use: %v", err)
}
}
func TestAFreshlyInstalledMachineAsMeasuredIsNotInUse(t *testing.T) {
// Captured with `ss -Hltunp` on a freshly installed lab machine: its resolver listens on TCP on
// every address, which the record's words alone would count.
raw, err := os.ReadFile("testdata/fresh-machine-listeners.txt")
if err != nil {
t.Fatal(err)
}
run := func(ctx context.Context, name string, args ...string) (string, error) {
if name == "ss" {
return string(raw), nil
}
return "", nil
}
containers, listeners, err := InUse(context.Background(), run, func(string) bool { return false })
if err != nil {
t.Fatal(err)
}
if len(containers) != 0 || len(listeners) != 0 {
t.Errorf("a fresh machine read as in use: containers %v, listeners %v", containers, listeners)
}
}
// Defends novox/hq ADR 0103: a machine raised adopted stays adopted if genesis is run again. The
// installer reads the mode from what the machine records, and refuses a flag that disagrees.
func TestARerunWithoutTheFlagOnAnAdoptedMachineIsRefused(t *testing.T) {
o := Options{State: filepath.Join(t.TempDir(), "state.json")}
adoptedState := store.State{Resources: []store.Applied{
{ID: "store", Type: "container", Target: "mesh-store", Origin: store.OriginCarried},
{ID: "adoption.guard", Type: "file", Target: "/etc/mesh/guard.nft", Origin: store.OriginCarried},
}}
if err := store.Save(o.State, adoptedState); err != nil {
t.Fatal(err)
}
m := inUseRunner{ss: inUseSockets}
err := RefuseAMachineInUse(context.Background(), o, m.run, quietly)
if err == nil || !strings.Contains(err.Error(), "pass --adopted") {
t.Fatalf("a re-run without --adopted on an adopted machine was not refused: %v", err)
}
o.Adopted = true
if err := RefuseAMachineInUse(context.Background(), o, m.run, quietly); err != nil {
t.Errorf("a re-run with --adopted on an adopted machine was refused: %v", err)
}
}
func TestARerunWithTheFlagOnAConvergedMachineIsRefused(t *testing.T) {
o := Options{State: filepath.Join(t.TempDir(), "state.json"), Adopted: true}
converged := store.State{Resources: []store.Applied{
{ID: "store", Type: "container", Target: "mesh-store", Origin: store.OriginCarried},
{ID: "base-filter", Type: "file", Target: "/etc/nftables.conf", Origin: store.OriginCarried},
},
// Converged by the controller from adopted: the firewall it found is still recorded.
Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, DisabledByMesh: true}}
if err := store.Save(o.State, converged); err != nil {
t.Fatal(err)
}
err := RefuseAMachineInUse(context.Background(), o, inUseRunner{ss: inUseSockets}.run, quietly)
if err == nil || !strings.Contains(err.Error(), "without --adopted") {
t.Fatalf("a re-run with --adopted on a converged machine was not refused: %v", err)
}
o.Adopted = false
if err := RefuseAMachineInUse(context.Background(), o, inUseRunner{ss: inUseSockets}.run, quietly); err != nil {
t.Errorf("a converged re-run of a converged machine was refused: %v", err)
}
}
func TestOnlyTheDaemonsTheMeasurementFoundAreQuiet(t *testing.T) {
// novox/hq ADR 0101: the exempt daemons are the ones a fresh machine was measured to run —
// the resolver and the network manager. A time client or a DHCP client listening beyond
// loopback is something somebody put there, and that is a machine in use.
sockets := `udp UNCONN 0 0 0.0.0.0:123 0.0.0.0:* users:(("systemd-timesyn",pid=260,fd=9))
udp UNCONN 0 0 0.0.0.0:68 0.0.0.0:* users:(("dhcpcd",pid=270,fd=9))
`
_, listeners, err := InUse(context.Background(), inUseRunner{ss: sockets}.run, func(string) bool { return false })
if err != nil {
t.Fatal(err)
}
if len(listeners) != 2 {
t.Errorf("counted %d listener(s), want the time client and the DHCP client: %+v", len(listeners), listeners)
}
}
+14
View File
@@ -124,6 +124,9 @@ func registerAndAssign(ctx context.Context, o Options, control controlPlane, mod
// the only place the reason appears.
say(indent(refusal))
}
if err := prepareModule(ctx, o, control, module, say); err != nil {
return out, err
}
return out, nil
}
@@ -209,3 +212,14 @@ func pinPlaceholder(manifest []byte, reference, module string) ([]byte, int, err
}
return pinned, places, nil
}
// prepareModule is what genesis tells the controller about a module on this node once it is
// assigned and before it is pushed: the ports this node gave it, and on an adopted node that the
// module is taken (novox/hq ADR 0100).
func prepareModule(ctx context.Context, o Options, control controlPlane, module string,
say func(string)) error {
if err := setFoundationSettings(ctx, o, control, module, say); err != nil {
return err
}
return takeIfAdopted(ctx, o, control, module, say)
}
+45 -6
View File
@@ -3,8 +3,10 @@ package bootstrap
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"strconv"
"strings"
"time"
)
@@ -87,6 +89,9 @@ func InstallFromCatalogue(ctx context.Context, o Options, control controlPlane,
if _, err := control.tell(ctx, "assign", o.Node, module); err != nil {
return err
}
if err := prepareModule(ctx, o, control, module, say); err != nil {
return err
}
if _, err := pushNode(ctx, o, control, say); err != nil {
return err
}
@@ -120,7 +125,7 @@ func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane,
Name: "endpoint",
Question: "Where do other machines reach this one for the private network? " +
"(host:port; the host other machines dial)",
Default: derivedEndpoint(brokerAddress),
Default: derivedEndpoint(brokerAddress, o.Ports.orDefaults().Hub),
}, o.Answers["endpoint"], o.Prompt, say)
if err != nil {
return err
@@ -129,6 +134,10 @@ func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane,
return fmt.Errorf("the private network needs an endpoint other machines can dial, and " +
"nothing said one: pass --endpoint, or --broker-address so one can be derived")
}
endpoint, err = endpointAgrees(endpoint, o.Ports.orDefaults().Hub)
if err != nil {
return err
}
if _, err := control.tell(ctx, "assign", o.Node, module); err != nil {
return err
@@ -146,16 +155,22 @@ func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane,
// ChooseAndInstallFilter picks the packet filter — required, so the question is which, not
// whether — and installs it.
func ChooseAndInstallFilter(ctx context.Context, o Options, control controlPlane, say func(string)) error {
func ChooseAndInstallFilter(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) {
filter, err := decide(Choice{
Name: "packet-filter",
Question: "Which packet filter should this machine run?",
Options: []string{"nftables"},
}, o.Answers["packet-filter"], o.Prompt, say)
if err != nil {
return err
return "", err
}
return InstallFromCatalogue(ctx, o, control, filter, say)
if o.Adopted {
// The firewall found here stays in force until the node converges; the filter is
// chosen now and assigned by the flip (novox/hq ADR 0100).
say(" not installed " + filter + " — this machine is adopted; converging it assigns " + filter)
return filter, nil
}
return filter, InstallFromCatalogue(ctx, o, control, filter, say)
}
// InstallExtras installs what was asked for beyond the floor.
@@ -200,14 +215,38 @@ func builds(manifest []byte) bool {
return m.Build != nil && len(m.Build.Artifacts) > 0
}
// endpointAgrees holds the endpoint other machines dial to the port this node gave the private
// network's hub (novox/hq ADR 0100): the hub binds what --hub-port says, so an endpoint naming
// another port is an address nothing answers on. A host alone takes the hub's port.
func endpointAgrees(endpoint string, hub int) (string, error) {
_, portText, err := net.SplitHostPort(endpoint)
var missing *net.AddrError
if errors.As(err, &missing) && missing.Err == "missing port in address" {
return net.JoinHostPort(strings.Trim(endpoint, "[]"), strconv.Itoa(hub)), nil
}
if err != nil {
return "", fmt.Errorf("--endpoint %q is not host:port: %w", endpoint, err)
}
port, err := strconv.Atoi(portText)
if err != nil {
return "", fmt.Errorf("--endpoint %q does not end in a port", endpoint)
}
if port != hub {
return "", fmt.Errorf("--endpoint %s names port %d and the private network's hub binds %d "+
"(--hub-port): other machines would dial a port nothing answers on. Give one port for the "+
"hub; nothing was changed", endpoint, port, hub)
}
return endpoint, nil
}
// derivedEndpoint is the default place other machines dial for the private network: the same host
// they already dial for the broker, on WireGuard's ordinary port. One fact, not two.
func derivedEndpoint(brokerAddress string) string {
func derivedEndpoint(brokerAddress string, hub int) string {
host, _, err := net.SplitHostPort(brokerAddress)
if err != nil || host == "" {
return ""
}
return net.JoinHostPort(host, "51820")
return net.JoinHostPort(host, strconv.Itoa(hub))
}
func refOr(ref string) string {
+24 -3
View File
@@ -1,14 +1,17 @@
package bootstrap
import "testing"
import (
"strings"
"testing"
)
// The endpoint other machines dial defaults to the host they already dial — the broker's — on
// WireGuard's port. One fact, not two that drift.
func TestTheEndpointDerivesFromTheBrokerAddress(t *testing.T) {
if got := derivedEndpoint("192.0.2.10:5671"); got != "192.0.2.10:51820" {
if got := derivedEndpoint("192.0.2.10:5671", 51820); got != "192.0.2.10:51820" {
t.Fatalf("derived %q", got)
}
if got := derivedEndpoint(""); got != "" {
if got := derivedEndpoint("", 51820); got != "" {
t.Fatalf("an endpoint was invented from nothing: %q", got)
}
}
@@ -23,3 +26,21 @@ func TestOnlyAManifestWithArtifactsBuilds(t *testing.T) {
t.Fatal("a manifest with nothing to build was built anyway")
}
}
// Defends novox/hq ADR 0100: the hub's port is the node's, and the endpoint other machines dial
// must name it — an endpoint on another port is an address nothing answers on.
func TestTheEndpointAgreesWithTheHubsPort(t *testing.T) {
if got, err := endpointAgrees("192.0.2.10:51820", 51820); err != nil || got != "192.0.2.10:51820" {
t.Errorf("an endpoint on the hub's port: %q %v", got, err)
}
if got, err := endpointAgrees("192.0.2.10", 51821); err != nil || got != "192.0.2.10:51821" {
t.Errorf("a host alone did not take the hub's port: %q %v", got, err)
}
_, err := endpointAgrees("192.0.2.10:51820", 51821)
if err == nil || !strings.Contains(err.Error(), "--hub-port") {
t.Errorf("two ports for one hub were accepted: %v", err)
}
if _, err := endpointAgrees("192.0.2.10:not-a-port", 51820); err == nil {
t.Error("an endpoint whose port is not a number was accepted")
}
}
+3
View File
@@ -122,6 +122,9 @@ func installProvider(ctx context.Context, o Options, control controlPlane, modul
if _, err := control.tell(ctx, "assign", o.Node, module); err != nil {
return err
}
if err := prepareModule(ctx, o, control, module, say); err != nil {
return err
}
if beforePush != nil {
if err := beforePush(); err != nil {
return err
+17 -10
View File
@@ -42,8 +42,9 @@ const (
// giteaDBRole/giteaDBName is gitea's own database in the foundation store.
giteaDBRole = "mesh_gitea"
giteaDBName = "mesh_gitea"
// giteaPort is where the raised server answers on the machine.
giteaPort = 3000
// defaultGiteaPort is where the raised server answers on the machine unless the node gave the
// package registry another port (novox/hq ADR 0100).
defaultGiteaPort = 3000
)
// RaisePackageRegistry puts a working npm registry in front of the base build. It is idempotent:
@@ -60,17 +61,18 @@ func RaisePackageRegistry(ctx context.Context, o Options, d Deps, control contro
}
say(" seeding gitea's database in the foundation store")
ports := o.Ports.orDefaults()
if err := seedGiteaDatabase(ctx, run, o.Timeout, dbPassword, say); err != nil {
return err
}
say(" raising the gitea server on that database")
if err := raiseGiteaServer(ctx, run, o.Timeout, dbPassword, say); err != nil {
if err := raiseGiteaServer(ctx, run, o.Timeout, dbPassword, ports, say); err != nil {
return err
}
say(" waiting for gitea to answer")
base := fmt.Sprintf("http://127.0.0.1:%d", giteaPort)
base := fmt.Sprintf("http://127.0.0.1:%d", ports.Packages)
if err := waitForGitea(ctx, d, o, base, say); err != nil {
return err
}
@@ -146,11 +148,11 @@ func seedGiteaDatabase(ctx context.Context, run Runner, timeout time.Duration, p
return nil
}
// raiseGiteaServer starts the gitea server container against the foundation store. It joins the
// store's network namespace so `127.0.0.1:5432` reaches postgres, and publishes its own port on the
// machine so the builder and this installer can reach it. Started if absent, left alone if present.
// raiseGiteaServer starts the gitea server container against the foundation store. It runs on the
// machine's own network, so `127.0.0.1` reaches the store where it publishes its port, and it binds
// its own port there for the builder and this installer. Started if absent, left alone if present.
func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, dbPassword string,
say func(string)) error {
ports FoundationPorts, say func(string)) error {
asking, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
@@ -164,7 +166,7 @@ func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, db
env := []string{
"-e", "GITEA__database__DB_TYPE=postgres",
// The store is reached on the shared network namespace's loopback.
"-e", "GITEA__database__HOST=127.0.0.1:5432",
"-e", fmt.Sprintf("GITEA__database__HOST=127.0.0.1:%d", ports.Store),
"-e", "GITEA__database__NAME=" + giteaDBName,
"-e", "GITEA__database__USER=" + giteaDBRole,
"-e", "GITEA__database__PASSWD=" + dbPassword,
@@ -174,9 +176,14 @@ func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, db
// package metadata hands npm a tarball URL built from ROOT_URL, and a client only sends its
// stored credential to the host it was stored for. A default ROOT_URL of localhost is a
// different host than the binding's 127.0.0.1, so the credential would not be sent.
"-e", fmt.Sprintf("GITEA__server__ROOT_URL=http://127.0.0.1:%d/", giteaPort),
"-e", fmt.Sprintf("GITEA__server__ROOT_URL=http://127.0.0.1:%d/", ports.Packages),
"-e", "USER_UID=1000", "-e", "USER_GID=1000",
}
if ports.Packages != defaultGiteaPort {
// On the machine's network the server binds its own port, so a port given for it is
// the one it is told to listen on.
env = append(env, "-e", fmt.Sprintf("GITEA__server__HTTP_PORT=%d", ports.Packages))
}
args := append([]string{
"run", "-d", "--name", giteaBootstrap,
// Host network, like the control plane: it reaches the foundation store on the machine's
+460
View File
@@ -0,0 +1,460 @@
package bootstrap
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net"
"sort"
"strconv"
"strings"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/reachable"
"github.com/novox/mesh-host/internal/store"
)
// FoundationPorts are the machine's ports the foundation binds (novox/hq ADR 0100).
//
// **The node's, not the catalogue's.** A machine in use may already hold one — a predecessor's
// registry on 5000, its broker's management port — and a port fixed in the bundle and the manifests
// surfaces as a container that fails to bind, and one changed at genesis would be changed back when
// the foundation is adopted as modules. So each is an input here, checked free, rewritten into the
// bundle, and handed to the controller as that node's setting for the module that binds it.
type FoundationPorts struct {
Store int `json:"store"`
Bus int `json:"bus"`
AMQP int `json:"amqp"`
Management int `json:"management"`
Registry int `json:"registry"`
Packages int `json:"packages"`
Hub int `json:"hub"`
}
// DefaultPorts are the catalogue's numbers.
func DefaultPorts() FoundationPorts {
return FoundationPorts{Store: 5432, Bus: 5671, AMQP: 5672, Management: 15672, Registry: 5000,
Packages: 3000, Hub: 51820}
}
// DefaultOverlayRange is the controller's default private-network range.
const DefaultOverlayRange = "10.42.0.0/16"
// orDefaults fills every port left unsaid.
func (p FoundationPorts) orDefaults() FoundationPorts {
d := DefaultPorts()
for _, f := range []struct{ got, def *int }{
{&p.Store, &d.Store}, {&p.Bus, &d.Bus}, {&p.AMQP, &d.AMQP}, {&p.Management, &d.Management},
{&p.Registry, &d.Registry}, {&p.Packages, &d.Packages}, {&p.Hub, &d.Hub},
} {
if *f.got == 0 {
*f.got = *f.def
}
}
return p
}
// named is each port with what it is and its protocol, in a fixed order.
func (p FoundationPorts) named() []namedPort {
return []namedPort{
{"the store", "tcp", p.Store}, {"the bus", "tcp", p.Bus}, {"the broker's AMQP", "tcp", p.AMQP},
{"the broker's management", "tcp", p.Management}, {"the registry", "tcp", p.Registry},
{"the package registry", "tcp", p.Packages}, {"the private network's hub", "udp", p.Hub},
}
}
type namedPort struct {
what, protocol string
port int
}
// Check refuses a port out of range, or one port given for two things.
func (p FoundationPorts) Check() error {
seen := map[string]string{}
for _, n := range p.named() {
if n.port < 1 || n.port > 65535 {
return fmt.Errorf("%s's port is %d, and a port is 1-65535", n.what, n.port)
}
key := n.protocol + "/" + strconv.Itoa(n.port)
if other, twice := seen[key]; twice {
return fmt.Errorf("%s and %s were both given %s", other, n.what, key)
}
seen[key] = n.what
}
return nil
}
// moduleSettings is what each foundation module is told about its ports on this node: the port it
// declares, to the machine's port it is given. Only what differs from the catalogue — a converged
// genesis on the defaults sets nothing, and so changes nothing it did before.
func (p FoundationPorts) moduleSettings() map[string]map[string]int {
d := DefaultPorts()
out := map[string]map[string]int{}
add := func(module string, declared, given int) {
if given == declared {
return
}
if out[module] == nil {
out[module] = map[string]int{}
}
out[module][strconv.Itoa(declared)] = given
}
add("postgres", d.Store, p.Store)
add("lavinmq", d.Bus, p.Bus)
add("lavinmq", d.AMQP, p.AMQP)
add("lavinmq", d.Management, p.Management)
add(RegistryModule, d.Registry, p.Registry)
return out
}
// PortsSetting is the controller's settings key for a module's given ports.
const PortsSetting = "ports"
// setFoundationSettings tells the controller the ports this node gave a foundation module — and,
// on an adopted node, that the registry is reached from anywhere, as a node pulls from it before it
// has a private-network address (novox/hq ADR 0100). Done after the module is registered and before
// the push that raises it, so the first declaration already names the node's ports.
func setFoundationSettings(ctx context.Context, o Options, control controlPlane, module string,
say func(string)) error {
values := map[string]any{}
if ports := o.Ports.orDefaults().moduleSettings()[module]; len(ports) > 0 {
values[PortsSetting] = ports
}
if o.Adopted && module == RegistryModule {
values["expose"] = map[string]string{strconv.Itoa(DefaultPorts().Registry): "anywhere"}
}
if len(values) == 0 {
return nil
}
raw, err := json.Marshal(values)
if err != nil {
return err
}
remote := "/" + module + "-settings.json"
if err := control.carrying(ctx, module+"-settings.json", raw, remote); err != nil {
return err
}
if _, err := control.tell(ctx, "settings", "set", module, remote, "--node", o.Node); err != nil {
return err
}
say(" settings " + module + " on " + o.Node + ": " + string(raw))
return nil
}
// PortsRewrite says what RewritePorts changed.
type PortsRewrite struct {
Places int
}
// RewritePorts puts the node's foundation ports into the produced bundle, in place of the
// template's, byte for byte like every other rewrite — so the file keeps its comments and a person
// can read what was applied. A port left at its default is not touched, so a genesis on the
// defaults produces exactly the bundle it did before.
//
// Only the machine's side moves: the outer port of each mapping, the addresses the control plane
// dials on the machine's loopback, and the address nodes are told to dial. What a container listens
// on inside itself, and what an action reaches inside the store's own network, stay as they are.
func RewritePorts(r *Rewritten, p FoundationPorts, overlayRange string) (PortsRewrite, error) {
var out PortsRewrite
p = p.orDefaults()
d := DefaultPorts()
bundle := r.Bundle
var err error
replace := func(from, to, what string) {
if err != nil || from == to {
return
}
bundle, err = replaceOnce(bundle, from, to, what)
out.Places++
}
if p.Store != d.Store {
replace(`"ports": ["5432:5432"]`, fmt.Sprintf(`"ports": ["%d:5432"]`, p.Store), "the store's published port")
}
if p.Bus != d.Bus || p.AMQP != d.AMQP || p.Management != d.Management {
replace(`"ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"]`,
fmt.Sprintf(`"ports": ["%d:5671", "%d:5672", "127.0.0.1:%d:15672"]`, p.Bus, p.AMQP, p.Management),
"the broker's published ports")
}
if err != nil {
return out, err
}
// The control plane runs on the machine's network and dials the store and the broker on its
// loopback, so its connection strings name the machine's ports. The schema step reaches the
// store inside the store's own network and keeps the container's port — so these are found by
// the control plane's environment, not by searching for the text.
control, cerr := controlPlaneIn(r.Declaration)
if cerr != nil {
return out, cerr
}
for _, key := range sortedKeys(control.Env) {
value := control.Env[key]
now := value
now = strings.ReplaceAll(now, "@127.0.0.1:5432/", fmt.Sprintf("@127.0.0.1:%d/", p.Store))
now = strings.ReplaceAll(now, "@127.0.0.1:5672/", fmt.Sprintf("@127.0.0.1:%d/", p.AMQP))
if strings.HasSuffix(now, "@127.0.0.1:15672") {
now = strings.TrimSuffix(now, "15672") + strconv.Itoa(p.Management)
}
if key == brokerAddressVar {
if host, port, splitErr := net.SplitHostPort(value); splitErr == nil && port == "5671" {
now = net.JoinHostPort(host, strconv.Itoa(p.Bus))
}
}
if now != value {
replace(`"`+key+`": "`+value+`"`, `"`+key+`": "`+now+`"`, "the control plane's "+key)
}
}
if err != nil {
return out, err
}
// The foundation's own filter, where the template carries one: it admits the bus and the
// registry from anywhere, on whatever port they are.
for _, f := range []struct{ def, now int }{{d.Bus, p.Bus}, {d.Registry, p.Registry}} {
if f.def == f.now {
continue
}
for _, form := range []string{"tcp dport %d accept", "ct original proto-dst %d accept"} {
from, to := fmt.Sprintf(form, f.def), fmt.Sprintf(form, f.now)
if n := bytes.Count(bundle, []byte(from)); n > 0 {
bundle = bytes.ReplaceAll(bundle, []byte(from), []byte(to))
out.Places += n
}
}
}
// The private network's range, when it is not the default, is the controller's to know.
if overlayRange != "" && overlayRange != DefaultOverlayRange {
replace(`"`+brokerAddressVar+`": `,
`"MESH_OVERLAY_CIDR": "`+overlayRange+`",
"`+brokerAddressVar+`": `, "where the control plane is told the private network's range")
if err != nil {
return out, err
}
}
if out.Places == 0 {
return out, nil
}
parsed, perr := declaration.ParseFileTrusted(bundle)
if perr != nil {
return out, fmt.Errorf("the bundle stopped being a declaration after its ports were rewritten, which is this installer's fault: %w", perr)
}
r.Bundle, r.Declaration, r.Resources = bundle, parsed, len(parsed.Resources)
if c, cerr := controlPlaneIn(parsed); cerr == nil {
r.BrokerAddress = c.Env[brokerAddressVar]
}
return out, nil
}
// PortsFree refuses a foundation port something else already holds, naming what holds it. What the
// mesh itself raised on an earlier run of genesis is not counted: ours says which holders are.
func PortsFree(ctx context.Context, run Runner, p FoundationPorts, ours func(reachable.Reach) bool) error {
out, err := run(ctx, "ss", "-Hltunp")
if err != nil {
return fmt.Errorf("cannot read which ports this machine holds, so the foundation's cannot be checked free: %w", err)
}
sockets := reachable.Sockets(out)
var published []reachable.Reach
if ps, err := run(ctx, "docker", "ps", "--format", "{{.Names}}\t{{.Ports}}"); err == nil {
published = reachable.Published(ps)
}
held := reachable.Merge(sockets, published)
var problems []string
for _, n := range p.orDefaults().named() {
var by []string
for _, r := range held {
if r.Protocol != n.protocol || r.Port != n.port || ours(r) {
continue
}
holder := r.By
if holder == "" {
holder = "something ss does not name"
}
if r.Published {
holder = "the container " + r.By
}
if !contains(by, holder) {
by = append(by, holder)
}
}
if len(by) > 0 {
problems = append(problems, fmt.Sprintf("%s's port %s/%d is held by %s",
n.what, n.protocol, n.port, strings.Join(by, ", ")))
}
}
if len(problems) > 0 {
return fmt.Errorf("the foundation's ports must be free before anything is raised:\n - %s\n"+
"Give it another with the matching flag (--store-port, --bus-port, --amqp-port, "+
"--management-port, --registry-port, --packages-port, --hub-port); nothing was changed",
strings.Join(problems, "\n - "))
}
return nil
}
// OverlayClear refuses a private-network range that overlaps an address or a route the machine
// already has — a predecessor's tunnel still running — naming the interface. The mesh's own
// interface is not counted.
func OverlayClear(ctx context.Context, run Runner, overlayRange string) error {
if overlayRange == "" {
overlayRange = DefaultOverlayRange
}
_, mine, err := net.ParseCIDR(overlayRange)
if err != nil {
return fmt.Errorf("the private network's range %q is not a range: %w", overlayRange, err)
}
var clashes []string
if out, err := run(ctx, "ip", "-o", "addr", "show"); err == nil {
for _, line := range strings.Split(out, "\n") {
f := strings.Fields(line)
// 3: wg0 inet 10.42.0.1/24 scope global wg0
if len(f) < 4 || (f[2] != "inet" && f[2] != "inet6") {
continue
}
iface := strings.TrimSuffix(f[1], ":")
if clash(mine, f[3]) && iface != meshInterface {
clashes = append(clashes, fmt.Sprintf("%s holds %s", iface, f[3]))
}
}
} else {
return fmt.Errorf("cannot read this machine's addresses to check the private network's range: %w", err)
}
if out, err := run(ctx, "ip", "-o", "route", "show"); err == nil {
for _, line := range strings.Split(out, "\n") {
f := strings.Fields(line)
// 10.42.0.0/16 dev wg0 proto kernel scope link src 10.42.0.1
if len(f) < 3 || f[0] == "default" {
continue
}
iface := ""
for i := range f {
if f[i] == "dev" && i+1 < len(f) {
iface = f[i+1]
}
}
if iface != meshInterface && clash(mine, f[0]) {
clashes = append(clashes, fmt.Sprintf("%s routes %s", iface, f[0]))
}
}
}
if len(clashes) > 0 {
return fmt.Errorf("the private network's range %s overlaps what this machine already has: %s.\n"+
"A tunnel a predecessor still runs would take the mesh's traffic. Give another range with "+
"--overlay-range; nothing was changed", overlayRange, strings.Join(clashes, "; "))
}
return nil
}
// meshInterface is the private network's own interface, which a re-run finds holding its range.
const meshInterface = "mesh0"
func clash(mine *net.IPNet, other string) bool {
if !strings.Contains(other, "/") {
if ip := net.ParseIP(other); ip != nil {
return mine.Contains(ip)
}
return false
}
ip, theirs, err := net.ParseCIDR(other)
if err != nil {
return false
}
return mine.Contains(theirs.IP) || theirs.Contains(mine.IP) || mine.Contains(ip)
}
// NamesFree refuses a foundation or bundle container name that a container already has, when no
// host made that container and this node has no record of it — a predecessor's container under the
// mesh's name, which raising the foundation would replace.
func NamesFree(ctx context.Context, run Runner, names []string, known store.State) error {
var taken []string
sorted := append([]string{}, names...)
sort.Strings(sorted)
for _, name := range sorted {
out, err := run(ctx, "docker", "inspect", "--format",
"{{index .Config.Labels \"mesh-host.spec\"}}", name)
if err != nil {
continue // no such container
}
label := strings.TrimSpace(out)
if label != "" && label != "<no value>" {
continue
}
if known.Recorded(string(declaration.TypeContainer), name) {
continue
}
if name == giteaBootstrap && len(known.Resources) > 0 {
// Genesis raises the package registry itself, by hand and before the host records
// anything of it, so on a re-run it is found under its own name with no label and no
// record. A machine that carries what an earlier genesis raised made it.
continue
}
taken = append(taken, name)
}
if len(taken) > 0 {
return fmt.Errorf("this machine already runs a container under the name the foundation uses, "+
"and nothing of the mesh's made it: %s.\nRaising the foundation would replace it. Rename or "+
"stop it first; nothing was changed", strings.Join(taken, ", "))
}
return nil
}
// CheckTheMachine is every check genesis makes before raising anything that this machine does not
// already hold what the foundation needs: its ports, its private network's range, its containers'
// names (novox/hq ADR 0100). A re-run of genesis finds the foundation it raised and does not count
// it.
func CheckTheMachine(ctx context.Context, o Options, run Runner, bundle *declaration.Declaration,
say func(string)) error {
known, err := store.Load(o.State)
if err != nil {
return err
}
rerun := len(known.Resources) > 0
names := foundationNames(bundle)
mine := map[string]bool{}
for _, n := range names {
mine[n] = true
}
p := o.Ports.orDefaults()
ours := func(r reachable.Reach) bool {
switch {
case mine[r.By]:
return true
case !rerun:
return false
case r.By == "gitea" && r.Port == p.Packages:
// The package registry runs on the machine's network, so ss names its process.
return true
case r.By == "" && r.Protocol == "udp" && r.Port == p.Hub:
// The private network's hub is a kernel interface and has no process.
return true
}
return false
}
if err := PortsFree(ctx, run, p, ours); err != nil {
return err
}
say(fmt.Sprintf(" ports free store %d, bus %d, amqp %d, management %d, registry %d, packages %d, hub %d/udp",
p.Store, p.Bus, p.AMQP, p.Management, p.Registry, p.Packages, p.Hub))
if err := OverlayClear(ctx, run, o.OverlayRange); err != nil {
return err
}
if err := NamesFree(ctx, run, names, known); err != nil {
return err
}
return nil
}
// foundationNames are the containers the foundation and genesis raise under fixed names.
func foundationNames(bundle *declaration.Declaration) []string {
names := []string{ControlPlaneModule, giteaBootstrap, "mesh-registry"}
for _, n := range containerNames(bundle) {
if !contains(names, n) {
names = append(names, n)
}
}
sort.Strings(names)
return names
}
+299
View File
@@ -0,0 +1,299 @@
package bootstrap
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/reachable"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0100: the foundation's ports are the node's — inputs to genesis, checked free,
// rewritten into the bundle, and handed to the controller as the node's settings.
func producedBundle(t *testing.T) Rewritten {
t.Helper()
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
if err != nil {
t.Skip("no example bundle beside this checkout")
}
r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32))
if err != nil {
t.Fatal(err)
}
if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
t.Fatal(err)
}
return r
}
func containerNamed(d *declaration.Declaration, name string) *declaration.Container {
for _, r := range d.Resources {
if c, ok := r.(*declaration.Container); ok && c.Name == name {
return c
}
}
return nil
}
func TestTheDefaultPortsLeaveTheBundleAsItWas(t *testing.T) {
r := producedBundle(t)
before := string(r.Bundle)
got, err := RewritePorts(&r, DefaultPorts(), DefaultOverlayRange)
if err != nil {
t.Fatal(err)
}
if got.Places != 0 || string(r.Bundle) != before {
t.Errorf("the default ports rewrote %d place(s)", got.Places)
}
}
func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) {
r := producedBundle(t)
p := FoundationPorts{Store: 5433, Bus: 5771, AMQP: 5772, Management: 15673, Registry: 5100}
got, err := RewritePorts(&r, p, "10.77.0.0/16")
if err != nil {
t.Fatal(err)
}
if got.Places == 0 {
t.Fatal("nothing was rewritten")
}
storeC := containerNamed(r.Declaration, "mesh-store")
if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" {
t.Errorf("the store publishes %v", storeC.Ports)
}
broker := containerNamed(r.Declaration, "mesh-broker")
if strings.Join(broker.Ports, " ") != "5771:5671 5772:5672 127.0.0.1:15673:15672" {
t.Errorf("the broker publishes %v", broker.Ports)
}
control, err := controlPlaneIn(r.Declaration)
if err != nil {
t.Fatal(err)
}
for key, value := range control.Env {
if strings.HasPrefix(key, "MESH_STORE_") && !strings.Contains(value, "@127.0.0.1:5433/") {
t.Errorf("%s still dials %s", key, value)
}
}
if !strings.Contains(control.Env["MESH_BROKER_AMQP"], "@127.0.0.1:5772/") ||
!strings.HasSuffix(control.Env["MESH_BROKER_MANAGEMENT"], "@127.0.0.1:15673") {
t.Errorf("the broker is dialled at %s and %s", control.Env["MESH_BROKER_AMQP"], control.Env["MESH_BROKER_MANAGEMENT"])
}
if control.Env["MESH_BROKER_ADDRESS"] != "192.0.2.10:5771" || r.BrokerAddress != "192.0.2.10:5771" {
t.Errorf("nodes are told to dial %s (%s)", control.Env["MESH_BROKER_ADDRESS"], r.BrokerAddress)
}
if control.Env["MESH_OVERLAY_CIDR"] != "10.77.0.0/16" {
t.Errorf("the control plane is told the range %q", control.Env["MESH_OVERLAY_CIDR"])
}
// The schema step reaches the store inside its own network, on the container's port.
text := string(r.Bundle)
if !strings.Contains(text, `MESH_STORE_INVENTORY=postgres://postgres:s@127.0.0.1:5432/inventory`) {
t.Error("the schema step's connection, inside the store's network, was moved off the container's port")
}
for _, want := range []string{"tcp dport 5771 accept", "ct original proto-dst 5771 accept",
"tcp dport 5100 accept", "ct original proto-dst 5100 accept"} {
if !strings.Contains(text, want) {
t.Errorf("the base filter does not say %q", want)
}
}
if strings.Contains(text, "dport 5671 accept") || strings.Contains(text, "dport 5000 accept") {
t.Error("the base filter still admits a default port")
}
}
func TestATemplateThatDoesNotSayItsPortsAsExpectedIsRefused(t *testing.T) {
r := producedBundle(t)
r.Bundle = []byte(strings.Replace(string(r.Bundle), `"ports": ["5432:5432"]`, `"ports": [ "5432:5432" ]`, 1))
if _, err := RewritePorts(&r, FoundationPorts{Store: 5433}, ""); err == nil {
t.Error("a store port the installer could not find was silently left")
}
}
func TestTwoThingsOnOnePortAreRefused(t *testing.T) {
p := DefaultPorts()
p.Registry = p.Store
if err := p.Check(); err == nil {
t.Error("the registry and the store were both given one port")
}
p = DefaultPorts()
p.Hub = 5432 // udp, beside the store's tcp: two different ports
if err := p.Check(); err != nil {
t.Errorf("a udp port beside a tcp one of the same number was refused: %v", err)
}
}
// machineRunner answers ss, docker ps, docker inspect and ip from fixtures.
type machineRunner struct {
ss, ps, addrs, routes string
unlabelled map[string]bool
labelled map[string]bool
}
func (m machineRunner) run(_ context.Context, name string, args ...string) (string, error) {
switch {
case name == "ss":
return m.ss, nil
case name == "docker" && args[0] == "ps":
return m.ps, nil
case name == "docker" && args[0] == "inspect":
n := args[len(args)-1]
if m.labelled[n] {
return "abc\n", nil
}
if m.unlabelled[n] {
return "\n", nil
}
return "", errors.New("no such container")
case name == "ip" && args[1] == "addr":
return m.addrs, nil
case name == "ip" && args[1] == "route":
return m.routes, nil
}
return "", nil
}
func noneOurs(reachable.Reach) bool { return false }
func TestABusyPortIsRefusedNamingItsHolder(t *testing.T) {
m := machineRunner{
ss: "tcp LISTEN 0 4096 0.0.0.0:5000 0.0.0.0:* users:((\"docker-proxy\",pid=1,fd=7))\n" +
"tcp LISTEN 0 4096 127.0.0.1:15672 0.0.0.0:* users:((\"beam.smp\",pid=2,fd=7))\n",
ps: "predecessor-registry\t0.0.0.0:5000->5000/tcp\n",
}
err := PortsFree(context.Background(), m.run, DefaultPorts(), noneOurs)
if err == nil {
t.Fatal("held ports were not refused")
}
for _, want := range []string{"predecessor-registry", "beam.smp", "tcp/5000", "tcp/15672", "--registry-port"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not say %q: %v", want, err)
}
}
p := DefaultPorts()
p.Registry, p.Management = 5100, 15673
if err := PortsFree(context.Background(), m.run, p, noneOurs); err != nil {
t.Errorf("other ports given and still refused: %v", err)
}
}
func TestTheFoundationsOwnContainersAreNotCountedOnARerun(t *testing.T) {
m := machineRunner{
ss: "tcp LISTEN 0 4096 0.0.0.0:5432 0.0.0.0:* users:((\"docker-proxy\",pid=1,fd=7))\n",
ps: "mesh-store\t0.0.0.0:5432->5432/tcp\n",
}
ours := func(r reachable.Reach) bool { return r.By == "mesh-store" }
if err := PortsFree(context.Background(), m.run, DefaultPorts(), ours); err != nil {
t.Errorf("the foundation's own store was counted as holding its port: %v", err)
}
}
func TestAnOverlappingTunnelIsRefusedNamingItsInterface(t *testing.T) {
m := machineRunner{
addrs: "1: lo inet 127.0.0.1/8 scope host lo\n5: wg0 inet 10.42.3.1/24 scope global wg0\n7: mesh0 inet 10.42.0.1/16 scope global mesh0\n",
routes: "default via 192.0.2.1 dev eth0\n10.42.3.0/24 dev wg0 proto kernel scope link src 10.42.3.1\n",
}
err := OverlayClear(context.Background(), m.run, "")
if err == nil || !strings.Contains(err.Error(), "wg0") || strings.Contains(err.Error(), "mesh0") {
t.Fatalf("the overlap was not named by its interface alone: %v", err)
}
if err := OverlayClear(context.Background(), m.run, "10.77.0.0/16"); err != nil {
t.Errorf("a clear range was refused: %v", err)
}
}
func TestAPredecessorsContainerUnderTheMeshsNameIsRefused(t *testing.T) {
m := machineRunner{unlabelled: map[string]bool{"mesh-registry": true}, labelled: map[string]bool{"mesh-store": true}}
err := NamesFree(context.Background(), m.run, []string{"mesh-store", "mesh-registry", "mesh-broker"}, store.State{})
if err == nil || !strings.Contains(err.Error(), "mesh-registry") || strings.Contains(err.Error(), "mesh-store") {
t.Fatalf("names: %v", err)
}
known := store.State{Resources: []store.Applied{{ID: "x", Type: "container", Target: "mesh-registry"}}}
if err := NamesFree(context.Background(), m.run, []string{"mesh-registry"}, known); err != nil {
t.Errorf("a container this node has a record of was refused: %v", err)
}
}
// controlRecorder is a control plane that answers everything and writes down what it was told,
// with the content of every settings file carried to it.
type controlRecorder struct {
told []string
settings map[string]string
}
func (c *controlRecorder) run(_ context.Context, name string, args ...string) (string, error) {
if name == "docker" && args[0] == "cp" {
raw, _ := os.ReadFile(args[1])
if strings.HasSuffix(args[2], "-settings.json") {
c.settings[filepath.Base(args[2])] = string(raw)
}
return "", nil
}
if name == "docker" && args[0] == "exec" {
c.told = append(c.told, strings.Join(args[3:], " "))
}
return "", nil
}
func (c *controlRecorder) index(prefix string) int {
for i, t := range c.told {
if strings.HasPrefix(t, prefix) {
return i
}
}
return -1
}
func TestTheNodesPortsAreSetBeforeTheModuleIsPushed(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
c := &controlRecorder{settings: map[string]string{}}
control := controlPlane{container: "temp-mesh-controller", run: c.run, timeout: time.Second}
o := Options{Node: "anchor", Ports: FoundationPorts{Registry: 5100}, Wait: time.Second}
if _, err := installModule(context.Background(), o, control, RegistryModule, []byte(`{}`), quietly); err != nil {
t.Fatal(err)
}
set, push := c.index("settings set distribution"), c.index("push anchor")
if set < 0 || push < 0 || set > push {
t.Fatalf("settings were not set before the push: %v", c.told)
}
if add := c.index("module add"); add > set {
t.Errorf("settings were set before the module existed: %v", c.told)
}
if !strings.Contains(c.told[set], "--node anchor") {
t.Errorf("the settings are not the node's: %s", c.told[set])
}
if got := c.settings["distribution-settings.json"]; got != `{"ports":{"5000":5100}}` {
t.Errorf("the registry was told %s", got)
}
}
func TestAGenesisOnTheDefaultsSetsNoSettings(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
c := &controlRecorder{settings: map[string]string{}}
control := controlPlane{container: "temp-mesh-controller", run: c.run, timeout: time.Second}
o := Options{Node: "anchor", Wait: time.Second}
if _, err := installModule(context.Background(), o, control, RegistryModule, []byte(`{}`), quietly); err != nil {
t.Fatal(err)
}
if c.index("settings") >= 0 {
t.Errorf("a converged genesis on the default ports set settings: %v", c.told)
}
}
func TestARerunOfGenesisFindsItsOwnPackageRegistry(t *testing.T) {
// Raised by genesis itself with no label and no record, so a re-run finds it unlabelled.
m := machineRunner{unlabelled: map[string]bool{giteaBootstrap: true}}
rerun := store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}}
if err := NamesFree(context.Background(), m.run, []string{giteaBootstrap, "mesh-store"}, rerun); err != nil {
t.Errorf("a re-run refused the package registry genesis raised: %v", err)
}
// On a machine genesis never ran on, a container under that name is a predecessor's.
if err := NamesFree(context.Background(), m.run, []string{giteaBootstrap}, store.State{}); err == nil {
t.Error("a container under the package registry's name on a fresh machine was not refused")
}
}
+5
View File
@@ -105,6 +105,11 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte
if err := waitForRuntime(ctx, d.Run, o.Timeout, o.Wait, say); err != nil {
return nil, err
}
// A converged genesis refuses a machine in use (novox/hq ADR 0100) — asked once the runtime
// answers, so what it runs can be counted, and before anything changes.
if err := RefuseAMachineInUse(ctx, o, d.Run, say); err != nil {
return nil, err
}
// 4. Can this machine reach what the bundle's images come from?
//
+18 -7
View File
@@ -132,18 +132,29 @@ func RetireTheTemporaryControlPlane(ctx context.Context, o Options, sys system.S
// where the comment explaining it lives. A comment that outlives the thing it describes is worse
// than no comment: it is the file telling somebody the machine has a control plane it does not.
func removeResource(bundle []byte, id string) ([]byte, error) {
previous, from, to, err := resourceAt(bundle, id)
if err != nil {
return nil, err
}
return cut(bundle, previous, from, to), nil
}
// resourceAt finds one resource's object in a bundle's text by its id: where the one before it
// ended, and where it starts and ends — comments and strings skipped, so an id quoted in a comment
// or a command is never mistaken for the resource.
func resourceAt(bundle []byte, id string) (previous, from, to int, err error) {
array := indexOutsideStrings(bundle, `"resources"`)
if array < 0 {
return nil, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it")
return 0, 0, 0, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it")
}
open := indexOutsideStrings(bundle[array:], "[")
if open < 0 {
return nil, fmt.Errorf("this bundle's resources are not a list")
return 0, 0, 0, fmt.Errorf("this bundle's resources are not a list")
}
open += array
depth, from := 0, -1
previous := open
depth := 0
from, previous = -1, open
inString, escaped, inLine, inBlock := false, false, false, false
for i := open + 1; i < len(bundle); i++ {
c := bundle[i]
@@ -181,16 +192,16 @@ func removeResource(bundle []byte, id string) ([]byte, error) {
break
}
if isResource(bundle[from:i+1], id) {
return cut(bundle, previous, from, i+1), nil
return previous, from, i + 1, nil
}
previous = i + 1
from = -1
case c == ']' && depth == 0:
return nil, fmt.Errorf(
return 0, 0, 0, fmt.Errorf(
"this bundle declares no %q, so there is nothing to take out of it", id)
}
}
return nil, fmt.Errorf("this bundle's resources list does not end")
return 0, 0, 0, fmt.Errorf("this bundle's resources list does not end")
}
// isResource reports whether one resource's text is the one wanted.
+12
View File
@@ -0,0 +1,12 @@
udp UNCONN 0 0 0.0.0.0:5353 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=17))
udp UNCONN 0 0 0.0.0.0:5355 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=13))
udp UNCONN 0 0 127.0.0.54:53 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=24))
udp UNCONN 0 0 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=22))
udp UNCONN 0 0 [::]:5353 [::]:* users:(("systemd-resolve",pid=262,fd=18))
udp UNCONN 0 0 [::]:5355 [::]:* users:(("systemd-resolve",pid=262,fd=15))
udp UNCONN 0 0 [fe80::1266:6aff:fe24:628d]%enp5s0:546 [::]:* users:(("systemd-network",pid=272,fd=36))
tcp LISTEN 0 4096 127.0.0.1:39473 0.0.0.0:* users:(("containerd",pid=394,fd=14))
tcp LISTEN 0 4096 0.0.0.0:5355 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=14))
tcp LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=23))
tcp LISTEN 0 4096 127.0.0.54:53 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=25))
tcp LISTEN 0 4096 [::]:5355 [::]:* users:(("systemd-resolve",pid=262,fd=16))
+107
View File
@@ -0,0 +1,107 @@
package declaration
import (
"strings"
"testing"
)
// Defends novox/hq ADR 0100: every declaration says whether the node is adopted and which of its
// modules are taken, and the host refuses one it cannot read that from unambiguously.
const adoptedResources = `"resources":[
{"id":"hello-web.page","type":"file","path":"/var/lib/hello-web/index.html","content":"a\n"},
{"id":"hello-web.server","type":"container","name":"hello-web","image":"sha256:` + sixtyFour + `"},
{"id":"hello-web.data","type":"directory","path":"/var/lib/hello-web"}
]`
const sixtyFour = "0000000000000000000000000000000000000000000000000000000000000000"
func TestAnAdoptionIsReadWithTheDeclaration(t *testing.T) {
d, err := Parse([]byte(`{"adoption":{"taken":["postgres"],"untaken":{"hello-web":["hello-web.page","hello-web.server"]}},
"declaration":1,` + adoptedResources + `}`))
if err != nil {
t.Fatal(err)
}
if d.Adoption == nil {
t.Fatal("the adoption was dropped")
}
if len(d.Adoption.Taken) != 1 || d.Adoption.Taken[0] != "postgres" {
t.Errorf("taken read as %v", d.Adoption.Taken)
}
if module, ok := d.Adoption.UntakenModuleOf("hello-web.server"); !ok || module != "hello-web" {
t.Errorf("the container's untaken module read as %q, %v", module, ok)
}
if _, ok := d.Adoption.UntakenModuleOf("hello-web.data"); ok {
t.Error("a resource the adoption does not name was said to be untaken")
}
}
func TestADeclarationWithNoAdoptionIsConverged(t *testing.T) {
d, err := Parse([]byte(`{"declaration":1,` + adoptedResources + `}`))
if err != nil {
t.Fatal(err)
}
if d.Adoption != nil {
t.Errorf("a declaration saying nothing about adoption read as adopted: %+v", d.Adoption)
}
if _, ok := d.Adoption.UntakenModuleOf("hello-web.page"); ok {
t.Error("a converged node has an untaken module")
}
}
func TestAnAdoptionNamingAnUnknownIDIsRefused(t *testing.T) {
refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.missing"]}},
"declaration":1,`+adoptedResources+`}`)
if !strings.Contains(strings.Join(refusal.Problems, "\n"), "hello-web.missing") {
t.Errorf("the unknown id was not named: %v", refusal.Problems)
}
}
func TestAnAdoptionMayNameAResourceOfAnyKind(t *testing.T) {
// A directory, a service or an action can reach what was found as surely as a file can, so
// the controller lists every resource of an untaken module (novox/hq ADR 0103).
d, err := Parse([]byte(`{"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.data"]}},
"declaration":1,` + adoptedResources + `}`))
if err != nil {
t.Fatalf("a directory of an untaken module was refused: %v", err)
}
if module, ok := d.Adoption.UntakenModuleOf("hello-web.data"); !ok || module != "hello-web" {
t.Errorf("the directory is not its module's: %q %v", module, ok)
}
}
func TestAnIDUnderTwoModulesIsRefused(t *testing.T) {
refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"a":["hello-web.page"],"b":["hello-web.page"]}},
"declaration":1,`+adoptedResources+`}`)
if !strings.Contains(strings.Join(refusal.Problems, "\n"), "both") {
t.Errorf("an id under two modules was accepted: %v", refusal.Problems)
}
}
func TestAModuleBothTakenAndUntakenIsRefused(t *testing.T) {
refusal := refusalFor(t, `{"adoption":{"taken":["hello-web"],"untaken":{"hello-web":["hello-web.page"]}},
"declaration":1,`+adoptedResources+`}`)
if !strings.Contains(strings.Join(refusal.Problems, "\n"), "both taken and untaken") {
t.Errorf("a module both taken and untaken was accepted: %v", refusal.Problems)
}
}
func TestTheMeshsOwnResourcesAreNeverUntaken(t *testing.T) {
refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"x":["adoption.guard"]}},
"declaration":1,"resources":[
{"id":"adoption.guard","type":"file","path":"/etc/mesh/guard.nft","content":"x"}]}`)
if !strings.Contains(strings.Join(refusal.Problems, "\n"), "belongs to no module") {
t.Errorf("an adoption. id was accepted as untaken: %v", refusal.Problems)
}
}
func TestAnAdoptionWithAnUnknownFieldIsRefused(t *testing.T) {
refusalFor(t, `{"adoption":{"taken":[],"held":["x"]},"declaration":1,`+adoptedResources+`}`)
}
func TestACarriedBundleCannotSayTheNodeIsAdopted(t *testing.T) {
_, err := ParseTrusted([]byte(`{"adoption":{"taken":[]},"declaration":1,` + adoptedResources + `}`))
if err == nil || !strings.Contains(err.Error(), "only the mesh can say") {
t.Fatalf("a bundle claiming adoption was not refused: %v", err)
}
}
+219 -2
View File
@@ -78,6 +78,12 @@ const (
// cadence. Tools, hooks and event consumers are not separate modes: they are loaded by a tool
// host, which is itself a process that stays up.
TypeProcess Type = "process"
// TypeOpening is a port the mesh needs reachable on an adopted node, converged through the
// firewall found there in that firewall's own terms (novox/hq ADR 0100). A state, not a
// command: the host adds the rule it marks as the mesh's when it is missing, and removes only
// what it marked — which is what lets it travel over the link.
TypeOpening Type = "opening"
)
// Resource is one thing that should be true of the machine.
@@ -148,6 +154,13 @@ type File struct {
// (ADR 0030), and it does not overwrite that either.
CreateOnce bool `json:"create-once,omitempty"`
// Into says the file is shared with software the mesh did not install, and the content is
// the mesh's part of it: written into what is there, never over it (novox/hq ADR 0102). Only
// "json" is spoken — the content is a JSON object whose keys the host sets in the file's
// object, keeping every other key as it found it and recording what each of its keys held
// before, so undeclaring the file gives those back.
Into string `json:"into,omitempty"`
// Sealed is content encrypted to this node's sealing key, for a file the mesh must deliver
// without being able to read.
//
@@ -218,6 +231,24 @@ func (f *File) validate(where string, _ bool) []string {
if f.Path == "" {
problems = append(problems, where+": a file needs a path")
}
switch f.Into {
case "":
case IntoJSON:
var object map[string]json.RawMessage
if err := json.Unmarshal([]byte(f.Content), &object); err != nil || object == nil {
problems = append(problems, where+
": a file written into JSON carries a JSON object of the keys it sets")
}
if f.Sealed != "" || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce {
problems = append(problems, where+
": a file written into says only its keys, in content — not sealed, bytes, "+
"secrets or create-once")
}
default:
problems = append(problems, fmt.Sprintf(
"%s: into %q; a file is written into \"json\", or omits it to be written whole",
where, f.Into))
}
var said []string
for name, value := range map[string]string{
"content": f.Content, "sealed": f.Sealed, "bytes": f.Bytes,
@@ -580,6 +611,12 @@ type Service struct {
// would be an action, and the link may not carry one (novox/hq ADR 0005) — so this is not a
// way around that rule, it is the shape the rule leaves.
RestartOn []string `json:"restart-on,omitempty"`
// ReloadOn names resources whose change means this service must be reloaded — for a service
// that re-reads its configuration when told to, where a restart would stop what it runs: the
// container runtime, whose restart stops every container on the machine (novox/hq ADR 0102).
// A change that is also in RestartOn restarts it, which covers a reload.
ReloadOn []string `json:"reload-on,omitempty"`
}
func (s *Service) Identity() string { return s.ID }
@@ -603,6 +640,77 @@ func (s *Service) validate(where string, _ bool) []string {
return problems
}
// IntoJSON is the one structured format a file is written into.
const IntoJSON = "json"
// Opening is a port reachable on an adopted node, from where, and on which path.
//
// **From** is everywhere or mesh — the private network, by its interface. **Path** is incoming,
// for something listening on the machine, or forwarded, for a published container port: the found
// firewall sees a published port after the runtime has translated it, so a forwarded opening names
// the container's own port in To as well as the machine's in Port.
type Opening struct {
ID string `json:"id"`
Type Type `json:"type"`
Port int `json:"port"`
Protocol string `json:"protocol"`
From string `json:"from"`
Path string `json:"path"`
To int `json:"to,omitempty"`
}
// Where an opening admits from, and the path it is on.
const (
FromEverywhere = "everywhere"
FromMesh = "mesh"
PathIncoming = "incoming"
PathForwarded = "forwarded"
)
func (o *Opening) Identity() string { return o.ID }
func (o *Opening) Kind() Type { return TypeOpening }
func (o *Opening) Target() string {
if o.Path == PathForwarded {
return fmt.Sprintf("%s/%d forwarded to %d from %s", o.Protocol, o.Port, o.To, o.From)
}
return fmt.Sprintf("%s/%d %s from %s", o.Protocol, o.Port, o.Path, o.From)
}
func (o *Opening) validate(where string, _ bool) []string {
var problems []string
if o.Port < 1 || o.Port > 65535 {
problems = append(problems, fmt.Sprintf("%s: an opening's port is 1-65535, not %d", where, o.Port))
}
if o.Protocol != "tcp" && o.Protocol != "udp" {
problems = append(problems, fmt.Sprintf("%s: an opening is tcp or udp, not %q", where, o.Protocol))
}
if o.From != FromEverywhere && o.From != FromMesh {
problems = append(problems, fmt.Sprintf(
"%s: an opening is from %q or %q, not %q", where, FromEverywhere, FromMesh, o.From))
}
switch o.Path {
case PathIncoming:
if o.To != 0 {
problems = append(problems, where+
": an incoming opening names no container port; only a forwarded one does")
}
case PathForwarded:
if o.To < 1 || o.To > 65535 {
problems = append(problems, where+
": a forwarded opening names the container's port it reaches, as to, 1-65535")
}
default:
problems = append(problems, fmt.Sprintf(
"%s: an opening's path is %q or %q, not %q", where, PathIncoming, PathForwarded, o.Path))
}
if !strings.HasPrefix(o.ID, AdoptionPrefix) {
problems = append(problems, fmt.Sprintf(
"%s: an opening is the mesh's own, so its id starts %q", where, AdoptionPrefix))
}
return problems
}
// Package is a package that should be present.
//
// Present is the whole of what it asserts, never a version: version is the package manager's
@@ -810,6 +918,8 @@ func newOf(t Type) Resource {
return &Access{}
case TypeProcess:
return &Process{}
case TypeOpening:
return &Opening{}
}
return nil
}
@@ -818,7 +928,7 @@ func newOf(t Type) Resource {
func Vocabulary() []Type {
return []Type{
TypeAccess, TypeAction, TypeArchive, TypeContainer, TypeDirectory, TypeFile,
TypeNetwork, TypePackage, TypeProcess, TypeService, TypeUser,
TypeNetwork, TypeOpening, TypePackage, TypeProcess, TypeService, TypeUser,
}
}
@@ -832,6 +942,99 @@ type Declaration struct {
// Resources, in the order they are applied. The host does not sort them: ordering is a
// decision, and deciding is not what the host does (novox/hq ADR 0005).
Resources []Resource
// Adoption says this node is adopted, and which of its modules have been taken. Nil is a
// converged node — which is every node the mesh raised before adoption existed, and so the
// only form an older controller ever sends (novox/hq ADR 0100).
Adoption *Adoption
}
// Adoption is a node's mode, as the controller records it: the node is adopted, and these are
// the modules taken on it so far (novox/hq ADR 0100).
//
// **Authoritative, and only ever stated by the controller.** A host does not work out whether it
// is adopted; it is told, in every declaration, so a host restarted from the declaration it kept
// is in the same mode it was in before.
//
// Untaken names, per module assigned here and not yet taken, the ids of its resources. Any kind
// may be listed: a file, a directory, a service's unit or a container can already be on the
// machine, and an action run inside a held container reaches what was found (novox/hq ADR 0103);
// the host decides per kind what can be held. The host cannot split a resource id into its
// module, because module names may contain dots, so the controller says which ids belong to which
// module rather than leaving the host to guess.
type Adoption struct {
Taken []string `json:"taken"`
Untaken map[string][]string `json:"untaken,omitempty"`
}
// AdoptionPrefix is the id prefix of what the mesh itself declares because a node is adopted —
// its openings and its guard. Nothing under it belongs to a module, so none of it is ever held.
const AdoptionPrefix = "adoption."
// UntakenModuleOf says which untaken module declares a resource, if any.
func (a *Adoption) UntakenModuleOf(id string) (string, bool) {
if a == nil {
return "", false
}
for module, ids := range a.Untaken {
if slices.Contains(ids, id) {
return module, true
}
}
return "", false
}
// checkAdoption holds what an adoption says against the resources beside it. Every problem is a
// refusal: a host that misread which module is untaken would replace a predecessor's service the
// operator never took.
func checkAdoption(a *Adoption, resources []Resource, allowActions bool) []string {
if a == nil {
return nil
}
if allowActions {
// The bundle is carried with the binary and raises a foundation before any mesh exists.
// Whether a node is adopted is the controller's record, and a bundle that claimed it would
// be the host deciding its own mode (novox/hq ADR 0100).
return []string{"a carried bundle says the node is adopted, and only the mesh can say " +
"that: a node's mode is the controller's record, sent in every declaration"}
}
kinds := map[string]Type{}
for _, r := range resources {
kinds[r.Identity()] = r.Kind()
}
var problems []string
for _, module := range a.Taken {
if _, both := a.Untaken[module]; both {
problems = append(problems, fmt.Sprintf(
"adoption: the module %q is said to be both taken and untaken", module))
}
}
owner := map[string]string{}
modules := make([]string, 0, len(a.Untaken))
for module := range a.Untaken {
modules = append(modules, module)
}
sort.Strings(modules)
for _, module := range modules {
for _, id := range a.Untaken[module] {
if strings.HasPrefix(id, AdoptionPrefix) {
problems = append(problems, fmt.Sprintf(
"adoption: %q is the mesh's own and belongs to no module, so it cannot be untaken", id))
continue
}
if first, twice := owner[id]; twice {
problems = append(problems, fmt.Sprintf(
"adoption: %q is said to belong to both %q and %q", id, first, module))
continue
}
owner[id] = module
if _, declared := kinds[id]; !declared {
problems = append(problems, fmt.Sprintf(
"adoption: %q of the untaken module %q is not in this declaration", id, module))
}
}
}
return problems
}
// RefusalError refuses a whole declaration, naming every problem at once.
@@ -872,6 +1075,7 @@ func ParseTrusted(raw []byte) (*Declaration, error) { return parse(raw, true) }
type envelope struct {
Version int `json:"declaration"`
For string `json:"for,omitempty"`
Adoption *Adoption `json:"adoption,omitempty"`
Resources []json.RawMessage `json:"resources"`
}
@@ -889,7 +1093,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
env.Version, Version)}}
}
d := &Declaration{Version: env.Version, For: env.For}
d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption}
var problems []string
if len(env.Resources) == 0 {
@@ -952,6 +1156,19 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
problems = append(problems, resource.validate(where, allowActions)...)
d.Resources = append(d.Resources, resource)
}
problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...)
if env.Adoption == nil {
for _, r := range d.Resources {
if r.Kind() == TypeOpening {
// On a converged node the mesh's own filter admits what is declared, and the
// found firewall is retired; an opening there would be a rule in a firewall the
// mesh has disabled (novox/hq ADR 0100).
problems = append(problems, fmt.Sprintf(
"resource %q: an opening is for an adopted node, and this declaration does not "+
"say the node is adopted", r.Identity()))
}
}
}
if len(problems) > 0 {
return nil, &RefusalError{Problems: problems}
+8 -4
View File
@@ -266,7 +266,7 @@ func TestAFieldTheNewTypesDoNotUseIsRefused(t *testing.T) {
}
}
func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) {
func TestTheVocabularyIsTheTwelveShapesTheMeshNeeds(t *testing.T) {
// Six of them the bootstrap uses (novox/hq 07-the-foundation.md), and removing one is a
// failing test rather than a discovery during a first-node install.
//
@@ -282,7 +282,7 @@ func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) {
}
for _, want := range []Type{
TypeDirectory, TypeFile, TypeService, TypePackage, TypeContainer, TypeAction,
TypeUser, TypeArchive, TypeNetwork, TypeAccess, TypeProcess,
TypeUser, TypeArchive, TypeNetwork, TypeAccess, TypeProcess, TypeOpening,
} {
if !speaks[want] {
t.Errorf("the host no longer speaks %q", want)
@@ -309,8 +309,12 @@ func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) {
// It is a full-host shape rather than a portable one: it needs a process supervisor to install
// into. It does NOT need a container runtime, which is the point — only software that
// genuinely needs isolation asks for a container.
if len(speaks) != 11 {
t.Errorf("the vocabulary is %d shapes rather than 11; every addition widens what a compromised "+
//
// `opening` is the twelfth, and novox/hq ADR 0100 is its decision: on an adopted node the
// firewall found there stays in force, and what the mesh needs reachable is converged through
// it as a state the host marks as the mesh's — never a command, which the link may not carry.
if len(speaks) != 12 {
t.Errorf("the vocabulary is %d shapes rather than 12; every addition widens what a compromised "+
"control plane can express, so a change here is a decision: %s",
len(speaks), vocabulary())
}
+35
View File
@@ -0,0 +1,35 @@
package declaration
import (
"strings"
"testing"
)
// Defends novox/hq ADR 0102: a file written into carries only a JSON object of its keys, in a
// format the host speaks, and a service may name what it is reloaded on.
func TestAFileWrittenIntoIsRefusedUnlessItIsAnObjectOfKeys(t *testing.T) {
for name, c := range map[string]struct{ resource, refusal string }{
"another format": {`{"id":"f","type":"file","path":"/etc/x","into":"toml","content":"a = 1"}`, `into "toml"`},
"not an object": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"[1,2]"}`, "JSON object"},
"with create-once": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","create-once":true}`, "create-once"},
} {
_, err := Parse([]byte(`{"declaration":1,"resources":[` + c.resource + `]}`))
if err == nil || !strings.Contains(err.Error(), c.refusal) {
t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err)
}
}
d, err := Parse([]byte(`{"declaration":1,"resources":[
{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{\"k\":1}"},
{"id":"s","type":"service","unit":"docker.service","state":"running","reload-on":["f"]}
]}`))
if err != nil {
t.Fatal(err)
}
if f := d.Resources[0].(*File); f.Into != IntoJSON {
t.Errorf("into was read as %q", f.Into)
}
if s := d.Resources[1].(*Service); len(s.ReloadOn) != 1 || s.ReloadOn[0] != "f" {
t.Errorf("reload-on was read as %v", s.ReloadOn)
}
}
+929
View File
@@ -0,0 +1,929 @@
// Package firewall speaks the firewall found on an adopted node, in that firewall's own terms
// (novox/hq ADR 0100).
//
// **The found firewall stays in force.** On an adopted node the mesh loads nothing that drops by
// default or holds an accept; what it needs reachable it converges as openings through what it
// found, marks each rule as its own, and removes only what it marked. It never resets or flushes:
// the rules the machine already had are the operator's, and they are what keeps it serving.
package firewall
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os/exec"
"regexp"
"strconv"
"strings"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/system"
)
// Runner executes a command.
type Runner = system.Runner
// Kind is what firewall a machine has, as far as the mesh is concerned.
type Kind string
const (
// UFW is an active ufw — the one kind found on the machines measured, and the one spoken.
UFW Kind = "ufw"
// None is a machine where nothing refuses anything, which needs no openings.
None Kind = "none"
// Unsupported is a firewall no host speaks yet. A machine with one is refused adoption: the
// mesh could neither open what it needs nor know what it would be closing.
Unsupported Kind = "unsupported"
)
// deletion is the arguments that delete a rule as `ufw show added` printed it. A route rule is
// deleted with `route delete …`: ufw refuses `delete route …` as invalid syntax. And ufw answers
// success when asked to delete a rule it does not hold, so every deletion is read back.
func deletion(rule string) []string {
w := words(rule)
if len(w) > 0 && w[0] == "route" {
return append([]string{"route", "delete"}, w[1:]...)
}
return append([]string{"delete"}, w...)
}
// MeshInterface is the private network's interface, the way an opening from the mesh is known.
// It must be the controller's overlay interface name.
const MeshInterface = "mesh0"
// Detect says which firewall this machine has. For Unsupported the string names it.
func Detect(ctx context.Context, run Runner) (Kind, string, error) {
if out, err := run(ctx, "firewall-cmd", "--state"); err == nil && strings.TrimSpace(out) == "running" {
return Unsupported, "firewalld", nil
}
ufwActive := false
if out, err := run(ctx, "ufw", "status"); err == nil {
ufwActive = statusActive(out)
}
noNft := false
out, err := run(ctx, "nft", "list", "ruleset")
switch {
case err == nil:
if refusing := Refusing(out, ufwActive); len(refusing) > 0 {
return Unsupported, "nftables rules that refuse traffic, in " + strings.Join(refusing, ", "), nil
}
case missing(err):
// **No nft on this machine does not mean no rules.** iptables-nft writes tables nft would
// have shown, and a machine whose only tool is iptables answers about them through that.
// Read as "nothing filters here", a machine with an iptables firewall would be adopted
// with no openings and nothing would reach the mesh (novox/hq ADR 0100).
noNft = true
default:
return "", "", fmt.Errorf("cannot read this machine's packet filter to know what it has: %w", err)
}
if !ufwActive {
// iptables with the legacy backend is invisible to nft; and where nft is not installed,
// the iptables command is the only way to see anything at all.
tools := []string{"iptables-legacy", "ip6tables-legacy"}
if noNft {
tools = append(tools, "iptables", "ip6tables")
}
for _, legacy := range tools {
out, err := run(ctx, legacy, "-S")
if err != nil {
continue
}
if refusing := RefusingLegacy(out); len(refusing) > 0 {
return Unsupported, legacy + " rules that refuse traffic, in " + strings.Join(refusing, ", "), nil
}
}
}
if ufwActive {
return UFW, "ufw", nil
}
return None, "", nil
}
func missing(err error) bool {
return errors.Is(err, exec.ErrNotFound)
}
func statusActive(out string) bool {
for _, line := range strings.Split(out, "\n") {
if strings.HasPrefix(strings.TrimSpace(line), "Status:") {
return strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "Status:")) == "active"
}
}
return false
}
// Refusing names the tables of an `nft list ruleset` holding something that refuses traffic — a
// drop or reject, or a base chain whose policy drops — and that is neither the mesh's own nor the
// container runtime's. With ufw active, the tables iptables-nft manages are ufw's and the runtime's
// and are not counted.
//
// **A ban is not a firewall.** fail2ban refuses the sources it banned and passes everything else;
// captured on a lab machine with both of its backends (testdata/fail2ban-nftables.nft,
// testdata/fail2ban-iptables.nft). The mesh opens nothing through it and it closes nothing the
// mesh needs, so a refusal that names the sources it refuses, in a table or a chain that accepts
// nothing and is entered only from chains whose policy accepts, is not counted.
func Refusing(ruleset string, ufwActive bool) []string {
type rule struct{ table, chain, line string }
type chainOf struct {
base, dropping, accepts bool
policyLine string
jumpedFrom []string
}
chains := map[string]*chainOf{} // by "table\x00chain"
tableAccepts := map[string]bool{}
var tables []string
var refusals []rule
managed := map[string]bool{}
var table, chain string
get := func(t, c string) *chainOf {
k := t + "\x00" + c
if chains[k] == nil {
chains[k] = &chainOf{}
}
return chains[k]
}
for _, raw := range strings.Split(ruleset, "\n") {
line := strings.TrimSpace(raw)
switch {
case strings.HasPrefix(line, "# Warning: table ") && strings.Contains(line, "managed by iptables-nft"):
name := strings.TrimPrefix(line, "# Warning: table ")
name, _, _ = strings.Cut(name, " is managed")
managed[name] = true
continue
case strings.HasPrefix(line, "table "):
table = strings.TrimSuffix(strings.TrimSpace(strings.TrimPrefix(line, "table ")), "{")
table = strings.TrimSpace(table)
tables = append(tables, table)
chain = ""
continue
case strings.HasPrefix(line, "chain "):
chain = strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(line, "chain "), "{"))
get(table, chain)
continue
case strings.HasPrefix(line, "set ") || strings.HasPrefix(line, "map ") ||
strings.HasPrefix(line, "flowtable "):
chain = ""
continue
case line == "" || line == "}" || strings.HasPrefix(line, "#") || chain == "":
continue
}
c := get(table, chain)
if strings.HasPrefix(line, "type ") {
c.base = true
c.policyLine = line
c.dropping = strings.Contains(line, "policy drop")
continue
}
for _, verb := range []string{"jump ", "goto "} {
if i := strings.Index(line, verb); i >= 0 {
target := strings.Fields(line[i+len(verb):])
if len(target) > 0 {
get(table, target[0]).jumpedFrom = append(get(table, target[0]).jumpedFrom, chain)
}
}
}
if accepts(line) {
c.accepts = true
tableAccepts[table] = true
}
if verdictRefuses(line) {
refusals = append(refusals, rule{table, chain, line})
}
}
skipped := func(table string) bool {
if table == "inet mesh" || table == "inet mesh_guard" {
return true
}
return (managed[table] || iptablesTable(table)) && ufwActive
}
// onlyBans is whether a refusal only refuses the sources it names: in a table that accepts
// nothing and whose base chains all accept by default, or in a chain that accepts nothing and
// is entered only from base chains that accept by default.
onlyBans := func(r rule) bool {
if !bansSources(r.line) {
return false
}
allAccepting := true
for k, c := range chains {
if strings.HasPrefix(k, r.table+"\x00") && c.base && !strings.Contains(c.policyLine, "policy accept") {
allAccepting = false
}
}
if !tableAccepts[r.table] && allAccepting {
return true
}
c := get(r.table, r.chain)
if c.base || c.accepts || len(c.jumpedFrom) == 0 {
return false
}
for _, from := range c.jumpedFrom {
caller := get(r.table, from)
if !caller.base || !strings.Contains(caller.policyLine, "policy accept") {
return false
}
}
return true
}
counted := map[string]bool{}
for k, c := range chains {
t, name, _ := strings.Cut(k, "\x00")
if skipped(t) || !c.dropping {
continue
}
if (managed[t] || iptablesTable(t)) && runtimes(t, name, c.policyLine) {
continue
}
counted[t] = true
}
for _, r := range refusals {
if skipped(r.table) || counted[r.table] {
continue
}
if (managed[r.table] || iptablesTable(r.table)) && runtimes(r.table, r.chain, r.line) {
continue
}
if onlyBans(r) {
continue
}
counted[r.table] = true
}
var refusing []string
for _, t := range tables {
if counted[t] {
counted[t] = false
refusing = append(refusing, "table "+t)
}
}
return refusing
}
// iptablesTable is whether a table is one iptables-nft writes. Named rather than read from the
// warning nft prints above it, because nft does not print that for every such table: a captured
// ruleset carried it on ip filter and not on ip raw, where the runtime keeps its drops.
func iptablesTable(table string) bool {
family, name, _ := strings.Cut(table, " ")
if family != "ip" && family != "ip6" {
return false
}
switch name {
case "filter", "nat", "raw", "mangle", "security":
return true
}
return false
}
// runtimes is whether a refusal in an iptables-nft table is the container runtime's own: in its
// DOCKER chains, its forward policy, or its guard against reaching a container's address directly
// from outside its bridge, in the raw table.
func runtimes(table, chain, line string) bool {
_, name, _ := strings.Cut(table, " ")
switch {
case strings.HasPrefix(chain, "DOCKER"):
return true
case name == "filter" && chain == "FORWARD" && strings.HasPrefix(line, "type "):
return true
case name == "raw" && chain == "PREROUTING":
return strings.Contains(line, "daddr") && strings.Contains(line, "iifname !=")
}
return false
}
// iptables-nft prints a REJECT target it cannot translate as `xt target "REJECT"`, measured in
// testdata/fail2ban-iptables.nft; a refusal written that way is a refusal too.
var verdict = regexp.MustCompile(`(^|\s)(drop|reject)(\s|$)|xt target "(DROP|REJECT)"`)
func verdictRefuses(line string) bool {
return verdict.MatchString(line)
}
var acceptVerdict = regexp.MustCompile(`(^|\s)accept(\s|;|$)|xt target "ACCEPT"`)
func accepts(line string) bool {
return acceptVerdict.MatchString(line)
}
// bansSources is whether a refusal names the sources it refuses — a set or an address — rather
// than refusing everyone but some.
func bansSources(line string) bool {
f := strings.Fields(line)
for i, w := range f {
if (w == "saddr" || w == "-s") && i+1 < len(f) && f[i+1] != "!=" && !strings.HasPrefix(f[i+1], "!") {
return i == 0 || f[i-1] != "!"
}
}
return false
}
// RefusingLegacy names the chains of an `iptables-legacy -S` that refuse traffic outside the
// container runtime's own. A ban — a refusal of the sources it names, in a chain that accepts
// nothing and is entered only from built-in chains whose policy accepts — is not counted, as in
// Refusing (testdata/fail2ban-iptables-S.txt).
func RefusingLegacy(rules string) []string {
policy := map[string]string{}
accepting := map[string]bool{}
jumpedFrom := map[string][]string{}
for _, line := range strings.Split(rules, "\n") {
fields := strings.Fields(line)
if len(fields) < 3 {
continue
}
switch fields[0] {
case "-P":
policy[fields[1]] = fields[2]
case "-A":
for i, f := range fields {
if (f == "-j" || f == "-g") && i+1 < len(fields) {
switch fields[i+1] {
case "ACCEPT":
accepting[fields[1]] = true
case "DROP", "REJECT", "RETURN", "LOG":
default:
jumpedFrom[fields[i+1]] = append(jumpedFrom[fields[i+1]], fields[1])
}
}
}
}
}
ban := func(chain, line string) bool {
if !bansSources(line) || accepting[chain] || len(jumpedFrom[chain]) == 0 {
return false
}
for _, from := range jumpedFrom[chain] {
if policy[from] != "ACCEPT" {
return false
}
}
return true
}
var refusing []string
seen := map[string]bool{}
for _, line := range strings.Split(rules, "\n") {
fields := strings.Fields(line)
if len(fields) < 3 {
continue
}
chain := fields[1]
refuses := false
switch fields[0] {
case "-P":
refuses = fields[2] == "DROP" && chain != "FORWARD"
case "-A":
for i, f := range fields {
if f == "-j" && i+1 < len(fields) && (fields[i+1] == "DROP" || fields[i+1] == "REJECT") {
refuses = !strings.HasPrefix(chain, "DOCKER") && !ban(chain, line)
}
}
}
if refuses && !seen[chain] {
seen[chain] = true
refusing = append(refusing, "chain "+chain)
}
}
return refusing
}
// --- ufw ---------------------------------------------------------------------------------------
// Mark is the comment every rule the mesh adds carries: whose it is, which opening, and a digest
// of the rule itself, so a rule the opening no longer describes is recognised as stale without the
// host having to know how ufw prints a rule back.
func Mark(o *declaration.Opening) string {
sum := sha256.Sum256([]byte(strings.Join(Rule(o), " ")))
return marker(o.ID) + " " + hex.EncodeToString(sum[:])[:8]
}
func marker(id string) string { return "mesh-host " + id }
// markedFor is whether a comment is the mesh's, for this opening.
func markedFor(comment, id string) bool {
return comment == marker(id) || strings.HasPrefix(comment, marker(id)+" ")
}
// Rule is the ufw rule an opening becomes, without its comment.
//
// incoming from everywhere allow proto tcp to any port P
// incoming from the mesh allow in on mesh0 proto tcp to any port P
// forwarded route allow [in on mesh0] proto tcp to any port <container port>
//
// A forwarded opening names the container's port because ufw's route rules are matched after the
// runtime's destination translation.
func Rule(o *declaration.Opening) []string {
var rule []string
port := o.Port
if o.Path == declaration.PathForwarded {
rule = append(rule, "route")
port = o.To
}
rule = append(rule, "allow")
if o.From == declaration.FromMesh {
rule = append(rule, "in", "on", MeshInterface)
}
return append(rule, "proto", o.Protocol, "to", "any", "port", strconv.Itoa(port))
}
var commentOf = regexp.MustCompile(`comment '([^']*)'`)
// added is every rule `ufw show added` lists, each without its leading "ufw".
func added(ctx context.Context, run Runner) ([]string, error) {
out, err := run(ctx, "ufw", "show", "added")
if err != nil {
return nil, fmt.Errorf("reading ufw's rules: %w", err)
}
var rules []string
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "ufw ") {
rules = append(rules, strings.TrimPrefix(line, "ufw "))
}
}
return rules, nil
}
func comment(rule string) string {
m := commentOf.FindStringSubmatch(rule)
if m == nil {
return ""
}
return m[1]
}
// words splits a rule as ufw printed it into arguments, keeping a quoted comment whole.
func words(rule string) []string {
var out []string
var cur strings.Builder
quoted, any := false, false
for _, r := range rule {
switch {
case r == '\'':
quoted = !quoted
any = true
case r == ' ' && !quoted:
if any {
out = append(out, cur.String())
cur.Reset()
any = false
}
default:
cur.WriteRune(r)
any = true
}
}
if any {
out = append(out, cur.String())
}
return out
}
// A ufw rule, as `ufw show added` prints it or as it is given, reduced to what ufw compares.
//
// **ufw treats two rules that differ only in their comment as one rule.** Measured on a lab
// machine (testdata/ufw-comment-only.txt): adding `route allow proto tcp to any port 8080 comment
// 'mesh-host …'` beside an operator's `route allow 8080/tcp` answers "Rule updated", and the
// operator's rule now carries the mesh's mark — so removing the opening later would delete the
// operator's rule. The same holds for an incoming rule and for one with a comment of its own.
type ufwRule struct {
route bool
action, in, out string
// dir is which way the rule matches: "" (ufw's default, incoming and forwarded), "in" or
// "out". A rule on the outgoing path admits nothing that arrives.
dir string
log string
from, fromPort, to string
port, proto, app string
comment string
}
// parseRule reads a rule in either of ufw's forms — the short `allow 22/tcp` and the long `allow
// in on mesh0 to any port 5432 proto tcp` — into the fields ufw compares. Not ok for anything it
// does not recognise, which is then never taken to answer an opening.
func parseRule(rule string) (ufwRule, bool) {
r := ufwRule{from: "any", to: "any", comment: comment(rule)}
// A log type may stand after the action or after the direction; either way it is a property
// of the rule, not of where it matches. The word after `comment` is the comment, whatever it
// says.
var w []string
all := words(rule)
for i := 0; i < len(all); i++ {
switch {
case all[i] == "comment" && i+1 < len(all):
w = append(w, all[i], all[i+1])
i++
case all[i] == "log" || all[i] == "log-all":
r.log = all[i]
default:
w = append(w, all[i])
}
}
i := 0
if i < len(w) && w[i] == "route" {
r.route = true
i++
}
if i >= len(w) {
return r, false
}
switch w[i] {
case "allow", "deny", "reject", "limit":
r.action = w[i]
default:
return r, false
}
i++
for i < len(w) && (w[i] == "in" || w[i] == "out") {
dir := w[i]
i++
iface := ""
if i+1 < len(w) && w[i] == "on" {
iface = w[i+1]
i += 2
}
r.dir = dir
if dir == "in" {
r.in = iface
} else {
r.out = iface
}
}
if i < len(w) && w[i] != "from" && w[i] != "to" && w[i] != "proto" && w[i] != "comment" &&
w[i] != "app" && w[i] != "log" && w[i] != "log-all" {
// The short form: a port with its protocol, a bare port, or an application's name.
port, proto, hasProto := strings.Cut(w[i], "/")
if isPorts(port) {
r.port = port
if hasProto {
r.proto = proto
}
} else {
r.app = w[i]
}
i++
}
for ; i < len(w); i++ {
next := func() string {
if i+1 < len(w) {
i++
return w[i]
}
return ""
}
switch w[i] {
case "from":
r.from = next()
if i+1 < len(w) && w[i+1] == "port" {
i++
r.fromPort = next()
}
case "to":
r.to = next()
if i+1 < len(w) && w[i+1] == "port" {
i++
r.port = next()
}
case "port":
r.port = next()
case "proto":
r.proto = next()
case "app":
r.app = next()
case "comment":
next()
case "log", "log-all":
default:
return r, false
}
}
if r.proto == "any" {
r.proto = ""
}
// Incoming is ufw's default direction, and it prints `allow in 9005/tcp` back as
// `allow 9005/tcp` and merges the two — captured in testdata/ufw-direction.txt. An outgoing
// rule is its own rule and stays one.
if r.dir == "in" && r.in == "" {
r.dir = ""
}
return r, true
}
func isPorts(s string) bool {
if s == "" {
return false
}
for _, c := range s {
if (c < '0' || c > '9') && c != ':' && c != ',' {
return false
}
}
return true
}
// sameAs is whether ufw would take two rules for one: everything equal but the comment, the
// action and the log type. Adding one beside the other updates it in place — its comment, and its
// action or log type — rather than adding a second.
func (r ufwRule) sameAs(o ufwRule) bool {
r.comment, o.comment = "", ""
r.action, o.action = "", ""
r.log, o.log = "", ""
return r == o
}
// admits is whether a rule already lets through what an opening says: the same path, allowed from
// any source to any address of this machine, on the opening's port and protocol — or on any
// protocol — and on any interface, or the private network's for an opening from it.
func (r ufwRule) admits(o *declaration.Opening) bool {
want, ok := parseRule(strings.Join(Rule(o), " "))
if !ok || r.route != want.route || r.action != "allow" || r.app != "" ||
r.from != "any" || r.fromPort != "" || r.to != "any" {
return false
}
// A rule on the outgoing path lets this machine reach others; it admits nothing that arrives,
// so it never answers an opening.
if r.dir == "out" || r.out != "" {
return false
}
if r.proto != "" && r.proto != want.proto {
return false
}
if r.in != "" && r.in != want.in {
return false
}
return portsInclude(r.port, want.port)
}
// portsInclude is whether a ufw port list — 80, 80,443, or 8000:8100 — names a port.
func portsInclude(list, port string) bool {
p, err := strconv.Atoi(port)
if err != nil {
return false
}
for _, part := range strings.Split(list, ",") {
lo, hi, isRange := strings.Cut(part, ":")
a, err := strconv.Atoi(lo)
if err != nil {
continue
}
b := a
if isRange {
if b, err = strconv.Atoi(hi); err != nil {
continue
}
}
if a <= p && p <= b {
return true
}
}
return false
}
// Converged is what converging an opening did. SatisfiedBy names the rule already there that
// answers the opening, when one does; the mesh then adds nothing, and so will remove nothing.
type Converged struct {
Action string
SatisfiedBy string
}
// Converge makes one opening true in ufw: its marked rule present, and any rule marked for it that
// no longer describes it deleted. Nothing unmarked is touched. The outcome is created, updated or
// unchanged, read back from ufw rather than assumed.
//
// **An opening a rule already answers is not added** (novox/hq ADR 0103). If ufw holds a rule not
// marked for this opening that already admits what it says — the operator's, or one the mesh
// added for another opening — the opening is satisfied by it: adding the mesh's would take that
// rule over if it differs only in its comment, and removing the opening would then delete it.
func Converge(ctx context.Context, run Runner, o *declaration.Opening) (Converged, error) {
rules, err := added(ctx, run)
if err != nil {
return Converged{}, err
}
mark := Mark(o)
present := false
var stale []string
satisfiedBy := ""
want, _ := parseRule(strings.Join(Rule(o), " "))
for _, rule := range rules {
c := comment(rule)
switch {
case c == mark:
present = true
case markedFor(c, o.ID):
stale = append(stale, rule)
default:
parsed, ok := parseRule(rule)
if !ok {
continue
}
// ufw would take the mesh's rule for this one and rewrite its action or log type:
// an operator's refusal, or a limit, would silently become an allow — and removing the
// opening would then delete it. A plain allow answers the opening; anything else is a
// conflict the operator decides (novox/hq ADR 0103).
if parsed.sameAs(want) && (parsed.action != "allow" || parsed.log != "") {
return Converged{}, fmt.Errorf("ufw holds %q, which ufw takes for the same rule as the "+
"mesh's opening for %s, differing in what it does; adding the opening would change "+
"it, so nothing was added. Change or remove that rule, or have the mesh stop "+
"declaring the opening", rule, o.Target())
}
if satisfiedBy == "" && parsed.admits(o) {
satisfiedBy = rule
}
}
}
if present && len(stale) == 0 {
return Converged{Action: "unchanged"}, nil
}
for _, rule := range stale {
if _, err := run(ctx, "ufw", deletion(rule)...); err != nil {
return Converged{}, fmt.Errorf("deleting the mesh's stale ufw rule %q: %w", rule, err)
}
}
if !present && satisfiedBy != "" {
after, err := added(ctx, run)
if err != nil {
return Converged{}, err
}
for _, rule := range after {
if markedFor(comment(rule), o.ID) {
return Converged{}, fmt.Errorf("ufw still lists a stale rule marked for %s after deleting it", o.ID)
}
}
action := "unchanged"
if len(stale) > 0 {
action = "updated"
}
return Converged{Action: action, SatisfiedBy: satisfiedBy}, nil
}
if !present {
args := append(Rule(o), "comment", mark)
if _, err := run(ctx, "ufw", args...); err != nil {
return Converged{}, fmt.Errorf("adding the ufw rule for %s: %w", o.Target(), err)
}
}
after, err := added(ctx, run)
if err != nil {
return Converged{}, err
}
found, leftover := false, 0
for _, rule := range after {
c := comment(rule)
if c == mark {
found = true
} else if markedFor(c, o.ID) {
leftover++
}
}
if !found {
return Converged{}, fmt.Errorf("ufw was asked for %s and does not list it afterwards", o.Target())
}
if leftover > 0 {
return Converged{}, fmt.Errorf("ufw still lists %d stale rule(s) marked for %s after deleting them", leftover, o.ID)
}
if len(stale) > 0 {
return Converged{Action: "updated"}, nil
}
return Converged{Action: "created"}, nil
}
// Remove deletes the rules marked for one opening, and nothing else.
func Remove(ctx context.Context, run Runner, id string) (int, error) {
rules, err := added(ctx, run)
if err != nil {
return 0, err
}
removed := 0
for _, rule := range rules {
if !markedFor(comment(rule), id) {
continue
}
if _, err := run(ctx, "ufw", deletion(rule)...); err != nil {
return removed, fmt.Errorf("deleting the mesh's ufw rule %q: %w", rule, err)
}
removed++
}
after, err := added(ctx, run)
if err != nil {
return removed, err
}
for _, rule := range after {
if markedFor(comment(rule), id) {
return removed, fmt.Errorf("ufw still lists a rule marked for %s after deleting it", id)
}
}
return removed, nil
}
// Enable turns ufw back on, as found, and reads back that it is.
func Enable(ctx context.Context, run Runner) error {
if _, err := run(ctx, "ufw", "--force", "enable"); err != nil {
return fmt.Errorf("enabling ufw again: %w", err)
}
return expectActive(ctx, run, true)
}
// Disable retires ufw without flushing it: its configuration stays on disk, and the container
// runtime's rules are not its to remove.
//
// **Nor is the forward policy ufw's to open.** Measured on a lab machine running the container
// runtime with a published port (testdata/ufw-disable-iptables-before.txt and -after.txt):
// `ufw disable` sets every built-in chain's policy to accept, the forward chain's among them. The
// runtime had set that one to drop when it turned forwarding on, and it does not set it again while
// forwarding stays on — not even on a restart. Left so, a retired ufw turns the machine into a
// router for anyone who can reach it. So each family's forward policy is read before, and one that
// was drop is put back and read back. before is ForwardPolicies as read before the first attempt.
func Disable(ctx context.Context, run Runner, before map[string]string) error {
if _, err := run(ctx, "ufw", "disable"); err != nil {
return fmt.Errorf("disabling ufw: %w", err)
}
if err := expectActive(ctx, run, false); err != nil {
return err
}
for _, tool := range []string{"iptables", "ip6tables"} {
if before[tool] != "DROP" {
continue
}
if now, ok := forwardPolicy(ctx, run, tool); ok && now == "DROP" {
continue
}
if _, err := run(ctx, tool, "-P", "FORWARD", "DROP"); err != nil {
return fmt.Errorf("ufw is disabled, and %s's forward policy, which was drop, could not be put back: %w",
tool, err)
}
if now, ok := forwardPolicy(ctx, run, tool); !ok || now != "DROP" {
return fmt.Errorf("ufw is disabled, and %s's forward policy was put back to drop and reads %q",
tool, now)
}
}
return nil
}
// MeshTable is the derived filter's table, the thing that must be in force before the firewall
// found on a machine is retired.
const MeshTable = "inet mesh"
// MeshTableLoaded asks the machine whether the mesh's own filter is loaded. Read from the machine
// rather than assumed from the declaration: a table declared and not loaded is exactly the case
// where disabling the found firewall would leave the machine with nothing.
func MeshTableLoaded(ctx context.Context, run Runner) (bool, error) {
out, err := run(ctx, "nft", "list", "tables")
if err != nil {
if missing(err) {
return false, nil
}
return false, fmt.Errorf("cannot read which tables this machine has loaded: %w", err)
}
for _, line := range strings.Split(out, "\n") {
rest, ok := strings.CutPrefix(strings.TrimSpace(line), "table "+MeshTable)
if ok && (rest == "" || strings.HasPrefix(rest, " ") || strings.HasPrefix(rest, "{")) {
return true, nil
}
}
return false, nil
}
// ForwardPolicies reads each family's forward policy, by the tool that sets it. Read before ufw is
// disabled and kept by the caller, so a retirement that fails half-way is retried with what the
// machine had — not with what the half-done disable left.
func ForwardPolicies(ctx context.Context, run Runner) map[string]string {
out := map[string]string{}
for _, tool := range []string{"iptables", "ip6tables"} {
if policy, ok := forwardPolicy(ctx, run, tool); ok {
out[tool] = policy
}
}
return out
}
// forwardPolicy reads the forward chain's policy the way iptables prints it: "-P FORWARD DROP".
// Not ok when the tool is absent or says nothing readable.
func forwardPolicy(ctx context.Context, run Runner, tool string) (string, bool) {
out, err := run(ctx, tool, "-S", "FORWARD")
if err != nil {
return "", false
}
for _, line := range strings.Split(out, "\n") {
f := strings.Fields(line)
if len(f) == 3 && f[0] == "-P" && f[1] == "FORWARD" {
return f[2], true
}
}
return "", false
}
func expectActive(ctx context.Context, run Runner, want bool) error {
out, err := run(ctx, "ufw", "status")
if err != nil {
return fmt.Errorf("reading ufw's status back: %w", err)
}
if statusActive(out) != want {
state := "inactive"
if want {
state = "active"
}
return fmt.Errorf("ufw was asked to be %s and says: %s", state, strings.TrimSpace(out))
}
return nil
}
+789
View File
@@ -0,0 +1,789 @@
package firewall
import (
"context"
"errors"
"fmt"
"os"
"os/exec"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
)
// Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force, the mesh opens
// what it needs through it in its own terms, and removes only what it marked.
func dockerOnly(t *testing.T) string {
t.Helper()
// Captured from a real machine running the container runtime and nothing else that filters:
// its nat, filter and raw tables as iptables-nft writes them.
raw, err := os.ReadFile("testdata/docker-only.nft")
if err != nil {
t.Fatal(err)
}
return string(raw)
}
const aDroppingTable = `
table inet filter {
chain input {
type filter hook input priority filter; policy drop;
ct state established,related accept
tcp dport 22 accept
}
}
`
const ufwChains = `
# Warning: table ip filter is managed by iptables-nft, do not touch!
table ip filter {
chain INPUT {
type filter hook input priority filter; policy drop;
counter packets 0 bytes 0 jump ufw-before-input
}
chain ufw-user-input {
tcp dport 22 counter packets 0 bytes 0 accept
}
chain ufw-reject-input {
counter packets 0 bytes 0 reject
}
}
`
const theMeshsOwn = `
table inet mesh {
chain input {
type filter hook input priority filter; policy drop;
iif lo accept
}
}
table inet mesh_guard {
chain prerouting {
type filter hook prerouting priority raw; policy accept;
iifname != "lo" tcp dport { 5432, 15672 } drop
}
}
`
func TestTheContainerRuntimesOwnRulesAreNotAFirewall(t *testing.T) {
if got := Refusing(dockerOnly(t), false); len(got) != 0 {
t.Errorf("the runtime's own rules read as a firewall: %v", got)
}
}
func TestTheMeshsOwnTablesAreNotAFirewall(t *testing.T) {
if got := Refusing(dockerOnly(t)+theMeshsOwn, false); len(got) != 0 {
t.Errorf("the mesh's own tables read as a found firewall: %v", got)
}
}
func TestATableThatDropsIsAFirewall(t *testing.T) {
got := Refusing(dockerOnly(t)+aDroppingTable, false)
if len(got) != 1 || got[0] != "table inet filter" {
t.Errorf("a dropping table was not named: %v", got)
}
}
func TestUfwsOwnChainsAreUfwsWhenItIsActive(t *testing.T) {
if got := Refusing(dockerOnly(t)+ufwChains, true); len(got) != 0 {
t.Errorf("ufw's own chains read as a second firewall: %v", got)
}
if got := Refusing(dockerOnly(t)+ufwChains, false); len(got) == 0 {
t.Error("iptables rules that refuse, with ufw not active, were not counted")
}
}
func TestLegacyIptablesThatRefusesIsAFirewall(t *testing.T) {
docker := "-P INPUT ACCEPT\n-P FORWARD DROP\n-P OUTPUT ACCEPT\n-N DOCKER\n-A DOCKER -i docker0 -j DROP\n"
if got := RefusingLegacy(docker); len(got) != 0 {
t.Errorf("the runtime's legacy rules read as a firewall: %v", got)
}
if got := RefusingLegacy(docker + "-A INPUT -p tcp --dport 25 -j REJECT\n"); len(got) != 1 {
t.Errorf("a legacy reject was not counted: %v", got)
}
}
// fakeUFW is ufw as far as the host can see it: a status, and user rules it prints back in its
// own canonical form — deliberately not the order the host wrote them in.
type fakeUFW struct {
active bool
installed bool
rules []string
ruleset string
firewalld bool
asked []string
// iptablesActive and iptablesInactive are what `iptables -S` prints with ufw active and
// after it is disabled; empty is a machine without iptables. forward is a policy set since.
iptablesActive, iptablesInactive string
forward string
// noNft is a machine with no nft binary; iptablesRules is what `iptables -S` prints there.
noNft bool
iptablesRules string
}
// iptables answers `iptables -S FORWARD` from the captured output for ufw's state, and records
// a forward policy set with -P.
func (f *fakeUFW) iptables(name string, args []string) (string, error) {
if f.iptablesRules != "" && len(args) == 1 && args[0] == "-S" {
if name == "ip6tables" {
return "", nil
}
return f.iptablesRules, nil
}
if f.iptablesActive == "" {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
if name == "ip6tables" {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
if len(args) == 3 && args[0] == "-P" && args[1] == "FORWARD" {
f.forward = args[2]
return "", nil
}
captured := f.iptablesInactive
if f.active {
captured = f.iptablesActive
}
var out []string
for _, line := range strings.Split(captured, "\n") {
fields := strings.Fields(line)
if len(fields) >= 2 && fields[1] == "FORWARD" {
if fields[0] == "-P" && f.forward != "" && !f.active {
line = "-P FORWARD " + f.forward
}
out = append(out, line)
}
}
return strings.Join(out, "\n") + "\n", nil
}
// canonical is a rule the way ufw prints it back, as captured (testdata/ufw-comment-only.txt): the
// short form `allow 5671/tcp` for a rule on no interface, the long form `allow in on mesh0 to any
// port 5432 proto tcp` for one on an interface; the comment last.
func canonical(args []string) (rule, commentText string) {
var route, in, port, proto string
for i := 0; i < len(args); i++ {
switch args[i] {
case "route":
route = "route "
case "in":
in = args[i+2]
i += 2
case "port":
port = args[i+1]
i++
case "proto":
proto = args[i+1]
i++
case "comment":
commentText = args[i+1]
i++
}
}
if in != "" {
return route + "allow in on " + in + " to any port " + port + " proto " + proto, commentText
}
return route + "allow " + port + "/" + proto, commentText
}
func (f *fakeUFW) run(_ context.Context, name string, args ...string) (string, error) {
f.asked = append(f.asked, name+" "+strings.Join(args, " "))
switch name {
case "firewall-cmd":
if f.firewalld {
return "running\n", nil
}
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
case "nft":
if f.noNft {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
return f.ruleset, nil
case "iptables-legacy", "ip6tables-legacy":
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
case "iptables", "ip6tables":
return f.iptables(name, args)
case "ufw":
default:
return "", fmt.Errorf("unexpected %s", name)
}
if !f.installed {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
switch {
case args[0] == "status":
if f.active {
return "Status: active\n\nTo Action From\n", nil
}
return "Status: inactive\n", nil
case args[0] == "show":
out := "Added user rules (see 'ufw status' for running firewall):\n"
for _, r := range f.rules {
out += "ufw " + r + "\n"
}
return out, nil
case args[0] == "--force" && args[1] == "enable":
f.active = true
return "Firewall is active and enabled on system startup\n", nil
case args[0] == "disable":
f.active = false
f.forward = ""
return "Firewall stopped and disabled on system startup\n", nil
case args[0] == "delete" && len(args) > 1 && args[1] == "route":
// As the real ufw answers it (captured in testdata/ufw-delete.txt): a route rule is
// deleted with `route delete`, never `delete route`.
return "", errors.New("ERROR: Invalid syntax")
case args[0] == "delete", args[0] == "route" && len(args) > 1 && args[1] == "delete":
rest := args[1:]
if args[0] == "route" {
rest = append([]string{"route"}, args[2:]...)
}
for i, r := range f.rules {
if strings.Join(words(r), "\x00") == strings.Join(rest, "\x00") {
f.rules = append(f.rules[:i], f.rules[i+1:]...)
return "Rule deleted\n", nil
}
}
return "", errors.New("Could not delete non-existent rule")
default:
rule, note := canonical(args)
line := rule
if note != "" {
line += " comment '" + note + "'"
}
// As the real ufw does (testdata/ufw-comment-only.txt): a rule differing from one it holds
// only in its comment is the same rule, and its comment is replaced.
for i, r := range f.rules {
if bare, _, _ := strings.Cut(r, " comment '"); bare == rule {
f.rules[i] = line
return "Rule updated\nRule updated (v6)\n", nil
}
}
f.rules = append(f.rules, line)
return "Rule added\nRule added (v6)\n", nil
}
}
func (f *fakeUFW) added() int {
n := 0
for _, a := range f.asked {
if strings.HasPrefix(a, "ufw allow") || strings.HasPrefix(a, "ufw route") {
n++
}
}
return n
}
func opening(id string, port int, from, path string, to int) *declaration.Opening {
return &declaration.Opening{ID: id, Type: declaration.TypeOpening, Port: port, Protocol: "tcp",
From: from, Path: path, To: to}
}
func TestAnOpeningBecomesTheUfwRuleForItsPathAndOrigin(t *testing.T) {
for _, c := range []struct {
o *declaration.Opening
want string
}{
{opening("adoption.a", 5671, "everywhere", "incoming", 0), "allow proto tcp to any port 5671"},
{opening("adoption.b", 5432, "mesh", "incoming", 0), "allow in on mesh0 proto tcp to any port 5432"},
{opening("adoption.c", 20001, "everywhere", "forwarded", 8080), "route allow proto tcp to any port 8080"},
{opening("adoption.d", 20001, "mesh", "forwarded", 8080), "route allow in on mesh0 proto tcp to any port 8080"},
} {
if got := strings.Join(Rule(c.o), " "); got != c.want {
t.Errorf("%s: %q, want %q", c.o.ID, got, c.want)
}
}
}
func TestAnOpeningIsAddedOnceAndMarkedAsTheMeshs(t *testing.T) {
f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp"}}
o := opening("adoption.opening-tcp-5671-incoming", 5671, "everywhere", "incoming", 0)
action, err := Converge(context.Background(), f.run, o)
if err != nil || action.Action != "created" {
t.Fatalf("first converge: %q %v", action, err)
}
if !strings.Contains(f.rules[1], "comment 'mesh-host adoption.opening-tcp-5671-incoming ") {
t.Errorf("the rule is not marked as the mesh's: %v", f.rules)
}
action, err = Converge(context.Background(), f.run, o)
if err != nil || action.Action != "unchanged" {
t.Fatalf("second converge: %q %v", action, err)
}
if f.added() != 1 {
t.Errorf("re-converging added again: %v", f.asked)
}
}
func TestAnOpeningLostToAReloadIsAddedAgain(t *testing.T) {
f := &fakeUFW{installed: true, active: true}
o := opening("adoption.x", 5671, "everywhere", "incoming", 0)
if _, err := Converge(context.Background(), f.run, o); err != nil {
t.Fatal(err)
}
f.rules = nil // what a reload that lost the rule leaves
action, err := Converge(context.Background(), f.run, o)
if err != nil || action.Action != "created" || len(f.rules) != 1 {
t.Fatalf("a lost opening was not put back: %q %v %v", action, err, f.rules)
}
}
func TestAChangedOpeningReplacesOnlyItsOwnRule(t *testing.T) {
f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 8080/tcp comment 'someone else'"}}
if _, err := Converge(context.Background(), f.run, opening("adoption.x", 5671, "everywhere", "incoming", 0)); err != nil {
t.Fatal(err)
}
action, err := Converge(context.Background(), f.run, opening("adoption.x", 5671, "mesh", "incoming", 0))
if err != nil || action.Action != "updated" {
t.Fatalf("%q %v", action, err)
}
if len(f.rules) != 3 || f.rules[0] != "allow 22/tcp" || f.rules[1] != "allow 8080/tcp comment 'someone else'" ||
!strings.Contains(f.rules[2], "in on mesh0") {
t.Errorf("rules afterwards: %v", f.rules)
}
}
func TestRemovingAnOpeningRemovesOnlyWhatWasMarkedForIt(t *testing.T) {
f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 8080/tcp"}}
for _, o := range []*declaration.Opening{
opening("adoption.a", 5671, "everywhere", "incoming", 0),
opening("adoption.ab", 5000, "everywhere", "incoming", 0),
} {
if _, err := Converge(context.Background(), f.run, o); err != nil {
t.Fatal(err)
}
}
n, err := Remove(context.Background(), f.run, "adoption.a")
if err != nil || n != 1 {
t.Fatalf("removed %d: %v", n, err)
}
if len(f.rules) != 3 || f.rules[0] != "allow 22/tcp" || f.rules[1] != "allow 8080/tcp" ||
!strings.Contains(f.rules[2], "adoption.ab") {
t.Errorf("more than the marked rule went: %v", f.rules)
}
}
func TestEnableAndDisableReadBack(t *testing.T) {
f := &fakeUFW{installed: true, active: true}
if err := Disable(context.Background(), f.run, nil); err != nil || f.active {
t.Fatalf("disable: %v", err)
}
if err := Enable(context.Background(), f.run); err != nil || !f.active {
t.Fatalf("enable: %v", err)
}
for _, a := range f.asked {
if strings.Contains(a, "reset") || strings.Contains(a, "flush") {
t.Errorf("the found firewall was reset: %s", a)
}
}
}
func TestDetectingTheFoundFirewall(t *testing.T) {
for _, c := range []struct {
name string
f *fakeUFW
want Kind
}{
{"nothing but the runtime", &fakeUFW{ruleset: dockerOnly(t)}, None},
{"ufw active", &fakeUFW{installed: true, active: true, ruleset: dockerOnly(t) + ufwChains}, UFW},
{"ufw installed and inactive", &fakeUFW{installed: true, ruleset: dockerOnly(t)}, None},
{"firewalld", &fakeUFW{firewalld: true, ruleset: dockerOnly(t)}, Unsupported},
{"an nftables table of its own", &fakeUFW{ruleset: dockerOnly(t) + aDroppingTable}, Unsupported},
{"ufw beside an nftables table", &fakeUFW{installed: true, active: true, ruleset: dockerOnly(t) + ufwChains + aDroppingTable}, Unsupported},
} {
got, name, err := Detect(context.Background(), c.f.run)
if err != nil {
t.Fatalf("%s: %v", c.name, err)
}
if got != c.want {
t.Errorf("%s: detected %s (%s), want %s", c.name, got, name, c.want)
}
if got == Unsupported && name == "" {
t.Errorf("%s: an unsupported firewall was not named", c.name)
}
}
}
// The fixtures below were captured from a real ufw 0.36.2 on a lab machine, not written by hand:
// ufw prints a rule back in its own shorter form, so the mark in the comment is the only thing the
// host relies on.
func TestTheMarksAreReadFromWhatUfwReallyPrints(t *testing.T) {
raw, err := os.ReadFile("testdata/ufw-show-added.txt")
if err != nil {
t.Fatal(err)
}
run := func(ctx context.Context, name string, args ...string) (string, error) { return string(raw), nil }
rules, err := added(context.Background(), run)
if err != nil {
t.Fatal(err)
}
if len(rules) != 7 {
t.Fatalf("read %d rules, want 7: %q", len(rules), rules)
}
marked := 0
for _, r := range rules {
if strings.HasPrefix(comment(r), "mesh-host ") {
marked++
}
}
if marked != 5 {
t.Errorf("read %d marked rules, want 5", marked)
}
if !markedFor(comment(rules[5]), "adoption.opening-tcp-8443-forwarded") {
t.Errorf("the forwarded rule from the mesh lost its mark: %q", rules[5])
}
}
func TestEveryRealRuleIsDeletedInTheFormUfwAccepts(t *testing.T) {
raw, err := os.ReadFile("testdata/ufw-show-added.txt")
if err != nil {
t.Fatal(err)
}
run := func(ctx context.Context, name string, args ...string) (string, error) { return string(raw), nil }
rules, _ := added(context.Background(), run)
// Each of these was run on the lab machine and answered "Rule deleted" (testdata/ufw-delete.txt).
want := map[string]string{
"allow 5671/tcp comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d'": "delete allow 5671/tcp comment|mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d",
"allow in on mesh0 to any port 5432 proto tcp comment 'mesh-host adoption.opening-tcp-5432-incoming deadbeef'": "delete allow in on mesh0 to any port 5432 proto tcp comment|mesh-host adoption.opening-tcp-5432-incoming deadbeef",
"route allow 80/tcp comment 'mesh-host adoption.opening-tcp-8081-forwarded 0badf00d'": "route delete allow 80/tcp comment|mesh-host adoption.opening-tcp-8081-forwarded 0badf00d",
"route allow in on mesh0 to any port 443 proto tcp comment 'mesh-host adoption.opening-tcp-8443-forwarded cafe0001'": "route delete allow in on mesh0 to any port 443 proto tcp comment|mesh-host adoption.opening-tcp-8443-forwarded cafe0001",
}
seen := 0
for _, r := range rules {
w, ok := want[r]
if !ok {
continue
}
seen++
d := deletion(r)
got := strings.Join(d[:len(d)-1], " ") + "|" + d[len(d)-1]
if got != w {
t.Errorf("deleting %q\n got %s\n want %s", r, got, w)
}
}
if seen != len(want) {
t.Errorf("matched %d of %d captured rules", seen, len(want))
}
}
func TestARealUfwRulesetIsUfw(t *testing.T) {
raw, err := os.ReadFile("testdata/ufw-active.nft")
if err != nil {
t.Fatal(err)
}
status, err := os.ReadFile("testdata/ufw-status-active.txt")
if err != nil {
t.Fatal(err)
}
if !statusActive(string(status)) {
t.Fatal("the captured status does not read as active")
}
if refusing := Refusing(string(raw), true); len(refusing) > 0 {
t.Errorf("a machine with ufw active and nothing else read as refusing in %v", refusing)
}
if refusing := Refusing(string(raw), false); len(refusing) == 0 {
t.Error("ufw's drop chains, with ufw not known to be active, read as refusing nothing")
}
}
func TestRetiringUfwKeepsTheMachineFromRoutingForOthers(t *testing.T) {
// Captured on a lab machine running the container runtime with a published port: ufw active,
// then `ufw disable`. Disabling set the forward policy the runtime had set to drop to accept.
before, err := os.ReadFile("testdata/ufw-disable-iptables-before.txt")
if err != nil {
t.Fatal(err)
}
after, err := os.ReadFile("testdata/ufw-disable-iptables-after.txt")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(before), "-P FORWARD DROP") || !strings.Contains(string(after), "-P FORWARD ACCEPT") {
t.Fatal("the captures no longer show ufw disable opening the forward policy")
}
f := &fakeUFW{installed: true, active: true, iptablesActive: string(before), iptablesInactive: string(after)}
if err := Disable(context.Background(), f.run, ForwardPolicies(context.Background(), f.run)); err != nil {
t.Fatal(err)
}
if f.active {
t.Fatal("ufw is still active")
}
if f.forward != "DROP" {
t.Errorf("the forward policy was left open after ufw was retired: %v", f.asked)
}
for _, a := range f.asked {
if strings.Contains(a, "-F") || strings.Contains(a, "flush") || strings.Contains(a, "reset") {
t.Errorf("retiring ufw flushed something: %s", a)
}
}
}
func TestRetiringUfwOnAMachineWithoutIptablesStillRetiresIt(t *testing.T) {
f := &fakeUFW{installed: true, active: true}
if err := Disable(context.Background(), f.run, nil); err != nil || f.active {
t.Fatalf("disable: %v, active %v", err, f.active)
}
}
// Captured on a lab machine with fail2ban banning one documentation address in its sshd jail,
// once with its nftables backend and once with its iptables backend (iptables-nft), ufw inactive.
func captured(t *testing.T, name string) string {
t.Helper()
raw, err := os.ReadFile("testdata/" + name)
if err != nil {
t.Fatal(err)
}
return string(raw)
}
func TestFail2bansBansAreNotAFirewall(t *testing.T) {
for _, name := range []string{"fail2ban-nftables.nft", "fail2ban-iptables.nft"} {
ruleset := captured(t, name)
if !strings.Contains(ruleset, "192.0.2.55") {
t.Fatalf("%s holds no ban", name)
}
if got := Refusing(ruleset, false); len(got) != 0 {
t.Errorf("%s: fail2ban's bans read as a firewall: %v", name, got)
}
kind, what, err := Detect(context.Background(), (&fakeUFW{ruleset: ruleset}).run)
if err != nil || kind != None {
t.Errorf("%s: a machine with only fail2ban detected as %s (%s) %v", name, kind, what, err)
}
}
if got := RefusingLegacy(captured(t, "fail2ban-iptables-S.txt")); len(got) != 0 {
t.Errorf("fail2ban's iptables bans read as a firewall: %v", got)
}
}
func TestARefusalOfEveryoneButSomeIsStillAFirewall(t *testing.T) {
// A ban names the sources it refuses. A table that refuses every source but some, or every
// port but some, closes what the mesh would open, whatever its policy says.
for name, table := range map[string]string{
"all but a range": "table inet own {\n\tchain input {\n\t\ttype filter hook input priority filter; policy accept;\n\t\tip saddr != 10.0.0.0/8 drop\n\t}\n}\n",
"all but ssh": "table inet own {\n\tchain input {\n\t\ttype filter hook input priority filter; policy accept;\n\t\ttcp dport != 22 drop\n\t}\n}\n",
"iptables reject": "# Warning: table ip filter is managed by iptables-nft, do not touch!\ntable ip filter {\n\tchain INPUT {\n\t\ttype filter hook input priority filter; policy accept;\n\t\tcounter packets 0 bytes 0 xt target \"REJECT\"\n\t}\n}\n",
"ban beside a dropping policy": "table inet own {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t\tip saddr 192.0.2.9 drop\n\t}\n}\n",
} {
if got := Refusing(dockerOnly(t)+table, false); len(got) == 0 {
t.Errorf("%s: not counted as a firewall", name)
}
}
legacy := "-P INPUT ACCEPT\n-N own\n-A INPUT -j own\n-A own ! -s 10.0.0.0/8 -j DROP\n"
if got := RefusingLegacy(legacy); len(got) == 0 {
t.Error("a legacy refusal of all but a range was not counted")
}
}
// Defends novox/hq ADR 0103: an opening a found rule already answers is not added, because ufw
// takes two rules differing only in their comment for one (testdata/ufw-comment-only.txt).
func TestUfwTakesTheMeshsRuleAndTheOperatorsForOne(t *testing.T) {
// The capture: each mesh rule answered "Rule updated" beside the operator's equivalent.
raw := captured(t, "ufw-comment-only.txt")
if strings.Count(raw, "Rule updated\n") != 3 {
t.Fatalf("the capture no longer shows ufw updating an equivalent rule:\n%s", raw)
}
for _, c := range []struct {
operators string
o *declaration.Opening
}{
{"route allow 8080/tcp", opening("adoption.opening-tcp-8080-forwarded", 20001, "everywhere", "forwarded", 8080)},
{"allow 5671/tcp", opening("adoption.opening-tcp-5671-incoming", 5671, "everywhere", "incoming", 0)},
{"allow in on mesh0 to any port 5432 proto tcp comment 'operator note'", opening("adoption.opening-tcp-5432-incoming", 5432, "mesh", "incoming", 0)},
} {
theirs, ok := parseRule(c.operators)
mine, ok2 := parseRule(strings.Join(Rule(c.o), " ") + " comment '" + Mark(c.o) + "'")
if !ok || !ok2 || !theirs.sameAs(mine) {
t.Errorf("%q and the mesh's %v are one rule to ufw, and read as two", c.operators, Rule(c.o))
}
if !theirs.admits(c.o) {
t.Errorf("%q does not read as answering %s", c.operators, c.o.Target())
}
}
}
func TestEveryCapturedRuleFormIsRead(t *testing.T) {
want := map[string]string{
"allow 22/tcp": "tcp 22 in= from=any", "allow 9200": " 9200 in= from=any",
"allow from 192.0.2.0/24 to any port 9300 proto tcp": "tcp 9300 in= from=192.0.2.0/24",
"allow in on eth0 to any port 9301 proto tcp": "tcp 9301 in=eth0 from=any",
"allow 9500:9510/tcp": "tcp 9500:9510 in= from=any",
"allow 80,443/tcp": "tcp 80,443 in= from=any",
"allow in on mesh0 to any port 5432 proto tcp": "tcp 5432 in=mesh0 from=any",
"route allow 8080/tcp": "tcp 8080 in= from=any",
"allow 9900/tcp": "tcp 9900 in= from=any",
"allow out 5671/tcp": "tcp 5671 in= from=any",
"deny out 5672/tcp": "tcp 5672 in= from=any",
"allow out on eth0 to any port 5673 proto tcp": "tcp 5673 in= from=any",
"allow log 9001/tcp": "tcp 9001 in= from=any",
"route allow log 8084/tcp": "tcp 8084 in= from=any",
"allow in on mesh0 log-all to any port 9002 proto tcp": "tcp 9002 in=mesh0 from=any",
}
rules, err := added(context.Background(), func(context.Context, string, ...string) (string, error) {
return captured(t, "ufw-forms.txt"), nil
})
if err != nil || len(rules) != 21 {
t.Fatalf("read %d rules: %v", len(rules), err)
}
for _, rule := range rules {
r, ok := parseRule(rule)
if !ok {
t.Errorf("a rule ufw printed was not read: %q", rule)
continue
}
if w, listed := want[rule]; listed {
if got := r.proto + " " + r.port + " in=" + r.in + " from=" + r.from; got != w {
t.Errorf("%q read as %q, want %q", rule, got, w)
}
}
}
}
func TestAnOpeningAFoundRuleAnswersIsNotAddedAndItsRemovalLeavesTheRule(t *testing.T) {
for _, c := range []struct {
name, operators string
o *declaration.Opening
}{
{"forwarded, the same rule", "route allow 8080/tcp", opening("adoption.fwd", 20001, "everywhere", "forwarded", 8080)},
{"incoming, the same rule", "allow 5671/tcp", opening("adoption.bus", 5671, "everywhere", "incoming", 0)},
{"with a comment of its own", "allow in on mesh0 to any port 5432 proto tcp comment 'operator note'", opening("adoption.store", 5432, "mesh", "incoming", 0)},
{"broader: from anywhere", "allow 5432/tcp", opening("adoption.store", 5432, "mesh", "incoming", 0)},
{"broader: any protocol, a range", "allow 5000:5100", opening("adoption.registry", 5000, "everywhere", "incoming", 0)},
} {
f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", c.operators}}
done, err := Converge(context.Background(), f.run, c.o)
if err != nil {
t.Fatalf("%s: %v", c.name, err)
}
if done.SatisfiedBy != c.operators || done.Action != "unchanged" || f.added() != 0 {
t.Errorf("%s: %+v, asked %v", c.name, done, f.asked)
}
if n, err := Remove(context.Background(), f.run, c.o.ID); err != nil || n != 0 {
t.Errorf("%s: removing the opening removed %d: %v", c.name, n, err)
}
if len(f.rules) != 2 || f.rules[1] != c.operators {
t.Errorf("%s: the operator's rule did not survive: %v", c.name, f.rules)
}
}
}
func TestARuleThatDoesNotAnswerTheOpeningLeavesItToBeAdded(t *testing.T) {
for _, operators := range []string{
"allow from 192.0.2.0/24 to any port 5671 proto tcp", // narrower: from one range
"allow in on eth0 to any port 5671 proto tcp", // narrower: one interface
"allow 5671/udp", // another protocol
"route allow 5671/tcp", // another path
"allow to 192.0.2.1 port 5671 proto tcp", // one address
} {
f := &fakeUFW{installed: true, active: true, rules: []string{operators}}
done, err := Converge(context.Background(), f.run, opening("adoption.bus", 5671, "everywhere", "incoming", 0))
if err != nil || done.Action != "created" || done.SatisfiedBy != "" {
t.Errorf("%q: %+v %v", operators, done, err)
}
}
}
func TestAnOpeningWhoseFoundRuleIsGoneIsAddedAgain(t *testing.T) {
f := &fakeUFW{installed: true, active: true, rules: []string{"allow 5671/tcp"}}
o := opening("adoption.bus", 5671, "everywhere", "incoming", 0)
if done, err := Converge(context.Background(), f.run, o); err != nil || done.SatisfiedBy == "" {
t.Fatalf("%+v %v", done, err)
}
f.rules = nil // the operator deleted theirs
if done, err := Converge(context.Background(), f.run, o); err != nil || done.Action != "created" {
t.Fatalf("the opening was not added once nothing answered it: %+v %v", done, err)
}
}
func TestARuleUfwWouldMergeThatDoesOtherThanAllowRefusesTheOpening(t *testing.T) {
// ufw takes two rules differing only in action or log type for one, and adding the mesh's
// would turn the operator's refusal into an allow (novox/hq ADR 0103).
for _, operators := range []string{
"deny 5671/tcp",
"reject 5671/tcp",
"limit 5671/tcp",
"allow log 5671/tcp",
"allow log-all proto tcp to any port 5671",
"deny in log to any port 5671 proto tcp comment 'operator note'",
} {
f := &fakeUFW{installed: true, active: true, rules: []string{operators}}
_, err := Converge(context.Background(), f.run, opening("adoption.bus", 5671, "everywhere", "incoming", 0))
if err == nil || !strings.Contains(err.Error(), operators) {
t.Errorf("%q: the conflict was not refused naming the rule: %v", operators, err)
}
if f.added() != 0 || len(f.rules) != 1 || f.rules[0] != operators {
t.Errorf("%q: something was added or changed: %v %v", operators, f.asked, f.rules)
}
}
}
func TestALogTypeIsReadInEitherPlace(t *testing.T) {
for rule, want := range map[string]string{
"allow log 22/tcp": "allow log 22 tcp in=",
"allow in log-all on mesh0 to any port 5432 proto tcp": "allow log-all 5432 tcp in=mesh0",
"route deny log in on mesh0 to any port 80 proto tcp": "deny log 80 tcp in=mesh0",
"allow 22/tcp comment 'log'": "allow 22 tcp in=",
} {
r, ok := parseRule(rule)
if got := r.action + " " + r.log + " " + r.port + " " + r.proto + " in=" + r.in; !ok || got != want {
t.Errorf("%q read as %q (%v), want %q", rule, got, ok, want)
}
}
}
func TestAnOutgoingRuleNeverAnswersAnOpening(t *testing.T) {
// `ufw allow out 5671/tcp` lets this machine reach others; nothing arrives through it, and
// ufw keeps it as a rule of its own — captured in testdata/ufw-direction.txt.
raw := captured(t, "ufw-direction.txt")
if !strings.Contains(raw, "ufw allow out 9007/tcp\nufw allow 9007/tcp") {
t.Fatalf("the capture no longer shows an outgoing rule standing beside an incoming one:\n%s", raw)
}
for _, operators := range []string{"allow out 5671/tcp", "allow out on eth0 to any port 5671 proto tcp",
"deny out 5671/tcp"} {
f := &fakeUFW{installed: true, active: true, rules: []string{operators}}
done, err := Converge(context.Background(), f.run, opening("adoption.bus", 5671, "everywhere", "incoming", 0))
if err != nil {
t.Errorf("%q: an outgoing rule was taken for a conflict: %v", operators, err)
continue
}
if done.Action != "created" || done.SatisfiedBy != "" {
t.Errorf("%q: an outgoing rule answered an incoming opening: %+v", operators, done)
}
}
}
func TestIncomingIsUfwsDefaultDirection(t *testing.T) {
// Captured: `deny in 9006/tcp` and `allow 9006/tcp` are one rule to ufw, so the mesh must read
// them as one too, or it would take an operator's refusal over.
raw := captured(t, "ufw-direction.txt")
if !strings.Contains(raw, "ufw allow 9005/tcp") || strings.Contains(raw, "ufw deny 9006/tcp") {
t.Fatalf("the capture no longer shows `in` as the default direction:\n%s", raw)
}
f := &fakeUFW{installed: true, active: true, rules: []string{"deny in to any port 5671 proto tcp"}}
if _, err := Converge(context.Background(), f.run, opening("adoption.bus", 5671, "everywhere", "incoming", 0)); err == nil {
t.Error("an incoming refusal ufw would merge was not refused")
}
}
func TestAMachineWithIptablesRulesAndNoNftIsNotReadAsUnfiltered(t *testing.T) {
// nft is not installed, and iptables-nft holds a firewall of somebody's. Read as "nothing
// filters here" the mesh would adopt it, open nothing, and be unreachable (novox/hq ADR 0100).
rules := "-P INPUT DROP\n-P FORWARD DROP\n-P OUTPUT ACCEPT\n-A INPUT -p tcp -m tcp --dport 22 -j ACCEPT\n"
f := &fakeUFW{noNft: true, iptablesRules: rules}
kind, what, err := Detect(context.Background(), f.run)
if err != nil {
t.Fatal(err)
}
if kind != Unsupported {
t.Errorf("a machine filtered by iptables with no nft read as %s (%s)", kind, what)
}
// And a machine with nothing but the runtime's own rules and no nft is still unfiltered.
docker := "-P INPUT ACCEPT\n-P FORWARD DROP\n-P OUTPUT ACCEPT\n-N DOCKER\n-A DOCKER -i docker0 -j DROP\n"
if kind, _, err := Detect(context.Background(), (&fakeUFW{noNft: true, iptablesRules: docker}).run); err != nil || kind != None {
t.Errorf("a machine with only the runtime's rules read as %s: %v", kind, err)
}
}
+297
View File
@@ -0,0 +1,297 @@
# Warning: table ip nat is managed by iptables-nft, do not touch!
table ip nat {
chain DOCKER {
iifname != "br-c70303d221ee" tcp dport 5680 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-c70303d221ee" tcp dport 15673 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-3636e05760a9" tcp dport 59000 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-3636e05760a9" tcp dport 59001 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-3636e05760a9" tcp dport 55672 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-3636e05760a9" tcp dport 55673 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-3636e05760a9" tcp dport 55432 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "docker0" tcp dport 57732 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "docker0" tcp dport 57733 counter packets 0 bytes 0 xt target "DNAT"
iifname != "docker0" tcp dport 5314 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-613eb68ef5fb" tcp dport 4848 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-b3240c822bce" tcp dport 5679 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-b3240c822bce" tcp dport 15672 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-4b504efd6080" tcp dport 9000 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-4b504efd6080" tcp dport 9001 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-ef6df03f71a0" tcp dport 5432 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-ef6df03f71a0" tcp dport 8081 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-ec480f77ac34" tcp dport 6379 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-af4c9c2aa60a" tcp dport 8001 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-669fda75f1ac" tcp dport 28080 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "br-af4c9c2aa60a" tcp dport 6789 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-af4c9c2aa60a" tcp dport 8770 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-af4c9c2aa60a" tcp dport 1212 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-af4c9c2aa60a" tcp dport 80 counter packets 0 bytes 0 xt target "DNAT"
iifname != "br-af4c9c2aa60a" tcp dport 443 counter packets 0 bytes 0 xt target "DNAT"
ip daddr 127.0.0.1 iifname != "docker0" tcp dport 55541 counter packets 0 bytes 0 xt target "DNAT"
}
chain PREROUTING {
type nat hook prerouting priority dstnat; policy accept;
xt match "addrtype" counter packets 437947 bytes 71410534 jump DOCKER
}
chain OUTPUT {
type nat hook output priority dstnat; policy accept;
ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 5761 bytes 423317 jump DOCKER
}
chain POSTROUTING {
type nat hook postrouting priority srcnat; policy accept;
ip saddr 172.22.0.0/16 oifname != "br-65f6dc782562" counter packets 124 bytes 16328 xt target "MASQUERADE"
ip saddr 172.28.0.0/16 oifname != "br-669fda75f1ac" counter packets 127 bytes 16928 xt target "MASQUERADE"
ip saddr 172.31.0.0/16 oifname != "br-ec480f77ac34" counter packets 127 bytes 16928 xt target "MASQUERADE"
ip saddr 192.168.48.0/20 oifname != "br-ef6df03f71a0" counter packets 127 bytes 16928 xt target "MASQUERADE"
ip saddr 172.25.0.0/16 oifname != "br-4b504efd6080" counter packets 129 bytes 17052 xt target "MASQUERADE"
ip saddr 192.168.32.0/20 oifname != "br-613eb68ef5fb" counter packets 1124 bytes 76748 xt target "MASQUERADE"
ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 1833 bytes 150990 xt target "MASQUERADE"
ip saddr 172.18.0.0/16 oifname != "br-07a5e2f2c42f" counter packets 504 bytes 65112 xt target "MASQUERADE"
ip saddr 172.27.0.0/16 oifname != "br-160f55da427c" counter packets 508 bytes 65784 xt target "MASQUERADE"
ip saddr 192.168.16.0/20 oifname != "br-dba077b9b543" counter packets 503 bytes 64784 xt target "MASQUERADE"
ip saddr 192.168.64.0/20 oifname != "br-d44fef8fd602" counter packets 1282 bytes 111308 xt target "MASQUERADE"
ip saddr 172.30.0.0/16 oifname != "br-af4c9c2aa60a" counter packets 2503 bytes 190324 xt target "MASQUERADE"
ip saddr 172.21.0.0/16 oifname != "br-9d6c95e8d80c" counter packets 1289 bytes 112644 xt target "MASQUERADE"
ip saddr 172.23.0.0/16 oifname != "br-679db9b21e00" counter packets 506 bytes 65384 xt target "MASQUERADE"
ip saddr 172.24.0.0/16 oifname != "br-40094534a5ee" counter packets 508 bytes 65784 xt target "MASQUERADE"
ip saddr 172.26.0.0/16 oifname != "br-3dcb6ef83ea1" counter packets 508 bytes 65784 xt target "MASQUERADE"
ip saddr 172.20.0.0/16 oifname != "br-3a760a74f4f6" counter packets 1153 bytes 104344 xt target "MASQUERADE"
ip saddr 192.168.80.0/20 oifname != "br-3636e05760a9" counter packets 546 bytes 70540 xt target "MASQUERADE"
ip saddr 172.29.0.0/16 oifname != "br-b3240c822bce" counter packets 551 bytes 71492 xt target "MASQUERADE"
ip saddr 172.19.0.0/16 oifname != "br-c70303d221ee" counter packets 1136 bytes 106592 xt target "MASQUERADE"
}
}
# Warning: table ip filter is managed by iptables-nft, do not touch!
table ip filter {
chain DOCKER {
ip daddr 172.17.0.5 iifname != "docker0" oifname "docker0" tcp dport 5432 counter packets 0 bytes 0 accept
ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 443 counter packets 0 bytes 0 accept
ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 80 counter packets 0 bytes 0 accept
ip daddr 172.30.0.10 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 3000 counter packets 0 bytes 0 accept
ip daddr 172.30.0.5 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 8000 counter packets 0 bytes 0 accept
ip daddr 172.30.0.3 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 6789 counter packets 0 bytes 0 accept
ip daddr 172.28.0.3 iifname != "br-669fda75f1ac" oifname "br-669fda75f1ac" tcp dport 8080 counter packets 0 bytes 0 accept
ip daddr 172.30.0.4 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 5540 counter packets 0 bytes 0 accept
ip daddr 172.31.0.2 iifname != "br-ec480f77ac34" oifname "br-ec480f77ac34" tcp dport 6379 counter packets 0 bytes 0 accept
ip daddr 192.168.48.3 iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" tcp dport 8081 counter packets 0 bytes 0 accept
ip daddr 192.168.48.2 iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" tcp dport 5432 counter packets 0 bytes 0 accept
ip daddr 172.25.0.2 iifname != "br-4b504efd6080" oifname "br-4b504efd6080" tcp dport 9001 counter packets 0 bytes 0 accept
ip daddr 172.25.0.2 iifname != "br-4b504efd6080" oifname "br-4b504efd6080" tcp dport 9000 counter packets 0 bytes 0 accept
ip daddr 172.29.0.2 iifname != "br-b3240c822bce" oifname "br-b3240c822bce" tcp dport 15672 counter packets 0 bytes 0 accept
ip daddr 172.29.0.2 iifname != "br-b3240c822bce" oifname "br-b3240c822bce" tcp dport 5672 counter packets 0 bytes 0 accept
ip daddr 192.168.32.2 iifname != "br-613eb68ef5fb" oifname "br-613eb68ef5fb" tcp dport 1433 counter packets 0 bytes 0 accept
ip daddr 172.17.0.3 iifname != "docker0" oifname "docker0" tcp dport 5000 counter packets 0 bytes 0 accept
ip daddr 172.17.0.2 iifname != "docker0" oifname "docker0" tcp dport 15672 counter packets 0 bytes 0 accept
ip daddr 172.17.0.2 iifname != "docker0" oifname "docker0" tcp dport 5672 counter packets 0 bytes 0 accept
ip daddr 192.168.80.4 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 5432 counter packets 0 bytes 0 accept
ip daddr 192.168.80.3 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 15672 counter packets 0 bytes 0 accept
ip daddr 192.168.80.3 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 5672 counter packets 0 bytes 0 accept
ip daddr 192.168.80.2 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 9001 counter packets 0 bytes 0 accept
ip daddr 192.168.80.2 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 9000 counter packets 0 bytes 0 accept
ip daddr 172.19.0.2 iifname != "br-c70303d221ee" oifname "br-c70303d221ee" tcp dport 15672 counter packets 0 bytes 0 accept
ip daddr 172.19.0.2 iifname != "br-c70303d221ee" oifname "br-c70303d221ee" tcp dport 5672 counter packets 0 bytes 0 accept
iifname != "br-c70303d221ee" oifname "br-c70303d221ee" counter packets 0 bytes 0 drop
iifname != "br-b3240c822bce" oifname "br-b3240c822bce" counter packets 0 bytes 0 drop
iifname != "br-3636e05760a9" oifname "br-3636e05760a9" counter packets 0 bytes 0 drop
iifname != "br-3a760a74f4f6" oifname "br-3a760a74f4f6" counter packets 0 bytes 0 drop
iifname != "br-3dcb6ef83ea1" oifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 drop
iifname != "br-40094534a5ee" oifname "br-40094534a5ee" counter packets 0 bytes 0 drop
iifname != "br-679db9b21e00" oifname "br-679db9b21e00" counter packets 0 bytes 0 drop
iifname != "br-9d6c95e8d80c" oifname "br-9d6c95e8d80c" counter packets 0 bytes 0 drop
iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
iifname != "br-d44fef8fd602" oifname "br-d44fef8fd602" counter packets 0 bytes 0 drop
iifname != "br-dba077b9b543" oifname "br-dba077b9b543" counter packets 0 bytes 0 drop
iifname != "br-160f55da427c" oifname "br-160f55da427c" counter packets 0 bytes 0 drop
iifname != "br-07a5e2f2c42f" oifname "br-07a5e2f2c42f" counter packets 0 bytes 0 drop
iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop
iifname != "br-613eb68ef5fb" oifname "br-613eb68ef5fb" counter packets 0 bytes 0 drop
iifname != "br-4b504efd6080" oifname "br-4b504efd6080" counter packets 0 bytes 0 drop
iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" counter packets 0 bytes 0 drop
iifname != "br-ec480f77ac34" oifname "br-ec480f77ac34" counter packets 0 bytes 0 drop
iifname != "br-669fda75f1ac" oifname "br-669fda75f1ac" counter packets 0 bytes 0 drop
iifname != "br-65f6dc782562" oifname "br-65f6dc782562" counter packets 0 bytes 0 drop
}
chain DOCKER-FORWARD {
counter packets 6530319 bytes 11196484299 jump DOCKER-CT
counter packets 3312487 bytes 5001091084 jump DOCKER-INTERNAL
counter packets 3312487 bytes 5001091084 jump DOCKER-BRIDGE
iifname "br-c70303d221ee" counter packets 0 bytes 0 accept
iifname "br-b3240c822bce" counter packets 0 bytes 0 accept
iifname "br-3636e05760a9" counter packets 43 bytes 9355 accept
iifname "br-3a760a74f4f6" counter packets 0 bytes 0 accept
iifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 accept
iifname "br-40094534a5ee" counter packets 0 bytes 0 accept
iifname "br-679db9b21e00" counter packets 0 bytes 0 accept
iifname "br-9d6c95e8d80c" counter packets 0 bytes 0 accept
iifname "br-af4c9c2aa60a" counter packets 2761311 bytes 4959915711 accept
iifname "br-d44fef8fd602" counter packets 0 bytes 0 accept
iifname "br-dba077b9b543" counter packets 0 bytes 0 accept
iifname "br-160f55da427c" counter packets 0 bytes 0 accept
iifname "br-07a5e2f2c42f" counter packets 0 bytes 0 accept
iifname "docker0" counter packets 460394 bytes 25754554 accept
iifname "br-613eb68ef5fb" counter packets 10805 bytes 1792862 accept
iifname "br-4b504efd6080" counter packets 33 bytes 2892 accept
iifname "br-ef6df03f71a0" counter packets 0 bytes 0 accept
iifname "br-ec480f77ac34" counter packets 0 bytes 0 accept
iifname "br-669fda75f1ac" counter packets 0 bytes 0 accept
iifname "br-65f6dc782562" counter packets 0 bytes 0 accept
}
chain DOCKER-BRIDGE {
oifname "br-c70303d221ee" counter packets 0 bytes 0 jump DOCKER
oifname "br-b3240c822bce" counter packets 0 bytes 0 jump DOCKER
oifname "br-3636e05760a9" counter packets 0 bytes 0 jump DOCKER
oifname "br-3a760a74f4f6" counter packets 0 bytes 0 jump DOCKER
oifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 jump DOCKER
oifname "br-40094534a5ee" counter packets 0 bytes 0 jump DOCKER
oifname "br-679db9b21e00" counter packets 0 bytes 0 jump DOCKER
oifname "br-9d6c95e8d80c" counter packets 0 bytes 0 jump DOCKER
oifname "br-af4c9c2aa60a" counter packets 118 bytes 8400 jump DOCKER
oifname "br-d44fef8fd602" counter packets 0 bytes 0 jump DOCKER
oifname "br-dba077b9b543" counter packets 0 bytes 0 jump DOCKER
oifname "br-160f55da427c" counter packets 0 bytes 0 jump DOCKER
oifname "br-07a5e2f2c42f" counter packets 0 bytes 0 jump DOCKER
oifname "docker0" counter packets 0 bytes 0 jump DOCKER
oifname "br-613eb68ef5fb" counter packets 0 bytes 0 jump DOCKER
oifname "br-4b504efd6080" counter packets 0 bytes 0 jump DOCKER
oifname "br-ef6df03f71a0" counter packets 0 bytes 0 jump DOCKER
oifname "br-ec480f77ac34" counter packets 0 bytes 0 jump DOCKER
oifname "br-669fda75f1ac" counter packets 0 bytes 0 jump DOCKER
oifname "br-65f6dc782562" counter packets 0 bytes 0 jump DOCKER
}
chain DOCKER-CT {
oifname "br-c70303d221ee" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-b3240c822bce" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-3636e05760a9" xt match "conntrack" counter packets 35 bytes 23113 accept
oifname "br-3a760a74f4f6" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-3dcb6ef83ea1" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-40094534a5ee" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-679db9b21e00" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-9d6c95e8d80c" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-af4c9c2aa60a" xt match "conntrack" counter packets 2488066 bytes 1053784742 accept
oifname "br-d44fef8fd602" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-dba077b9b543" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-160f55da427c" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-07a5e2f2c42f" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "docker0" xt match "conntrack" counter packets 666252 bytes 5079577802 accept
oifname "br-613eb68ef5fb" xt match "conntrack" counter packets 7926 bytes 7636543 accept
oifname "br-4b504efd6080" xt match "conntrack" counter packets 29 bytes 10870 accept
oifname "br-ef6df03f71a0" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-ec480f77ac34" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-669fda75f1ac" xt match "conntrack" counter packets 0 bytes 0 accept
oifname "br-65f6dc782562" xt match "conntrack" counter packets 0 bytes 0 accept
}
chain DOCKER-INTERNAL {
}
chain FORWARD {
type filter hook forward priority filter; policy drop;
counter packets 33747166 bytes 176750349038 jump DOCKER-USER
counter packets 6530319 bytes 11196484299 jump DOCKER-FORWARD
}
chain DOCKER-USER {
oifname "mlab*" counter packets 15657582 bytes 162801189460 accept
iifname "mlab*" counter packets 10958315 bytes 687322055 accept
oifname "incusbr0" counter packets 388768 bytes 2053271282 accept
iifname "incusbr0" counter packets 212182 bytes 12081942 accept
}
}
# Warning: table ip6 nat is managed by iptables-nft, do not touch!
table ip6 nat {
chain DOCKER {
}
chain PREROUTING {
type nat hook prerouting priority dstnat; policy accept;
xt match "addrtype" counter packets 532 bytes 113834 jump DOCKER
}
chain OUTPUT {
type nat hook output priority dstnat; policy accept;
ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
}
table ip6 filter {
chain DOCKER {
}
chain DOCKER-FORWARD {
counter packets 0 bytes 0 jump DOCKER-CT
counter packets 0 bytes 0 jump DOCKER-INTERNAL
counter packets 0 bytes 0 jump DOCKER-BRIDGE
}
chain DOCKER-BRIDGE {
}
chain DOCKER-CT {
}
chain DOCKER-INTERNAL {
}
chain FORWARD {
type filter hook forward priority filter; policy accept;
counter packets 0 bytes 0 jump DOCKER-USER
counter packets 0 bytes 0 jump DOCKER-FORWARD
}
chain DOCKER-USER {
}
}
table ip raw {
chain PREROUTING {
type filter hook prerouting priority raw; policy accept;
ip daddr 172.19.0.2 iifname != "br-c70303d221ee" counter packets 0 bytes 0 drop
ip daddr 192.168.80.2 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop
ip daddr 192.168.80.3 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 15673 counter packets 0 bytes 0 drop
ip daddr 192.168.80.4 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop
ip daddr 172.17.0.2 iifname != "docker0" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 57732 counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 57733 counter packets 0 bytes 0 drop
ip daddr 172.17.0.3 iifname != "docker0" counter packets 0 bytes 0 drop
ip daddr 172.17.0.4 iifname != "docker0" counter packets 0 bytes 0 drop
ip daddr 192.168.32.2 iifname != "br-613eb68ef5fb" counter packets 0 bytes 0 drop
ip daddr 172.30.0.7 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 172.29.0.2 iifname != "br-b3240c822bce" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 15672 counter packets 0 bytes 0 drop
ip daddr 172.25.0.2 iifname != "br-4b504efd6080" counter packets 0 bytes 0 drop
ip daddr 172.30.0.9 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 192.168.48.2 iifname != "br-ef6df03f71a0" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 5432 counter packets 0 bytes 0 drop
ip daddr 192.168.48.3 iifname != "br-ef6df03f71a0" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 8081 counter packets 0 bytes 0 drop
ip daddr 172.30.0.8 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 172.31.0.2 iifname != "br-ec480f77ac34" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 6379 counter packets 0 bytes 0 drop
ip daddr 172.30.0.4 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 8001 counter packets 0 bytes 0 drop
ip daddr 172.31.0.3 iifname != "br-ec480f77ac34" counter packets 0 bytes 0 drop
ip daddr 172.28.0.2 iifname != "br-669fda75f1ac" counter packets 0 bytes 0 drop
ip daddr 172.30.0.6 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 172.28.0.3 iifname != "br-669fda75f1ac" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 28080 counter packets 0 bytes 0 drop
ip daddr 172.30.0.3 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 6789 counter packets 0 bytes 0 drop
ip daddr 172.30.0.5 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 172.22.0.2 iifname != "br-65f6dc782562" counter packets 0 bytes 0 drop
ip daddr 172.30.0.10 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 172.22.0.3 iifname != "br-65f6dc782562" counter packets 0 bytes 0 drop
ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop
ip daddr 172.17.0.5 iifname != "docker0" counter packets 0 bytes 0 drop
ip daddr 127.0.0.1 iifname != "lo" tcp dport 55541 counter packets 0 bytes 0 drop
}
}
table ip mangle {
chain FORWARD {
type filter hook forward priority mangle; policy accept;
tcp flags & (syn | rst) == syn counter packets 13760 bytes 825476 xt target "TCPMSS"
}
}
+22
View File
@@ -0,0 +1,22 @@
-P INPUT ACCEPT
-P FORWARD DROP
-P OUTPUT ACCEPT
-N DOCKER
-N DOCKER-BRIDGE
-N DOCKER-CT
-N DOCKER-FORWARD
-N DOCKER-INTERNAL
-N DOCKER-USER
-N f2b-sshd
-A INPUT -p tcp -m multiport --dports 22 -j f2b-sshd
-A FORWARD -j DOCKER-USER
-A FORWARD -j DOCKER-FORWARD
-A DOCKER ! -i docker0 -o docker0 -j DROP
-A DOCKER-BRIDGE -o docker0 -j DOCKER
-A DOCKER-CT -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A DOCKER-FORWARD -j DOCKER-CT
-A DOCKER-FORWARD -j DOCKER-INTERNAL
-A DOCKER-FORWARD -j DOCKER-BRIDGE
-A DOCKER-FORWARD -i docker0 -j ACCEPT
-A f2b-sshd -s 192.0.2.55/32 -j REJECT --reject-with icmp-port-unreachable
-A f2b-sshd -j RETURN
+103
View File
@@ -0,0 +1,103 @@
table ip nat {
chain DOCKER {
}
chain PREROUTING {
type nat hook prerouting priority dstnat; policy accept;
xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
chain OUTPUT {
type nat hook output priority dstnat; policy accept;
ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
chain POSTROUTING {
type nat hook postrouting priority srcnat; policy accept;
ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 0 bytes 0 xt target "MASQUERADE"
}
}
table ip filter {
chain DOCKER {
iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop
}
chain DOCKER-FORWARD {
counter packets 0 bytes 0 jump DOCKER-CT
counter packets 0 bytes 0 jump DOCKER-INTERNAL
counter packets 0 bytes 0 jump DOCKER-BRIDGE
iifname "docker0" counter packets 0 bytes 0 accept
}
chain DOCKER-BRIDGE {
oifname "docker0" counter packets 0 bytes 0 jump DOCKER
}
chain DOCKER-CT {
oifname "docker0" xt match "conntrack" counter packets 0 bytes 0 accept
}
chain DOCKER-INTERNAL {
}
chain FORWARD {
type filter hook forward priority filter; policy drop;
counter packets 0 bytes 0 jump DOCKER-USER
counter packets 0 bytes 0 jump DOCKER-FORWARD
}
chain DOCKER-USER {
}
chain f2b-sshd {
ip saddr 192.0.2.55 counter packets 0 bytes 0 xt target "REJECT"
counter packets 0 bytes 0 return
}
chain INPUT {
type filter hook input priority filter; policy accept;
ip protocol tcp xt match "multiport" counter packets 0 bytes 0 jump f2b-sshd
}
}
table ip6 nat {
chain DOCKER {
}
chain PREROUTING {
type nat hook prerouting priority dstnat; policy accept;
xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
chain OUTPUT {
type nat hook output priority dstnat; policy accept;
ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
}
table ip6 filter {
chain DOCKER {
}
chain DOCKER-FORWARD {
counter packets 0 bytes 0 jump DOCKER-CT
counter packets 0 bytes 0 jump DOCKER-INTERNAL
counter packets 0 bytes 0 jump DOCKER-BRIDGE
}
chain DOCKER-BRIDGE {
}
chain DOCKER-CT {
}
chain DOCKER-INTERNAL {
}
chain FORWARD {
type filter hook forward priority filter; policy accept;
counter packets 0 bytes 0 jump DOCKER-USER
counter packets 0 bytes 0 jump DOCKER-FORWARD
}
chain DOCKER-USER {
}
}
+105
View File
@@ -0,0 +1,105 @@
table ip nat {
chain DOCKER {
}
chain PREROUTING {
type nat hook prerouting priority dstnat; policy accept;
xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
chain OUTPUT {
type nat hook output priority dstnat; policy accept;
ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
chain POSTROUTING {
type nat hook postrouting priority srcnat; policy accept;
ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 0 bytes 0 xt target "MASQUERADE"
}
}
table ip filter {
chain DOCKER {
iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop
}
chain DOCKER-FORWARD {
counter packets 0 bytes 0 jump DOCKER-CT
counter packets 0 bytes 0 jump DOCKER-INTERNAL
counter packets 0 bytes 0 jump DOCKER-BRIDGE
iifname "docker0" counter packets 0 bytes 0 accept
}
chain DOCKER-BRIDGE {
oifname "docker0" counter packets 0 bytes 0 jump DOCKER
}
chain DOCKER-CT {
oifname "docker0" xt match "conntrack" counter packets 0 bytes 0 accept
}
chain DOCKER-INTERNAL {
}
chain FORWARD {
type filter hook forward priority filter; policy drop;
counter packets 0 bytes 0 jump DOCKER-USER
counter packets 0 bytes 0 jump DOCKER-FORWARD
}
chain DOCKER-USER {
}
}
table ip6 nat {
chain DOCKER {
}
chain PREROUTING {
type nat hook prerouting priority dstnat; policy accept;
xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
chain OUTPUT {
type nat hook output priority dstnat; policy accept;
ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
}
table ip6 filter {
chain DOCKER {
}
chain DOCKER-FORWARD {
counter packets 0 bytes 0 jump DOCKER-CT
counter packets 0 bytes 0 jump DOCKER-INTERNAL
counter packets 0 bytes 0 jump DOCKER-BRIDGE
}
chain DOCKER-BRIDGE {
}
chain DOCKER-CT {
}
chain DOCKER-INTERNAL {
}
chain FORWARD {
type filter hook forward priority filter; policy accept;
counter packets 0 bytes 0 jump DOCKER-USER
counter packets 0 bytes 0 jump DOCKER-FORWARD
}
chain DOCKER-USER {
}
}
table inet f2b-table {
set addr-set-sshd {
type ipv4_addr
flags interval
elements = { 192.0.2.55 }
}
chain f2b-chain {
type filter hook input priority filter - 1; policy accept;
tcp dport 22 ip saddr @addr-set-sshd reject with icmp port-unreachable
}
}
+478
View File
@@ -0,0 +1,478 @@
table ip nat {
chain DOCKER {
}
chain PREROUTING {
type nat hook prerouting priority dstnat; policy accept;
xt match "addrtype" counter packets 2 bytes 1160 jump DOCKER
}
chain OUTPUT {
type nat hook output priority dstnat; policy accept;
ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
chain POSTROUTING {
type nat hook postrouting priority srcnat; policy accept;
ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 0 bytes 0 xt target "MASQUERADE"
}
}
table ip filter {
chain DOCKER {
iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop
}
chain DOCKER-FORWARD {
counter packets 0 bytes 0 jump DOCKER-CT
counter packets 0 bytes 0 jump DOCKER-INTERNAL
counter packets 0 bytes 0 jump DOCKER-BRIDGE
iifname "docker0" counter packets 0 bytes 0 accept
}
chain DOCKER-BRIDGE {
oifname "docker0" counter packets 0 bytes 0 jump DOCKER
}
chain DOCKER-CT {
oifname "docker0" xt match "conntrack" counter packets 0 bytes 0 accept
}
chain DOCKER-INTERNAL {
}
chain FORWARD {
type filter hook forward priority filter; policy drop;
counter packets 0 bytes 0 jump DOCKER-USER
counter packets 0 bytes 0 jump DOCKER-FORWARD
counter packets 0 bytes 0 jump ufw-before-logging-forward
counter packets 0 bytes 0 jump ufw-before-forward
counter packets 0 bytes 0 jump ufw-after-forward
counter packets 0 bytes 0 jump ufw-after-logging-forward
counter packets 0 bytes 0 jump ufw-reject-forward
counter packets 0 bytes 0 jump ufw-track-forward
}
chain DOCKER-USER {
}
chain ufw-before-logging-input {
}
chain ufw-before-logging-output {
}
chain ufw-before-logging-forward {
}
chain ufw-before-input {
iifname "lo" counter packets 0 bytes 0 accept
xt match "conntrack" counter packets 0 bytes 0 accept
xt match "conntrack" counter packets 0 bytes 0 jump ufw-logging-deny
xt match "conntrack" counter packets 0 bytes 0 drop
ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept
ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept
ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept
ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept
udp sport 67 udp dport 68 counter packets 0 bytes 0 accept
counter packets 0 bytes 0 jump ufw-not-local
ip daddr 224.0.0.251 udp dport 5353 counter packets 0 bytes 0 accept
ip daddr 239.255.255.250 udp dport 1900 counter packets 0 bytes 0 accept
counter packets 0 bytes 0 jump ufw-user-input
}
chain ufw-before-output {
oifname "lo" counter packets 0 bytes 0 accept
xt match "conntrack" counter packets 0 bytes 0 accept
counter packets 0 bytes 0 jump ufw-user-output
}
chain ufw-before-forward {
xt match "conntrack" counter packets 0 bytes 0 accept
ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept
ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept
ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept
ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept
counter packets 0 bytes 0 jump ufw-user-forward
}
chain ufw-after-input {
udp dport 137 counter packets 0 bytes 0 jump ufw-skip-to-policy-input
udp dport 138 counter packets 0 bytes 0 jump ufw-skip-to-policy-input
tcp dport 139 counter packets 0 bytes 0 jump ufw-skip-to-policy-input
tcp dport 445 counter packets 0 bytes 0 jump ufw-skip-to-policy-input
udp dport 67 counter packets 0 bytes 0 jump ufw-skip-to-policy-input
udp dport 68 counter packets 0 bytes 0 jump ufw-skip-to-policy-input
xt match "addrtype" counter packets 0 bytes 0 jump ufw-skip-to-policy-input
}
chain ufw-after-output {
}
chain ufw-after-forward {
}
chain ufw-after-logging-input {
limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG"
}
chain ufw-after-logging-output {
}
chain ufw-after-logging-forward {
limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG"
}
chain ufw-reject-input {
}
chain ufw-reject-output {
}
chain ufw-reject-forward {
}
chain ufw-track-input {
}
chain ufw-track-output {
ip protocol tcp xt match "conntrack" counter packets 0 bytes 0 accept
ip protocol udp xt match "conntrack" counter packets 0 bytes 0 accept
}
chain ufw-track-forward {
}
chain INPUT {
type filter hook input priority filter; policy drop;
counter packets 1 bytes 76 jump ufw-before-logging-input
counter packets 1 bytes 76 jump ufw-before-input
counter packets 0 bytes 0 jump ufw-after-input
counter packets 0 bytes 0 jump ufw-after-logging-input
counter packets 0 bytes 0 jump ufw-reject-input
counter packets 0 bytes 0 jump ufw-track-input
}
chain OUTPUT {
type filter hook output priority filter; policy accept;
counter packets 1 bytes 76 jump ufw-before-logging-output
counter packets 1 bytes 76 jump ufw-before-output
counter packets 1 bytes 76 jump ufw-after-output
counter packets 1 bytes 76 jump ufw-after-logging-output
counter packets 1 bytes 76 jump ufw-reject-output
counter packets 1 bytes 76 jump ufw-track-output
}
chain ufw-logging-deny {
xt match "conntrack" limit rate 3/minute burst 10 packets counter packets 0 bytes 0 return
limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG"
}
chain ufw-logging-allow {
limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG"
}
chain ufw-skip-to-policy-input {
counter packets 0 bytes 0 drop
}
chain ufw-skip-to-policy-output {
counter packets 0 bytes 0 accept
}
chain ufw-skip-to-policy-forward {
counter packets 0 bytes 0 drop
}
chain ufw-not-local {
xt match "addrtype" counter packets 0 bytes 0 return
xt match "addrtype" counter packets 0 bytes 0 return
xt match "addrtype" counter packets 0 bytes 0 return
limit rate 3/minute burst 10 packets counter packets 0 bytes 0 jump ufw-logging-deny
counter packets 0 bytes 0 drop
}
chain ufw-user-input {
tcp dport 22 counter packets 0 bytes 0 accept
tcp dport 8080 counter packets 0 bytes 0 accept
udp dport 51820 counter packets 0 bytes 0 accept
}
chain ufw-user-output {
}
chain ufw-user-forward {
tcp dport 80 counter packets 0 bytes 0 accept
iifname "mesh0" tcp dport 443 counter packets 0 bytes 0 accept
}
chain ufw-user-logging-input {
}
chain ufw-user-logging-output {
}
chain ufw-user-logging-forward {
}
chain ufw-user-limit {
limit rate 3/minute burst 5 packets counter packets 0 bytes 0 xt target "LOG"
counter packets 0 bytes 0 xt target "REJECT"
}
chain ufw-user-limit-accept {
counter packets 0 bytes 0 accept
}
}
table ip6 nat {
chain DOCKER {
}
chain PREROUTING {
type nat hook prerouting priority dstnat; policy accept;
xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
chain OUTPUT {
type nat hook output priority dstnat; policy accept;
ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER
}
}
table ip6 filter {
chain DOCKER {
}
chain DOCKER-FORWARD {
counter packets 0 bytes 0 jump DOCKER-CT
counter packets 0 bytes 0 jump DOCKER-INTERNAL
counter packets 0 bytes 0 jump DOCKER-BRIDGE
}
chain DOCKER-BRIDGE {
}
chain DOCKER-CT {
}
chain DOCKER-INTERNAL {
}
chain FORWARD {
type filter hook forward priority filter; policy drop;
counter packets 0 bytes 0 jump DOCKER-USER
counter packets 0 bytes 0 jump DOCKER-FORWARD
counter packets 0 bytes 0 jump ufw6-before-logging-forward
counter packets 0 bytes 0 jump ufw6-before-forward
counter packets 0 bytes 0 jump ufw6-after-forward
counter packets 0 bytes 0 jump ufw6-after-logging-forward
counter packets 0 bytes 0 jump ufw6-reject-forward
counter packets 0 bytes 0 jump ufw6-track-forward
}
chain DOCKER-USER {
}
chain ufw6-before-logging-input {
}
chain ufw6-before-logging-output {
}
chain ufw6-before-logging-forward {
}
chain ufw6-before-input {
iifname "lo" counter packets 0 bytes 0 accept
xt match "rt" counter packets 0 bytes 0 drop
xt match "conntrack" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
xt match "conntrack" counter packets 0 bytes 0 jump ufw6-logging-deny
xt match "conntrack" counter packets 0 bytes 0 drop
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 ip6 daddr fe80::/10 udp sport 547 udp dport 546 counter packets 0 bytes 0 accept
ip6 daddr ff02::fb udp dport 5353 counter packets 0 bytes 0 accept
ip6 daddr ff02::f udp dport 1900 counter packets 0 bytes 0 accept
counter packets 0 bytes 0 jump ufw6-user-input
}
chain ufw6-before-output {
oifname "lo" counter packets 0 bytes 0 accept
xt match "rt" counter packets 0 bytes 0 drop
xt match "conntrack" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept
counter packets 0 bytes 0 jump ufw6-user-output
}
chain ufw6-before-forward {
xt match "rt" counter packets 0 bytes 0 drop
xt match "conntrack" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept
counter packets 0 bytes 0 jump ufw6-user-forward
}
chain ufw6-after-input {
udp dport 137 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input
udp dport 138 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input
tcp dport 139 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input
tcp dport 445 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input
udp dport 546 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input
udp dport 547 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input
}
chain ufw6-after-output {
}
chain ufw6-after-forward {
}
chain ufw6-after-logging-input {
limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG"
}
chain ufw6-after-logging-output {
}
chain ufw6-after-logging-forward {
limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG"
}
chain ufw6-reject-input {
}
chain ufw6-reject-output {
}
chain ufw6-reject-forward {
}
chain ufw6-track-input {
}
chain ufw6-track-output {
meta l4proto tcp xt match "conntrack" counter packets 0 bytes 0 accept
meta l4proto udp xt match "conntrack" counter packets 0 bytes 0 accept
}
chain ufw6-track-forward {
}
chain INPUT {
type filter hook input priority filter; policy drop;
counter packets 1 bytes 128 jump ufw6-before-logging-input
counter packets 1 bytes 128 jump ufw6-before-input
counter packets 0 bytes 0 jump ufw6-after-input
counter packets 0 bytes 0 jump ufw6-after-logging-input
counter packets 0 bytes 0 jump ufw6-reject-input
counter packets 0 bytes 0 jump ufw6-track-input
}
chain OUTPUT {
type filter hook output priority filter; policy accept;
counter packets 4 bytes 304 jump ufw6-before-logging-output
counter packets 4 bytes 304 jump ufw6-before-output
counter packets 0 bytes 0 jump ufw6-after-output
counter packets 0 bytes 0 jump ufw6-after-logging-output
counter packets 0 bytes 0 jump ufw6-reject-output
counter packets 0 bytes 0 jump ufw6-track-output
}
chain ufw6-logging-deny {
xt match "conntrack" limit rate 3/minute burst 10 packets counter packets 0 bytes 0 return
limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG"
}
chain ufw6-logging-allow {
limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG"
}
chain ufw6-skip-to-policy-input {
counter packets 0 bytes 0 drop
}
chain ufw6-skip-to-policy-output {
counter packets 0 bytes 0 accept
}
chain ufw6-skip-to-policy-forward {
counter packets 0 bytes 0 drop
}
chain ufw6-user-input {
tcp dport 22 counter packets 0 bytes 0 accept
tcp dport 8080 counter packets 0 bytes 0 accept
udp dport 51820 counter packets 0 bytes 0 accept
}
chain ufw6-user-output {
}
chain ufw6-user-forward {
tcp dport 80 counter packets 0 bytes 0 accept
iifname "mesh0" tcp dport 443 counter packets 0 bytes 0 accept
}
chain ufw6-user-logging-input {
}
chain ufw6-user-logging-output {
}
chain ufw6-user-logging-forward {
}
chain ufw6-user-limit {
limit rate 3/minute burst 5 packets counter packets 0 bytes 0 xt target "LOG"
counter packets 0 bytes 0 xt target "REJECT"
}
chain ufw6-user-limit-accept {
counter packets 0 bytes 0 accept
}
}
+37
View File
@@ -0,0 +1,37 @@
$ ufw allow 22/tcp
Rules updated
Rules updated (v6)
$ ufw --force enable
Firewall is active and enabled on system startup
$ ufw route allow 8080/tcp
Rule added
Rule added (v6)
$ ufw route allow proto tcp to any port 8080 comment 'mesh-host adoption.opening-tcp-8080-forwarded 1a2b3c4d'
Rule updated
Rule updated (v6)
$ ufw allow 5671/tcp
Rule added
Rule added (v6)
$ ufw allow proto tcp to any port 5671 comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d'
Rule updated
Rule updated (v6)
$ ufw allow in on mesh0 to any port 5432 proto tcp comment 'operator note'
Rule added
Rule added (v6)
$ ufw allow in on mesh0 proto tcp to any port 5432 comment 'mesh-host adoption.opening-tcp-5432-incoming 1a2b3c4d'
Rule updated
Rule updated (v6)
$ ufw show added
Added user rules (see 'ufw status' for running firewall):
ufw allow 22/tcp
ufw route allow 8080/tcp comment 'mesh-host adoption.opening-tcp-8080-forwarded 1a2b3c4d'
ufw allow 5671/tcp comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d'
ufw allow in on mesh0 to any port 5432 proto tcp comment 'mesh-host adoption.opening-tcp-5432-incoming 1a2b3c4d'
$ ufw route delete allow 8080/tcp comment 'mesh-host adoption.opening-tcp-8080-forwarded 1a2b3c4d'
Rule deleted
Rule deleted (v6)
$ ufw show added
Added user rules (see 'ufw status' for running firewall):
ufw allow 22/tcp
ufw allow 5671/tcp comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d'
ufw allow in on mesh0 to any port 5432 proto tcp comment 'mesh-host adoption.opening-tcp-5432-incoming 1a2b3c4d'
+40
View File
@@ -0,0 +1,40 @@
Rule deleted
Rule deleted (v6)
rc=0
Rule deleted
Rule deleted (v6)
rc=0
ERROR: Invalid syntax
rc=1
===ADDED2
Added user rules (see 'ufw status' for running firewall):
ufw allow 22/tcp
ufw allow 8080/tcp
ufw route allow 80/tcp comment 'mesh-host adoption.opening-tcp-8081-forwarded 0badf00d'
ufw route allow in on mesh0 to any port 443 proto tcp comment 'mesh-host adoption.opening-tcp-8443-forwarded cafe0001'
ufw allow 51820/udp comment 'mesh-host adoption.opening-udp-51820-incoming 11112222'
Firewall reloaded
reload rc=0
===AFTERRELOAD
3
Firewall stopped and disabled on system startup
===DISABLED
Status: inactive
/etc/ufw/user.rules
3
Firewall is active and enabled on system startup
Status: active
Rule deleted
Rule deleted (v6)
rc=0
Rule deleted
Rule deleted (v6)
rc=0
Could not delete non-existent rule
Could not delete non-existent rule (v6)
wrongcomment rc=0
Added user rules (see 'ufw status' for running firewall):
ufw allow 22/tcp
ufw allow 8080/tcp
ufw allow 51820/udp comment 'mesh-host adoption.opening-udp-51820-incoming 11112222'
Status: active
+21
View File
@@ -0,0 +1,21 @@
$ ufw allow in 9005/tcp
Rules updated
Rules updated (v6)
$ ufw deny in 9006/tcp
Rules updated
Rules updated (v6)
$ ufw allow 9006/tcp
Rules updated
Rules updated (v6)
$ ufw allow out 9007/tcp
Rules updated
Rules updated (v6)
$ ufw allow 9007/tcp
Rules updated
Rules updated (v6)
$ ufw show added
Added user rules (see 'ufw status' for running firewall):
ufw allow 9005/tcp
ufw allow 9006/tcp
ufw allow out 9007/tcp
ufw allow 9007/tcp
@@ -0,0 +1,55 @@
-P INPUT ACCEPT
-P FORWARD ACCEPT
-P OUTPUT ACCEPT
-N DOCKER
-N DOCKER-BRIDGE
-N DOCKER-CT
-N DOCKER-FORWARD
-N DOCKER-INTERNAL
-N DOCKER-USER
-N ufw-after-forward
-N ufw-after-input
-N ufw-after-logging-forward
-N ufw-after-logging-input
-N ufw-after-logging-output
-N ufw-after-output
-N ufw-before-forward
-N ufw-before-input
-N ufw-before-logging-forward
-N ufw-before-logging-input
-N ufw-before-logging-output
-N ufw-before-output
-N ufw-reject-forward
-N ufw-reject-input
-N ufw-reject-output
-N ufw-track-forward
-N ufw-track-input
-N ufw-track-output
-A INPUT -j ufw-before-logging-input
-A INPUT -j ufw-before-input
-A INPUT -j ufw-after-input
-A INPUT -j ufw-after-logging-input
-A INPUT -j ufw-reject-input
-A INPUT -j ufw-track-input
-A FORWARD -j DOCKER-USER
-A FORWARD -j DOCKER-FORWARD
-A FORWARD -j ufw-before-logging-forward
-A FORWARD -j ufw-before-forward
-A FORWARD -j ufw-after-forward
-A FORWARD -j ufw-after-logging-forward
-A FORWARD -j ufw-reject-forward
-A FORWARD -j ufw-track-forward
-A OUTPUT -j ufw-before-logging-output
-A OUTPUT -j ufw-before-output
-A OUTPUT -j ufw-after-output
-A OUTPUT -j ufw-after-logging-output
-A OUTPUT -j ufw-reject-output
-A OUTPUT -j ufw-track-output
-A DOCKER -d 172.17.0.2/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT
-A DOCKER ! -i docker0 -o docker0 -j DROP
-A DOCKER-BRIDGE -o docker0 -j DOCKER
-A DOCKER-CT -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A DOCKER-FORWARD -j DOCKER-CT
-A DOCKER-FORWARD -j DOCKER-INTERNAL
-A DOCKER-FORWARD -j DOCKER-BRIDGE
-A DOCKER-FORWARD -i docker0 -j ACCEPT
@@ -0,0 +1,119 @@
-P INPUT DROP
-P FORWARD DROP
-P OUTPUT ACCEPT
-N DOCKER
-N DOCKER-BRIDGE
-N DOCKER-CT
-N DOCKER-FORWARD
-N DOCKER-INTERNAL
-N DOCKER-USER
-N ufw-after-forward
-N ufw-after-input
-N ufw-after-logging-forward
-N ufw-after-logging-input
-N ufw-after-logging-output
-N ufw-after-output
-N ufw-before-forward
-N ufw-before-input
-N ufw-before-logging-forward
-N ufw-before-logging-input
-N ufw-before-logging-output
-N ufw-before-output
-N ufw-logging-allow
-N ufw-logging-deny
-N ufw-not-local
-N ufw-reject-forward
-N ufw-reject-input
-N ufw-reject-output
-N ufw-skip-to-policy-forward
-N ufw-skip-to-policy-input
-N ufw-skip-to-policy-output
-N ufw-track-forward
-N ufw-track-input
-N ufw-track-output
-N ufw-user-forward
-N ufw-user-input
-N ufw-user-limit
-N ufw-user-limit-accept
-N ufw-user-logging-forward
-N ufw-user-logging-input
-N ufw-user-logging-output
-N ufw-user-output
-A INPUT -j ufw-before-logging-input
-A INPUT -j ufw-before-input
-A INPUT -j ufw-after-input
-A INPUT -j ufw-after-logging-input
-A INPUT -j ufw-reject-input
-A INPUT -j ufw-track-input
-A FORWARD -j DOCKER-USER
-A FORWARD -j DOCKER-FORWARD
-A FORWARD -j ufw-before-logging-forward
-A FORWARD -j ufw-before-forward
-A FORWARD -j ufw-after-forward
-A FORWARD -j ufw-after-logging-forward
-A FORWARD -j ufw-reject-forward
-A FORWARD -j ufw-track-forward
-A OUTPUT -j ufw-before-logging-output
-A OUTPUT -j ufw-before-output
-A OUTPUT -j ufw-after-output
-A OUTPUT -j ufw-after-logging-output
-A OUTPUT -j ufw-reject-output
-A OUTPUT -j ufw-track-output
-A DOCKER -d 172.17.0.2/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT
-A DOCKER ! -i docker0 -o docker0 -j DROP
-A DOCKER-BRIDGE -o docker0 -j DOCKER
-A DOCKER-CT -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A DOCKER-FORWARD -j DOCKER-CT
-A DOCKER-FORWARD -j DOCKER-INTERNAL
-A DOCKER-FORWARD -j DOCKER-BRIDGE
-A DOCKER-FORWARD -i docker0 -j ACCEPT
-A ufw-after-input -p udp -m udp --dport 137 -j ufw-skip-to-policy-input
-A ufw-after-input -p udp -m udp --dport 138 -j ufw-skip-to-policy-input
-A ufw-after-input -p tcp -m tcp --dport 139 -j ufw-skip-to-policy-input
-A ufw-after-input -p tcp -m tcp --dport 445 -j ufw-skip-to-policy-input
-A ufw-after-input -p udp -m udp --dport 67 -j ufw-skip-to-policy-input
-A ufw-after-input -p udp -m udp --dport 68 -j ufw-skip-to-policy-input
-A ufw-after-input -m addrtype --dst-type BROADCAST -j ufw-skip-to-policy-input
-A ufw-after-logging-forward -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW BLOCK] "
-A ufw-after-logging-input -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW BLOCK] "
-A ufw-before-forward -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A ufw-before-forward -p icmp -m icmp --icmp-type 3 -j ACCEPT
-A ufw-before-forward -p icmp -m icmp --icmp-type 11 -j ACCEPT
-A ufw-before-forward -p icmp -m icmp --icmp-type 12 -j ACCEPT
-A ufw-before-forward -p icmp -m icmp --icmp-type 8 -j ACCEPT
-A ufw-before-forward -j ufw-user-forward
-A ufw-before-input -i lo -j ACCEPT
-A ufw-before-input -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A ufw-before-input -m conntrack --ctstate INVALID -j ufw-logging-deny
-A ufw-before-input -m conntrack --ctstate INVALID -j DROP
-A ufw-before-input -p icmp -m icmp --icmp-type 3 -j ACCEPT
-A ufw-before-input -p icmp -m icmp --icmp-type 11 -j ACCEPT
-A ufw-before-input -p icmp -m icmp --icmp-type 12 -j ACCEPT
-A ufw-before-input -p icmp -m icmp --icmp-type 8 -j ACCEPT
-A ufw-before-input -p udp -m udp --sport 67 --dport 68 -j ACCEPT
-A ufw-before-input -j ufw-not-local
-A ufw-before-input -d 224.0.0.251/32 -p udp -m udp --dport 5353 -j ACCEPT
-A ufw-before-input -d 239.255.255.250/32 -p udp -m udp --dport 1900 -j ACCEPT
-A ufw-before-input -j ufw-user-input
-A ufw-before-output -o lo -j ACCEPT
-A ufw-before-output -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A ufw-before-output -j ufw-user-output
-A ufw-logging-allow -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW ALLOW] "
-A ufw-logging-deny -m conntrack --ctstate INVALID -m limit --limit 3/min --limit-burst 10 -j RETURN
-A ufw-logging-deny -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW BLOCK] "
-A ufw-not-local -m addrtype --dst-type LOCAL -j RETURN
-A ufw-not-local -m addrtype --dst-type MULTICAST -j RETURN
-A ufw-not-local -m addrtype --dst-type BROADCAST -j RETURN
-A ufw-not-local -m limit --limit 3/min --limit-burst 10 -j ufw-logging-deny
-A ufw-not-local -j DROP
-A ufw-skip-to-policy-forward -j DROP
-A ufw-skip-to-policy-input -j DROP
-A ufw-skip-to-policy-output -j ACCEPT
-A ufw-track-output -p tcp -m conntrack --ctstate NEW -j ACCEPT
-A ufw-track-output -p udp -m conntrack --ctstate NEW -j ACCEPT
-A ufw-user-input -p tcp -m tcp --dport 22 -j ACCEPT
-A ufw-user-input -p tcp -m tcp --dport 5671 -j ACCEPT
-A ufw-user-input -i mesh0 -p tcp -m tcp --dport 5432 -j ACCEPT
-A ufw-user-limit -m limit --limit 3/min -j LOG --log-prefix "[UFW LIMIT BLOCK] "
-A ufw-user-limit -j REJECT --reject-with icmp-port-unreachable
-A ufw-user-limit-accept -j ACCEPT
+22
View File
@@ -0,0 +1,22 @@
Added user rules (see 'ufw status' for running firewall):
ufw allow 22/tcp
ufw allow 9200
ufw allow from 192.0.2.0/24 to any port 9300 proto tcp
ufw allow in on eth0 to any port 9301 proto tcp
ufw deny 9400/tcp
ufw limit 2222/tcp
ufw allow 9500:9510/tcp
ufw route allow in on mesh0 out on docker0 to any port 8082 proto tcp
ufw allow to 192.0.2.1 port 9600 proto tcp
ufw allow 9700/udp
ufw route allow in on mesh0 to any port 8083 proto tcp
ufw allow 9900/tcp
ufw allow 80,443/tcp
ufw allow in on mesh0 to any port 5432 proto tcp
ufw route allow 8080/tcp
ufw allow out 5671/tcp
ufw deny out 5672/tcp
ufw allow out on eth0 to any port 5673 proto tcp
ufw allow log 9001/tcp
ufw route allow log 8084/tcp
ufw allow in on mesh0 log-all to any port 9002 proto tcp
+8
View File
@@ -0,0 +1,8 @@
Added user rules (see 'ufw status' for running firewall):
ufw allow 22/tcp
ufw allow 8080/tcp
ufw allow 5671/tcp comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d'
ufw allow in on mesh0 to any port 5432 proto tcp comment 'mesh-host adoption.opening-tcp-5432-incoming deadbeef'
ufw route allow 80/tcp comment 'mesh-host adoption.opening-tcp-8081-forwarded 0badf00d'
ufw route allow in on mesh0 to any port 443 proto tcp comment 'mesh-host adoption.opening-tcp-8443-forwarded cafe0001'
ufw allow 51820/udp comment 'mesh-host adoption.opening-udp-51820-incoming 11112222'
+21
View File
@@ -0,0 +1,21 @@
Status: active
To Action From
-- ------ ----
22/tcp ALLOW Anywhere
8080/tcp ALLOW Anywhere
5671/tcp ALLOW Anywhere # mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d
5432/tcp on mesh0 ALLOW Anywhere # mesh-host adoption.opening-tcp-5432-incoming deadbeef
51820/udp ALLOW Anywhere # mesh-host adoption.opening-udp-51820-incoming 11112222
22/tcp (v6) ALLOW Anywhere (v6)
8080/tcp (v6) ALLOW Anywhere (v6)
5671/tcp (v6) ALLOW Anywhere (v6) # mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d
5432/tcp (v6) on mesh0 ALLOW Anywhere (v6) # mesh-host adoption.opening-tcp-5432-incoming deadbeef
51820/udp (v6) ALLOW Anywhere (v6) # mesh-host adoption.opening-udp-51820-incoming 11112222
80/tcp ALLOW FWD Anywhere # mesh-host adoption.opening-tcp-8081-forwarded 0badf00d
443/tcp ALLOW FWD Anywhere on mesh0 # mesh-host adoption.opening-tcp-8443-forwarded cafe0001
80/tcp (v6) ALLOW FWD Anywhere (v6) # mesh-host adoption.opening-tcp-8081-forwarded 0badf00d
443/tcp (v6) ALLOW FWD Anywhere (v6) on mesh0 # mesh-host adoption.opening-tcp-8443-forwarded cafe0001
===STATUSV
+13
View File
@@ -220,6 +220,19 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
if len(fields) != 5 {
t.Errorf("the token has %d fields, expected 5: %v", len(fields), fields)
}
// novox/hq ADR 0100: an adopted node's token says so, and a converged one's is unchanged.
raw, err = json.Marshal(Token{Version: 1, Secret: "s", Adopted: true})
if err != nil {
t.Fatal(err)
}
fields = map[string]any{}
if err := json.Unmarshal(raw, &fields); err != nil {
t.Fatal(err)
}
if fields["adopted"] != true {
t.Errorf("an adopted token does not say \"adopted\": %v", fields)
}
}
func TestACompleteTokenParses(t *testing.T) {
+5
View File
@@ -30,6 +30,11 @@ type Token struct {
Fingerprint string `json:"fingerprint,omitempty"`
Signer []byte `json:"signer,omitempty"`
Secret string `json:"secret"`
// Adopted says this node joins adopted (novox/hq ADR 0100). The host checks it speaks the
// firewall found here before enrolling, because an adopted node keeps that firewall in force.
// Absent for a converged node.
Adopted bool `json:"adopted,omitempty"`
}
// ParseToken reads a token a person pasted.
+42
View File
@@ -1,5 +1,7 @@
package link
import "time"
// The wire formats shared with the control plane, which defines them separately because this
// binary requires nothing present and does not import it. A test on each side asserts the field
// names, so a rename breaks both at once rather than on a real machine months later.
@@ -85,4 +87,44 @@ type Report struct {
// than the send, and the machine reads as caught up with words it has not read yet. Clocks
// cannot answer "which"; the digest is the answer itself.
Declared string `json:"declared,omitempty"`
// Held is what this adopted node found and is keeping as it was until its module is taken
// (novox/hq ADR 0100). Without it an adopted node reads as converged.
Held []Held `json:"held,omitempty"`
// Firewall is the firewall found on this machine — "ufw" or "none" — and empty on a node that
// was never asked, which is every converged one.
Firewall string `json:"firewall,omitempty"`
// Reachable is what can be reached on this machine now: every listening socket and every
// published container port. Only an adopted node reports it; it is what converging the node
// previews, so nothing closes without being named first.
Reachable []Reach `json:"reachable,omitempty"`
}
// Held is one file or container found on an adopted node and kept as it was.
type Held struct {
ID string `json:"id"`
Module string `json:"module"`
Kind string `json:"kind"`
Target string `json:"target"`
Since time.Time `json:"since"`
// Changed is what something other than the mesh did to it since — rewritten, stopped,
// replaced or gone — and empty while it is as found.
Changed string `json:"changed,omitempty"`
// Kept is where a file's original was kept.
Kept string `json:"kept,omitempty"`
}
// Reach is one thing reachable on the machine: a listening socket, or a published container port.
type Reach struct {
Protocol string `json:"protocol"`
Address string `json:"address"`
Port int `json:"port"`
// By is what holds it — a process, or a container's name.
By string `json:"by,omitempty"`
// Published is a container port the runtime publishes, reached on the forwarded path; its
// container's own port is ContainerPort.
Published bool `json:"published,omitempty"`
ContainerPort int `json:"container-port,omitempty"`
}
+8
View File
@@ -120,6 +120,14 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
{Signed{Declaration: []byte("{}"), Signature: []byte("x")}, []string{"declaration", "signature"}},
{Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"},
[]string{"node", "applied", "failed", "refused"}},
// novox/hq ADR 0100: what an adopted node holds, the firewall it was found with, and what
// is reachable on it.
{Report{Node: "n", Held: []Held{{ID: "i"}}, Firewall: "ufw", Reachable: []Reach{{Port: 1}}},
[]string{"node", "held", "firewall", "reachable"}},
{Held{ID: "i", Module: "m", Kind: "file", Target: "/t", Changed: "rewritten", Kept: "/k"},
[]string{"id", "module", "kind", "target", "since", "changed", "kept"}},
{Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "c", Published: true, ContainerPort: 80},
[]string{"protocol", "address", "port", "by", "published", "container-port"}},
} {
raw, err := json.Marshal(c.value)
if err != nil {
+32 -7
View File
@@ -70,15 +70,29 @@ type Announce func(string)
type Roused <-chan struct{}
func Hold(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration) error {
return HoldRoused(ctx, m, apply, say, timeout, nil)
return HoldRoused(ctx, m, apply, say, timeout, nil, nil)
}
// HoldRoused is Hold, told when the machine has reason to think its link is stale.
// Outbox carries reports the node has to say without having been sent anything — what a
// reconcile found changed on an adopted node (novox/hq ADR 0100). Published while the link is up;
// a report made while it is down waits in the channel for the next one. Nil is allowed.
type Outbox <-chan Unasked
// Unasked is one such report, with the way to say whether it reached the mesh. Done is called
// with true only when the broker took it — a node that marked a change said because it queued it
// would never say it again, and the mesh would go on believing nothing changed.
type Unasked struct {
Report Report
Done func(published bool)
}
// HoldRoused is Hold, told when the machine has reason to think its link is stale, and handed
// reports to publish between deliveries.
func HoldRoused(ctx context.Context, m Membership, apply Applier, say Announce,
timeout time.Duration, roused Roused) error {
timeout time.Duration, roused Roused, outbox Outbox) error {
return holdWith(ctx, func(ctx context.Context) error {
return Run(ctx, m, apply, say, timeout)
return Run(ctx, m, apply, say, timeout, outbox)
}, say, roused)
}
@@ -171,7 +185,8 @@ func holdWith(ctx context.Context, run attempt, say Announce, roused Roused) err
//
// Outbound only, and nothing listens on this machine. Returns when the link ends, for any reason;
// Hold is what decides whether to open it again.
func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration) error {
func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration,
outbox Outbox) error {
if say == nil {
say = func(string) {}
}
@@ -254,6 +269,13 @@ func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout
return nil
case <-beat.C:
publishAlive(ctx, channel, m, say, timeout)
case unasked := <-outbox:
// Said without having been asked: a reconcile found what an adopted node holds, or
// its firewall, changed since it last said.
published := publishReport(ctx, channel, m, unasked.Report, say, timeout)
if unasked.Done != nil {
unasked.Done(published)
}
case reason := <-closed:
return fmt.Errorf("the link closed: %v", reason)
case delivery, ok := <-deliveries:
@@ -354,13 +376,14 @@ func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) R
return apply(ctx, signed.Declaration, signed.Signature)
}
// publishReport tells the mesh what this node did, and says whether the broker took it.
func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report,
say Announce, timeout time.Duration) {
say Announce, timeout time.Duration) bool {
report.Node = m.Node
body, err := json.Marshal(report)
if err != nil {
say("cannot encode this node's own report: " + err.Error())
return
return false
}
publish, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
@@ -371,7 +394,9 @@ func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, rep
if err := channel.PublishWithContext(publish, Exchange, KeyReport, true, false,
amqp.Publishing{ContentType: "application/json", Body: body}); err != nil {
say(fmt.Sprintf("applied, and could not tell the mesh: %v", err))
return false
}
return true
}
// publishAlive says this node is here, and nothing else.
+181
View File
@@ -0,0 +1,181 @@
// Package reachable reads what can be reached on this machine now: every listening socket, and
// every container port the runtime publishes (novox/hq ADR 0100).
//
// It is what converging an adopted node previews — each port, whether a module declares it or it
// will close — and what a converged genesis counts before refusing a machine in use. It reads; it
// never decides what is the mesh's.
package reachable
import (
"context"
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/system"
)
// Runner executes a command.
type Runner = system.Runner
// Reach is one thing reachable on this machine, in the words the report carries.
type Reach = link.Reach
// Collect reads the machine's listening sockets and the runtime's published ports. A published
// port is reported once, as published, rather than again as the runtime's proxy listening for it.
func Collect(ctx context.Context, run Runner) ([]Reach, error) {
out, err := run(ctx, "ss", "-Hltunp")
if err != nil {
return nil, fmt.Errorf("reading this machine's listening sockets: %w", err)
}
sockets := Sockets(out)
var published []Reach
if ps, err := run(ctx, "docker", "ps", "--format", "{{.Names}}\t{{.Ports}}"); err == nil {
published = Published(ps)
}
return Merge(sockets, published), nil
}
var process = regexp.MustCompile(`users:\(\("([^"]+)"`)
// Sockets parses `ss -Hltunp`: each line a netid, a state, two queues, the local address and
// port, the peer, and the process when ss may name it.
func Sockets(out string) []Reach {
var reached []Reach
for _, line := range strings.Split(out, "\n") {
fields := strings.Fields(line)
if len(fields) < 5 {
continue
}
protocol := fields[0]
if protocol != "tcp" && protocol != "udp" {
continue
}
address, port, ok := splitLocal(fields[4])
if !ok {
continue
}
r := Reach{Protocol: protocol, Address: address, Port: port}
if m := process.FindStringSubmatch(line); m != nil {
r.By = m[1]
}
reached = append(reached, r)
}
return reached
}
// splitLocal reads "127.0.0.1:53", "[::]:22", "*:22" and "[fe80::1]%veth0:123".
func splitLocal(local string) (string, int, bool) {
i := strings.LastIndex(local, ":")
if i < 0 {
return "", 0, false
}
port, err := strconv.Atoi(local[i+1:])
if err != nil {
return "", 0, false
}
address := local[:i]
if at := strings.Index(address, "%"); at >= 0 {
address = address[:at]
}
address = strings.TrimSuffix(strings.TrimPrefix(address, "["), "]")
if address == "*" {
address = "0.0.0.0"
}
return address, port, true
}
// Published parses `docker ps --format '{{.Names}}\t{{.Ports}}'`. Only what is published on the
// machine counts; a port a container exposes and nothing publishes is not reachable from outside it.
func Published(out string) []Reach {
var reached []Reach
for _, line := range strings.Split(out, "\n") {
name, ports, ok := strings.Cut(strings.TrimSpace(line), "\t")
if !ok {
continue
}
for _, mapping := range strings.Split(ports, ",") {
reached = append(reached, mappingOf(name, strings.TrimSpace(mapping))...)
}
}
return reached
}
// mappingOf reads "0.0.0.0:9000-9001->9000-9001/tcp" into one reach per port.
func mappingOf(name, mapping string) []Reach {
outer, inner, ok := strings.Cut(mapping, "->")
if !ok {
return nil
}
inner, protocol, ok := strings.Cut(inner, "/")
if !ok {
return nil
}
i := strings.LastIndex(outer, ":")
if i < 0 {
return nil
}
address := strings.TrimSuffix(strings.TrimPrefix(outer[:i], "["), "]")
from, to, ok := portRange(outer[i+1:])
if !ok {
return nil
}
cfrom, _, ok := portRange(inner)
if !ok {
return nil
}
var reached []Reach
for p := from; p <= to; p++ {
reached = append(reached, Reach{Protocol: protocol, Address: address, Port: p, By: name,
Published: true, ContainerPort: cfrom + (p - from)})
}
return reached
}
func portRange(s string) (int, int, bool) {
a, b, isRange := strings.Cut(s, "-")
from, err := strconv.Atoi(a)
if err != nil {
return 0, 0, false
}
if !isRange {
return from, from, true
}
to, err := strconv.Atoi(b)
if err != nil || to < from {
return 0, 0, false
}
return from, to, true
}
// Merge puts the published ports beside the sockets, dropping the runtime proxy's own socket for a
// port that is reported as published already, and sorts the whole by port.
func Merge(sockets, published []Reach) []Reach {
key := func(r Reach) string { return r.Protocol + " " + r.Address + " " + strconv.Itoa(r.Port) }
isPublished := map[string]bool{}
for _, p := range published {
isPublished[key(p)] = true
}
var out []Reach
for _, s := range sockets {
if s.By == "docker-proxy" && isPublished[key(s)] {
continue
}
out = append(out, s)
}
out = append(out, published...)
sort.SliceStable(out, func(i, j int) bool {
if out[i].Port != out[j].Port {
return out[i].Port < out[j].Port
}
if out[i].Protocol != out[j].Protocol {
return out[i].Protocol < out[j].Protocol
}
return out[i].Address < out[j].Address
})
return out
}
+100
View File
@@ -0,0 +1,100 @@
package reachable
import (
"context"
"os"
"strings"
"testing"
)
// Defends novox/hq ADR 0100: converging previews every listening socket and every published
// container port. Fixtures are captured from a real machine.
func fixture(t *testing.T, name string) string {
t.Helper()
raw, err := os.ReadFile("testdata/" + name)
if err != nil {
t.Fatal(err)
}
return string(raw)
}
func find(rs []Reach, protocol, address string, port int) (Reach, bool) {
for _, r := range rs {
if r.Protocol == protocol && r.Address == address && r.Port == port {
return r, true
}
}
return Reach{}, false
}
func TestSocketsAreReadWithWhatHoldsThem(t *testing.T) {
got := Sockets(fixture(t, "ss.txt"))
if r, ok := find(got, "tcp", "0.0.0.0", 22); !ok || r.By != "sshd" {
t.Errorf("ssh not read: %+v", r)
}
if r, ok := find(got, "tcp", "::", 445); !ok || r.By != "smbd" {
t.Errorf("an IPv6 wildcard listener not read: %+v", r)
}
if _, ok := find(got, "udp", "fe80::849e:ccff:fea8:24c7", 123); !ok {
t.Error("a link-local address with a scope was not read")
}
if r, ok := find(got, "udp", "127.0.0.1", 53); !ok || r.By != "dnsmasq" {
t.Errorf("a loopback udp socket not read: %+v", r)
}
}
func TestPublishedPortsNameTheirContainerAndItsPort(t *testing.T) {
got := Published(fixture(t, "docker-ps.txt"))
if r, ok := find(got, "tcp", "0.0.0.0", 8770); !ok || r.By != "whisper" || r.ContainerPort != 8000 || !r.Published {
t.Errorf("a published port: %+v", r)
}
if r, ok := find(got, "tcp", "0.0.0.0", 9001); !ok || r.ContainerPort != 9001 {
t.Errorf("a published range was not expanded: %+v", r)
}
if r, ok := find(got, "tcp", "127.0.0.1", 15673); !ok || r.ContainerPort != 15672 {
t.Errorf("a loopback-published port: %+v", r)
}
for _, r := range got {
if r.By == "umami_db" {
t.Errorf("an exposed and unpublished port was reported reachable: %+v", r)
}
}
}
func TestAPublishedPortIsReportedOnceAsPublished(t *testing.T) {
merged := Merge(Sockets(fixture(t, "ss.txt")), Published(fixture(t, "docker-ps.txt")))
n := 0
for _, r := range merged {
if r.Protocol == "tcp" && r.Address == "0.0.0.0" && r.Port == 8770 {
n++
if !r.Published {
t.Errorf("the runtime's proxy was reported instead of the published port: %+v", r)
}
}
}
if n != 1 {
t.Errorf("port 8770 reported %d times", n)
}
if _, ok := find(merged, "tcp", "0.0.0.0", 22); !ok {
t.Error("a socket was lost in the merge")
}
}
func TestCollectAsksSsAndTheRuntime(t *testing.T) {
var asked []string
run := func(_ context.Context, name string, args ...string) (string, error) {
asked = append(asked, name+" "+strings.Join(args, " "))
if name == "ss" {
return fixture(t, "ss.txt"), nil
}
return fixture(t, "docker-ps.txt"), nil
}
got, err := Collect(context.Background(), run)
if err != nil || len(got) == 0 {
t.Fatalf("%v %v", got, err)
}
if len(asked) != 2 {
t.Errorf("asked %v", asked)
}
}
+7
View File
@@ -0,0 +1,7 @@
mesh-controller-check-adoption 127.0.0.1:55541->5432/tcp
umami_db 5432/tcp
whisper 0.0.0.0:8770->8000/tcp, [::]:8770->8000/tcp
keycloak 8443/tcp, 127.0.0.1:28080->8080/tcp
minio-lb 0.0.0.0:9000-9001->9000-9001/tcp, [::]:9000-9001->9000-9001/tcp
wonderful_mahavira
anton-lavinmq 0.0.0.0:5680->5672/tcp, [::]:5680->5672/tcp, 127.0.0.1:15673->15672/tcp
+23
View File
@@ -0,0 +1,23 @@
udp UNCONN 0 0 0.0.0.0:55558 0.0.0.0:* users:(("firefox",pid=2283907,fd=288))
udp UNCONN 0 0 0.0.0.0:59541 0.0.0.0:* users:(("firefox",pid=2283907,fd=241))
udp UNCONN 0 0 127.0.0.1:53 0.0.0.0:* users:(("dnsmasq",pid=1189392,fd=6))
udp UNCONN 0 0 0.0.0.0:33525 0.0.0.0:* users:(("firefox",pid=2283907,fd=304))
udp UNCONN 0 0 0.0.0.0:41749 0.0.0.0:* users:(("firefox",pid=2283907,fd=351))
tcp LISTEN 0 4096 127.0.0.1:55541 0.0.0.0:* users:(("docker-proxy",pid=4108732,fd=7))
tcp LISTEN 0 4096 0.0.0.0:9001 0.0.0.0:* users:(("docker-proxy",pid=1849130,fd=7))
tcp LISTEN 0 4096 0.0.0.0:8770 0.0.0.0:* users:(("docker-proxy",pid=1920035,fd=7))
tcp LISTEN 0 50 0.0.0.0:445 0.0.0.0:* users:(("smbd",pid=1248,fd=29))
tcp LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=1188536,fd=6))
tcp LISTEN 0 50 0.0.0.0:139 0.0.0.0:* users:(("smbd",pid=1248,fd=30))
tcp LISTEN 0 4096 127.0.0.1:5432 0.0.0.0:* users:(("docker-proxy",pid=1854543,fd=7))
tcp LISTEN 0 32 127.0.0.1:53 0.0.0.0:* users:(("dnsmasq",pid=1189392,fd=7))
tcp LISTEN 0 4096 127.0.0.1:15673 0.0.0.0:* users:(("docker-proxy",pid=3170,fd=7))
tcp LISTEN 0 4096 [::]:9001 [::]:* users:(("docker-proxy",pid=1849138,fd=7))
tcp LISTEN 0 4096 [::]:8770 [::]:* users:(("docker-proxy",pid=1920043,fd=7))
tcp LISTEN 0 50 [::]:445 [::]:* users:(("smbd",pid=1248,fd=27))
tcp LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=1188536,fd=7))
tcp LISTEN 0 50 [::]:139 [::]:* users:(("smbd",pid=1248,fd=28))
udp UNCONN 0 0 [fd42:f8c5:dae:d74c::1]:53 [::]:*
udp UNCONN 0 0 [fe80::849e:ccff:fea8:24c7]%veth6b2b7ba:123 [::]:*
udp UNCONN 0 0 [fe80::e45a:90ff:feca:148f]%vethb5e5a61:123 [::]:*
udp UNCONN 0 0 [fe80::c4ed:ccff:feb1:afd2]%veth005a182:123 [::]:*
+128
View File
@@ -69,6 +69,23 @@ type Applied struct {
// person who edits a managed file watches their change vanish every few minutes with nothing
// anywhere saying why.
Wrote string `json:"wrote,omitempty"`
// Into is set for a file written into rather than over (novox/hq ADR 0102): the format, what
// each of the mesh's keys held before it set them, which of them were absent, and whether the
// file itself was — so undeclaring it gives the machine back exactly what it had.
Into *Into `json:"into,omitempty"`
}
// Into is what a file written into held before the mesh's keys.
type Into struct {
Format string `json:"format"`
Before map[string]json.RawMessage `json:"before,omitempty"`
Absent []string `json:"absent,omitempty"`
Created bool `json:"created,omitempty"`
// Added is, for each key whose declared value is a list, exactly the members the mesh added
// to the machine's list — never a member that was already there. Undeclared, only these go,
// and drift is judged on these alone (novox/hq ADR 0102).
Added map[string][]json.RawMessage `json:"added,omitempty"`
}
// State is the whole of what a node knows about what it has done.
@@ -77,6 +94,117 @@ type State struct {
// undoing in reverse is the only ordering the host can derive without deciding anything.
Resources []Applied `json:"resources"`
UpdatedAt time.Time `json:"updated_at"`
// Held is what this host found on the machine and is keeping as it is, until the module
// declaring it is taken (novox/hq ADR 0100). Never a Resource: nothing here was applied, so
// nothing here is ever removed as an orphan — what is held is not the host's to remove, even
// when its module is unassigned.
Held []Held `json:"held,omitempty"`
// Firewall is the firewall found on this machine when it was first adopted, and whether the
// mesh has since retired it (novox/hq ADR 0100). Nil on a node that was never adopted.
Firewall *FoundFirewall `json:"firewall,omitempty"`
}
// FoundFirewall is what the host found filtering this machine, and what it did about it.
type FoundFirewall struct {
// Kind is ufw or none: an unsupported kind is refused adoption, never recorded.
Kind string `json:"kind"`
// WasActive is whether it was in force when found — which is what converging the node
// retires, and returning it to adopted restores.
WasActive bool `json:"was_active,omitempty"`
// DisabledByMesh is set when converging retired it, so returning to adopted enables it again
// and nothing else ever does.
DisabledByMesh bool `json:"disabled_by_mesh,omitempty"`
// Forward is each family's forward policy as it was before the mesh disabled the firewall,
// by the tool that sets it — recorded before, so a retirement retried puts back what the
// machine had.
Forward map[string]string `json:"forward,omitempty"`
FoundAt time.Time `json:"found_at"`
}
// Held is one thing found on an adopted node — a file, directory or container present at a declared
// path or name, or a service's unit, with no record of this host having made it — kept as it was
// found; or what would reach one: a container mounting found data, an action run in a held
// container (novox/hq ADR 0100, ADR 0103).
type Held struct {
ID string `json:"id"`
Module string `json:"module"`
Kind string `json:"kind"`
Target string `json:"target"`
// Since is when it was first found. It stays held from then until its module is taken, even
// if it disappears: a vanished file is reported, not recreated.
Since time.Time `json:"since"`
// A file's content as found, by digest; its mode and owner; and where the original was kept
// before anything else could happen to it.
Digest string `json:"digest,omitempty"`
Mode string `json:"mode,omitempty"`
Owner string `json:"owner,omitempty"`
Kept string `json:"kept,omitempty"`
// A container's id as found, and whether it was running — or a service's unit, whether it
// was running.
Container string `json:"container,omitempty"`
Running bool `json:"running,omitempty"`
// Why says what was found when it is not the resource's own target: the path or volume a
// container would mount, or the held container an action would run in (novox/hq ADR 0103).
Why string `json:"why,omitempty"`
// Changed is what something other than the mesh has done to it since it was found —
// rewritten, stopped, replaced or gone — and empty while it is as found. Reported, never
// reverted: that is how a predecessor still writing is caught.
Changed string `json:"changed,omitempty"`
ChangedAt time.Time `json:"changed_at,omitempty"`
}
// Recorded reports whether this host has a record, of any origin, of putting something of this
// kind at this target. What it has a record of is not found: it wrote it, in this life of the node
// or an earlier one — including a foundation raised from the bundle and adopted as modules later
// (novox/hq ADR 0078).
func (s State) Recorded(kind, target string) bool {
for _, r := range s.Resources {
if r.Type == kind && r.Target == target {
return true
}
}
return false
}
// HeldAt returns what is held under a resource id.
func (s State) HeldAt(id string) (Held, bool) {
for _, h := range s.Held {
if h.ID == id {
return h, true
}
}
return Held{}, false
}
// RecordHeld adds or replaces what is held under one id, preserving order.
func (s *State) RecordHeld(h Held) {
for i, existing := range s.Held {
if existing.ID == h.ID {
s.Held[i] = h
return
}
}
s.Held = append(s.Held, h)
}
// Release drops a hold, once its module is taken and the host has converged what was held.
func (s *State) Release(id string) {
kept := s.Held[:0]
for _, h := range s.Held {
if h.ID != id {
kept = append(kept, h)
}
}
s.Held = kept
if len(s.Held) == 0 {
s.Held = nil
}
}
// Find returns what was applied under an identity.
+30
View File
@@ -246,3 +246,33 @@ func (arch) AddUserToGroup(ctx context.Context, run Runner, name, group string)
}
return nil
}
// ServiceUnitFile says where the service manager loads a unit from — systemd's FragmentPath. It
// is how the host tells a unit an administrator installed, under /etc or /run, from one a package
// ships under /usr (novox/hq ADR 0103). Empty, with no error, for a unit that loads from nowhere.
func (arch) ServiceUnitFile(ctx context.Context, run Runner, unit string) (string, error) {
out, err := run(ctx, "systemctl", "show", unit, "--property=FragmentPath")
if err != nil {
return "", fmt.Errorf("the service manager did not say where %s comes from: %w", unit, err)
}
for _, line := range strings.Split(out, "\n") {
if path, ok := strings.CutPrefix(strings.TrimSpace(line), "FragmentPath="); ok {
return strings.TrimSpace(path), nil
}
}
return "", nil
}
// ReloadUnits has systemd read its unit files again. A unit file that changed on disk is otherwise
// ignored: a restart runs the unit systemd already loaded, and the new text only takes effect
// after a reload nobody asked for.
func (arch) ReloadUnits(ctx context.Context, run Runner) error {
_, err := run(ctx, "systemctl", "daemon-reload")
return err
}
// ReloadService tells a running unit to read its configuration again, without stopping it.
func (arch) ReloadService(ctx context.Context, run Runner, unit string) error {
_, err := run(ctx, "systemctl", "reload", unit)
return err
}
+3
View File
@@ -178,6 +178,9 @@ func everyShape() []declaration.Type {
// it: the mesh's own code runs as a process on the machine, and only software that
// genuinely needs isolation asks for a container.
declaration.TypeProcess,
// An opening is a rule in the firewall found on the machine, which a partial host neither
// has nor can manage (novox/hq ADR 0100).
declaration.TypeOpening,
}
}