diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index e48aa1a..f8ae993 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -18,6 +18,7 @@ package main import ( "context" "encoding/json" + "errors" "flag" "fmt" "io" @@ -25,6 +26,7 @@ import ( "net/http" "os" "os/signal" + "strconv" "syscall" "time" @@ -126,6 +128,19 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh --packet-filter which packet filter to run (nftables) --extras catalogue modules beyond the floor, comma-separated + The foundation's ports are this machine's, each checked free before anything is + raised and kept as the node's setting for the module that binds it: + --store-port 5432 --bus-port 5671 --amqp-port 5672 --management-port 15672 + --registry-port 5000 (follows --registry, and must agree with it) + --packages-port 3000 --hub-port 51820/udp + --overlay-range the private network's range (default 10.42.0.0/16); refused + if it overlaps an interface or route the machine already has + + --adopted raise a machine in use as an adopted node: what it runs and its + firewall stay as they are, the foundation's filter is not loaded and + the mesh guards its own ports instead, and each module is taken on it + one at a time. Without it, a machine in use is refused + The installer carries a builder, not a control plane. What raises a mesh is therefore the same thing that will maintain it, and the control plane a mesh ends up running is one it built itself, from a repository and a commit it can name and build again. @@ -176,7 +191,9 @@ func parseArgs(args []string) (string, bootstrap.Options, bool, error) { HostService: defaultService, // Longer than the host's 10s: these probes reach a container runtime that may be busy // pulling, and a probe that times out on a working machine is a false refusal. - Timeout: 30 * time.Second, + Ports: bootstrap.DefaultPorts(), + OverlayRange: bootstrap.DefaultOverlayRange, + Timeout: 30 * time.Second, // A socket-activated runtime queued behind the network, and a control plane running its // first `initdb`-shaped wait, are both minutes rather than seconds. Wait: 3 * time.Minute, @@ -210,9 +227,50 @@ func parseArgs(args []string) (string, bootstrap.Options, bool, error) { return "", opts, false, fmt.Errorf( "unexpected argument %q — try `mesh-bootstrap help`", positionals[0]) } + if err := registryAgrees(set, &opts); err != nil { + return "", opts, false, err + } return command, opts, jsonOut, nil } +// registryAgrees makes --registry and --registry-port say one port (novox/hq ADR 0100): the +// registry is raised on the port the node gives it, and every node pulls from the address given. +// Either may be said alone and the other follows; said both ways, they must agree. +func registryAgrees(set *flag.FlagSet, opts *bootstrap.Options) error { + said := map[string]bool{} + set.Visit(func(f *flag.Flag) { said[f.Name] = true }) + host, portText, err := net.SplitHostPort(opts.Registry) + var missing *net.AddrError + if errors.As(err, &missing) && missing.Err == "missing port in address" { + // A host alone, as --registry took before its port became the node's: the registry's + // port — the one given, or the catalogue's — completes it. + port := opts.Ports.Registry + if port == 0 { + port = bootstrap.DefaultPorts().Registry + } + opts.Ports.Registry = port + opts.Registry = net.JoinHostPort(strings.Trim(opts.Registry, "[]"), strconv.Itoa(port)) + return nil + } + if err != nil { + return fmt.Errorf("--registry %q is not host:port: %w", opts.Registry, err) + } + port, err := strconv.Atoi(portText) + if err != nil { + return fmt.Errorf("--registry %q does not end in a port", opts.Registry) + } + switch { + case said["registry-port"] && said["registry"] && port != opts.Ports.Registry: + return fmt.Errorf("--registry %s and --registry-port %d name two ports for one registry", + opts.Registry, opts.Ports.Registry) + case said["registry-port"]: + opts.Registry = net.JoinHostPort(host, strconv.Itoa(opts.Ports.Registry)) + case said["registry"]: + opts.Ports.Registry = port + } + return nil +} + func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { set := flag.NewFlagSet("mesh-bootstrap", flag.ContinueOnError) set.SetOutput(os.Stderr) @@ -255,6 +313,27 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { set.StringVar(&opts.SDKSource.Ref, "sdk-ref", opts.SDKSource.Ref, "what of it to build (default main)") set.StringVar(&opts.Site, "site", "main", "where this machine sits, for the private network") + + // The foundation's ports are this node's (novox/hq ADR 0100): each is checked free before + // anything is raised, and becomes the node's setting for the module that binds it. + for _, p := range []struct { + name, what string + into *int + }{ + {"store-port", "the store", &opts.Ports.Store}, + {"bus-port", "the bus (amqps)", &opts.Ports.Bus}, + {"amqp-port", "the broker's AMQP", &opts.Ports.AMQP}, + {"management-port", "the broker's management, on loopback", &opts.Ports.Management}, + {"registry-port", "the registry", &opts.Ports.Registry}, + {"packages-port", "the package registry", &opts.Ports.Packages}, + {"hub-port", "the private network's hub (udp)", &opts.Ports.Hub}, + } { + set.IntVar(p.into, p.name, *p.into, "the machine's port for "+p.what) + } + set.BoolVar(&opts.Adopted, "adopted", false, + "raise this machine adopted: keep what it runs and its firewall until each module is taken") + set.StringVar(&opts.OverlayRange, "overlay-range", opts.OverlayRange, + "the private network's address range; must not overlap a tunnel the machine already runs") if opts.Answers == nil { opts.Answers = map[string]string{} } diff --git a/cmd/mesh-bootstrap/main_test.go b/cmd/mesh-bootstrap/main_test.go index 2fadbcb..8d19d6c 100644 --- a/cmd/mesh-bootstrap/main_test.go +++ b/cmd/mesh-bootstrap/main_test.go @@ -164,3 +164,43 @@ func TestTheNodeNameCanBeSaid(t *testing.T) { t.Errorf("--catalog parsed as %q", opts.Catalogue) } } + +// Defends novox/hq ADR 0100: the foundation's ports are inputs to genesis, and the registry's port +// and the address nodes pull from say one port. +func TestTheFoundationsPortsAreGiven(t *testing.T) { + _, opts, _, err := parseArgs([]string{"--store-port", "5433", "--hub-port", "51821", "--overlay-range", "10.77.0.0/16"}) + if err != nil { + t.Fatal(err) + } + if opts.Ports.Store != 5433 || opts.Ports.Hub != 51821 || opts.Ports.Bus != 5671 || opts.OverlayRange != "10.77.0.0/16" { + t.Errorf("ports read as %+v, range %s", opts.Ports, opts.OverlayRange) + } +} + +func TestTheRegistrysPortAndAddressAgree(t *testing.T) { + _, opts, _, err := parseArgs([]string{"--registry-port", "5100"}) + if err != nil || opts.Registry != "127.0.0.1:5100" { + t.Errorf("--registry-port alone: %s %v", opts.Registry, err) + } + _, opts, _, err = parseArgs([]string{"--registry", "192.0.2.10:5100"}) + if err != nil || opts.Ports.Registry != 5100 { + t.Errorf("--registry alone: %d %v", opts.Ports.Registry, err) + } + if _, _, _, err := parseArgs([]string{"--registry", "192.0.2.10:5000", "--registry-port", "5100"}); err == nil { + t.Error("two ports for one registry were accepted") + } +} + +func TestARegistryGivenAsAHostAloneTakesTheRegistrysPort(t *testing.T) { + _, opts, _, err := parseArgs([]string{"--registry", "192.0.2.10"}) + if err != nil || opts.Registry != "192.0.2.10:5000" || opts.Ports.Registry != 5000 { + t.Errorf("--registry host alone: %s %d %v", opts.Registry, opts.Ports.Registry, err) + } + _, opts, _, err = parseArgs([]string{"--registry", "192.0.2.10", "--registry-port", "5100"}) + if err != nil || opts.Registry != "192.0.2.10:5100" { + t.Errorf("--registry host with --registry-port: %s %v", opts.Registry, err) + } + if _, _, _, err := parseArgs([]string{"--registry", "192.0.2.10:notaport"}); err == nil { + t.Error("a registry with a port that is not a number was accepted") + } +} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index f4822b0..681eb3c 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -17,8 +17,10 @@ import ( "fmt" "os" "os/signal" + "path/filepath" "sort" "strings" + "sync" "syscall" "text/tabwriter" "time" @@ -26,10 +28,12 @@ import ( "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/bundle" "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/firewall" "github.com/novox/mesh-host/internal/identity" "github.com/novox/mesh-host/internal/inventory" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/profile" + "github.com/novox/mesh-host/internal/reachable" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" "github.com/novox/mesh-host/internal/upgrade" @@ -436,6 +440,23 @@ func enrol(ctx context.Context, opts options) error { return err } + // An adopted node keeps the firewall it was found with (novox/hq ADR 0100), so a host that + // cannot speak that firewall must say so now — before the mesh records a node it could never + // open anything on. + if token.Adopted { + kind, name, err := firewall.Detect(ctx, apply.ExecRunner) + if err != nil { + return err + } + if kind == firewall.Unsupported { + return fmt.Errorf( + "this token joins this machine adopted, keeping the firewall found on it, and it is "+ + "filtered by %s, which no host speaks yet. Nothing was enrolled", name) + } + fmt.Printf("joining adopted: what is on this machine is kept, and its firewall (%s) stays in force\n", + string(kind)) + } + fmt.Printf("token for broker %s\n", token.Broker) fmt.Printf(" pinned certificate %s\n", token.Fingerprint) fmt.Printf(" signing key %s\n", @@ -615,7 +636,28 @@ func runLink(ctx context.Context, opts options) error { // new declarations; this holds the machine in the last one whether the link is up or not. A // laptop shut for a week comes back and reconciles — it does not come back and ask what it is // (novox/hq ADR 0004). - go holdTheMachine(ctx, opts, mine, say, sched) + // Reports a reconcile has to make unasked — what an adopted node holds changed, or its + // firewall did — go out over the link when it is up (novox/hq ADR 0100). + outbox := make(chan link.Unasked, 1) + watch := &adoptionWatch{} + applier = watch.noting(applier) + go holdTheMachine(ctx, opts, mine, say, sched, func(r link.Report) { + if !watch.differs(r) { + return + } + select { + case <-outbox: + // An older one nobody has published yet; this one says everything it did. + default: + } + // Counted as said only once the broker has taken it: queued and lost — the link down, the + // publish refused — the change would never be said again (novox/hq ADR 0100). + outbox <- link.Unasked{Report: r, Done: func(published bool) { + if published { + watch.said(r) + } + }} + }) return link.HoldRoused(ctx, link.Membership{ Node: mine.Node, @@ -623,7 +665,64 @@ func runLink(ctx context.Context, opts options) error { Fingerprint: mine.Membership.Fingerprint, Password: mine.Membership.Password, Signer: mine.Membership.Signer, - }, applier, say, opts.timeout, rousedBySignal(ctx)) + }, applier, say, opts.timeout, rousedBySignal(ctx), outbox) +} + +// adoptionWatch remembers what the node last said about what it holds and its firewall, so a +// reconcile speaks unasked only when that changed. +type adoptionWatch struct { + mu sync.Mutex + last string +} + +// fingerprint is what a report says about adoption: each hold and whether it changed, the +// firewall, and what is reachable on the machine — which only an adopted node reports, and which +// is what the controller previews a flip from, so a port that opens or closes between deliveries +// must reach it too (novox/hq ADR 0100). +func adoptionFingerprint(r link.Report) string { + parts := []string{"firewall=" + r.Firewall} + for _, h := range r.Held { + parts = append(parts, "held "+h.ID+"="+h.Changed) + } + for _, reach := range r.Reachable { + parts = append(parts, fmt.Sprintf("reach %s %s:%d %s %v %d", reach.Protocol, reach.Address, + reach.Port, reach.By, reach.Published, reach.ContainerPort)) + } + sort.Strings(parts[1:]) + return strings.Join(parts, "\n") +} + +// differs says whether a report says anything the last one that went out did not. It records +// nothing: what was said is what reached the mesh, not what was written down to send. +func (w *adoptionWatch) differs(r link.Report) bool { + w.mu.Lock() + defer w.mu.Unlock() + return adoptionFingerprint(r) != w.last +} + +// said records a report the mesh has actually been told. +func (w *adoptionWatch) said(r link.Report) { + w.mu.Lock() + defer w.mu.Unlock() + w.last = adoptionFingerprint(r) +} + +// changed is differs and said together, for a report published as it is made. +func (w *adoptionWatch) changed(r link.Report) bool { + if !w.differs(r) { + return false + } + w.said(r) + return true +} + +// noting wraps the applier, so a report the link publishes after a delivery counts as said. +func (w *adoptionWatch) noting(apply link.Applier) link.Applier { + return func(ctx context.Context, raw, signature []byte) link.Report { + r := apply(ctx, raw, signature) + w.changed(r) + return r + } } // rousedBySignal is the machine telling this process that its link is probably stale. @@ -671,7 +770,7 @@ func rousedBySignal(ctx context.Context) link.Roused { const ReconcileEvery = 5 * time.Minute func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, say link.Announce, - sched *apply.Scheduler) { + sched *apply.Scheduler, publish func(link.Report)) { ticker := time.NewTicker(ReconcileEvery) defer ticker.Stop() @@ -694,6 +793,12 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s } report := applyDeclared(ctx, opts, declared, sched) + // A reconcile is otherwise silent. On an adopted node it speaks when what it holds or + // its firewall changed, because that is how a predecessor still writing is caught + // (novox/hq ADR 0100); publish decides whether anything did. + if publish != nil && report.Refused == "" && (len(report.Held) > 0 || report.Firewall != "") { + publish(report) + } switch { case report.Refused != "": say("what this node was last told no longer applies: " + report.Refused) @@ -712,10 +817,22 @@ func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.S return applyAndKeep(ctx, opts, raw, nil, sched) } +// applying serialises applies on this node. +// +// **Two things apply here: the link and the reconcile loop**, and each reads the node's state, +// acts on the machine, and writes the state back. Run at the same time they interleave, and the +// one that saves last writes a state read before the other acted — losing what the first recorded: +// a hold, the firewall found here, a resource just applied. The machine would then be one thing +// and its record another, which is the fault every read-back in this package exists to prevent. +var applying sync.Mutex + // applyAndKeep applies a declaration and, when it came from the mesh, keeps it so this node can -// go on obeying it while disconnected. +// go on obeying it while disconnected. One at a time, whoever asks. func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared, sched *apply.Scheduler) link.Report { + applying.Lock() + defer applying.Unlock() + declared, err := declaration.Parse(raw) if err != nil { return link.Report{Refused: err.Error()} @@ -740,8 +857,8 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D // Declared, not carried. A declaration from the mesh removes only what the mesh previously // declared — never what this machine raised for itself from its bundle (04-ISSUES/010). - outcome, updated, applyErr := apply.Apply(ctx, built, declared, known, store.OriginDeclared, - apply.ExecRunner, nil, sealOpener(opts.state)) + outcome, updated, applyErr := apply.ApplyKeeping(ctx, built, declared, known, store.OriginDeclared, + apply.ExecRunner, nil, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state))) // Saved whichever way it went. Recording only on success would lose the footprint of a // failed apply, and that footprint is on the machine either way. @@ -756,11 +873,36 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D // declared is forgotten — and after a host restart the first apply rebuilds them all. A nil // scheduler is the one-shot CLI path, which exits rather than staying up to fire anything. if sched != nil { - sched.Sync(declared) + held := map[string]bool{} + for _, h := range updated.Held { + held[h.ID] = true + } + sched.Sync(declared, held) } report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw)} + // What this node found and holds, its firewall, and what is reachable on it — so an adopted + // node never reads as converged (novox/hq ADR 0100). + for _, h := range updated.Held { + report.Held = append(report.Held, link.Held{ID: h.ID, Module: h.Module, Kind: h.Kind, + Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept}) + } + if declared.Adoption != nil { + if updated.Firewall != nil { + report.Firewall = updated.Firewall.Kind + } + reached, err := reachable.Collect(ctx, apply.ExecRunner) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read what is reachable here: %v\n", err) + } + report.Reachable = reached + } for _, change := range outcome.Outcomes { + // What is held is not what this machine owns: it was found, and is kept as it was until + // its module is taken (novox/hq ADR 0100). + if change.Action == "held" { + continue + } report.Applied = append(report.Applied, change.ID) } // Kept whichever way it went, so a node that is disconnected next minute still knows what it diff --git a/cmd/mesh-host/main_test.go b/cmd/mesh-host/main_test.go index 562c36d..4caab47 100644 --- a/cmd/mesh-host/main_test.go +++ b/cmd/mesh-host/main_test.go @@ -1,9 +1,17 @@ package main import ( - "github.com/novox/mesh-host/internal/store" + "context" + "errors" + "os" + "path/filepath" "testing" "time" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" ) // Argument handling gets tests because it already failed silently once: `mesh-host inventory @@ -135,3 +143,122 @@ func TestAFlagAfterAPositionalIsRead(t *testing.T) { } } } + +// Defends novox/hq ADR 0100: a reconcile on an adopted node speaks unasked only when what it holds +// or its firewall changed — which is how a predecessor still writing is caught, without a report +// every five minutes saying nothing new. +func TestAReconcileSpeaksOnlyWhenWhatIsHeldChanged(t *testing.T) { + w := &adoptionWatch{} + held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page"}, {ID: "hello-web.server"}}} + if !w.changed(held) { + t.Fatal("the first report of a hold was not said") + } + again := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.server"}, {ID: "hello-web.page"}}} + if w.changed(again) { + t.Error("the same holds in another order were said again") + } + rewritten := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}, {ID: "hello-web.server"}}} + if !w.changed(rewritten) { + t.Error("a held file rewritten by something else was not said") + } + if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) { + t.Error("a changed firewall was not said") + } +} + +func TestWhatTheLinkPublishedCountsAsSaid(t *testing.T) { + w := &adoptionWatch{} + report := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "a"}}} + applier := w.noting(func(context.Context, []byte, []byte) link.Report { return report }) + applier(context.Background(), nil, nil) + if w.changed(report) { + t.Error("a reconcile repeated what the link had just published") + } +} + +func TestAReconcileSpeaksWhenWhatIsReachableChanged(t *testing.T) { + // The controller previews a flip from what the node last said is reachable; a port that opened + // since must reach it without waiting for the next delivery (novox/hq ADR 0100). + w := &adoptionWatch{} + before := link.Report{Firewall: "ufw", Reachable: []link.Reach{ + {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}} + if !w.changed(before) { + t.Fatal("the first report was not said") + } + reordered := link.Report{Firewall: "ufw", Reachable: []link.Reach{ + {Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}} + if w.changed(reordered) { + t.Error("the same reachable set was said again") + } + opened := link.Report{Firewall: "ufw", Reachable: append(before.Reachable, + link.Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, ContainerPort: 80})} + if !w.changed(opened) { + t.Error("a newly published port was not said") + } +} + +// Defends the node's own record: the link and the reconcile loop both apply, and each reads the +// state, acts, and writes it back — so they must not run at the same time, or the last save loses +// what the other recorded. +func TestOnlyOneApplyRunsAtATime(t *testing.T) { + // A host is built for one system at link time, and a test binary has no link time: this asks + // the machine it runs on, and stands aside where the answer is no. + built, err := system.For("arch") + if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil { + t.Skip("this machine is not one these tests can apply on") + } + was := builtFor + builtFor = "arch" + t.Cleanup(func() { builtFor = was }) + dir := t.TempDir() + opts := options{state: filepath.Join(dir, "state.json")} + raw := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` + + filepath.Join(dir, "a.conf") + `","content":"x\n"}]}`) + + // Whatever else is applying — the link, while this is the reconcile — this waits for it. + applying.Lock() + done := make(chan link.Report, 1) + go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil) }() + select { + case report := <-done: + applying.Unlock() + t.Fatalf("an apply ran while another held the node: %+v", report) + case <-time.After(50 * time.Millisecond): + } + if _, err := os.Stat(filepath.Join(dir, "a.conf")); !errors.Is(err, os.ErrNotExist) { + applying.Unlock() + t.Fatal("the waiting apply had already touched the machine") + } + applying.Unlock() + + select { + case report := <-done: + if report.Refused != "" { + t.Fatalf("refused: %s", report.Refused) + } + case <-time.After(10 * time.Second): + t.Fatal("the apply never ran once the node was free") + } + known, loadErr := store.Load(opts.state) + if loadErr != nil || len(known.Resources) != 1 { + t.Errorf("the apply recorded %d resource(s): %v", len(known.Resources), loadErr) + } +} + +// Defends novox/hq ADR 0100: a change is counted as said only once the mesh has been told. Queued +// and lost — the link down when the reconcile spoke — it must be said again. +func TestAChangeThatNeverReachedTheMeshIsSaidAgain(t *testing.T) { + w := &adoptionWatch{} + held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}}} + if !w.differs(held) { + t.Fatal("the first report of a change was not new") + } + // The link was down: nothing published it, so nothing says it was said. + if !w.differs(held) { + t.Error("a change that never reached the mesh was counted as said") + } + w.said(held) + if w.differs(held) { + t.Error("a change the mesh was told was said again") + } +} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 893aa17..d473f3f 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -50,6 +50,8 @@ type Outcome struct { // wrote is a digest of what this apply put there, kept so the next one can tell a machine // that drifted from one the mesh changed its mind about. Not reported: it is bookkeeping. wrote string + // into is what a file written into held before the mesh's keys (novox/hq ADR 0102). + into *store.Into } // Report is what an apply did, in the order it did it. @@ -61,7 +63,8 @@ type Report struct { // nothing is the ordinary steady state, and saying so is not the same as saying it failed. func (r Report) Changed() bool { for _, o := range r.Outcomes { - if o.Action != "unchanged" { + // Holding is keeping the machine as it was found, which is not moving it. + if o.Action != "unchanged" && o.Action != "held" { return true } } @@ -111,7 +114,9 @@ func (e *Error) Unwrap() error { return e.Err } // Removal happens FIRST, and the order is not arbitrary. A resource that leaves a declaration // while another arrives at the same path is an ordinary rename: removing afterwards would // delete the file that had just been written. Removing first risks losing the old state if the -// apply then fails — a recovery concern, where the other is a correctness one. +// apply then fails — a recovery concern, where the other is a correctness one. The one exception +// is what protects an adopted node, the openings and the guard: that goes last, and only when +// everything else applied (novox/hq ADR 0103). func Apply( ctx context.Context, sys system.System, @@ -121,6 +126,23 @@ func Apply( run Runner, log func(string), unseal Unseal, +) (Report, store.State, error) { + return ApplyKeeping(ctx, sys, d, known, origin, run, log, unseal, nil) +} + +// ApplyKeeping is Apply on a node that may be adopted: keep is where the original of a file found +// there is recorded before anything else happens to it (novox/hq ADR 0100). Nil is a caller that +// can never be handed an adopted declaration — the carried bundle, which may not say it. +func ApplyKeeping( + ctx context.Context, + sys system.System, + d *declaration.Declaration, + known store.State, + origin string, + run Runner, + log func(string), + unseal Unseal, + keep Keep, ) (Report, store.State, error) { if log == nil { log = func(string) {} @@ -132,10 +154,27 @@ func Apply( declared[r.Identity()] = true } - for _, orphan := range known.Orphans(declared, origin) { - action, detail, err := remove(ctx, sys, orphan, run) + // Which firewall is found here, before anything else, since an unsupported one refuses the + // whole declaration (novox/hq ADR 0100). Nothing for a converged node. + fw, err := foundFirewall(ctx, d, &known, run, log) + if err != nil { + return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report} + } + + // What an adopted node's untaken modules find on the machine, looked at before anything in + // this apply — a removal included — could change it or its records (novox/hq ADR 0103). + before := lookBefore(ctx, sys, d, known, run) + + removeOrphan := func(orphan store.Applied) error { + var action, detail string + var err error + if declaration.Type(orphan.Type) == declaration.TypeOpening { + action, detail, err = removeOpening(ctx, orphan, run, known.Firewall) + } else { + action, detail, err = remove(ctx, sys, orphan, run) + } if err != nil { - return report, known, &Error{Resource: orphan.ID, Err: err, Done: report} + return &Error{Resource: orphan.ID, Err: err, Done: report} } known.Forget(orphan.ID) report.Outcomes = append(report.Outcomes, Outcome{ @@ -143,6 +182,74 @@ func Apply( Action: action, Detail: detail, }) log(fmt.Sprintf(" %s %s (%s)", action, orphan.ID, orphan.Target)) + return nil + } + + // **What protects an adopted node goes last on the flip, and first on the way back** (novox/hq + // ADR 0103). The openings and the guard are what keep the mesh reachable through the found + // firewall and the store unreachable from outside. + // + // When a node is converged they leave the declaration, and removing them first would leave the + // store open from the moment the guard stops until the derived filter loads — and for ever, if + // the filter then fails. So on a converged declaration they are removed only once everything + // else applied and the found firewall is retired; if anything failed, they stay, recorded, for + // the next try. + // + // Returned to adopted, it is the mirror image: removing the derived filter first would leave + // the store open until the guard loads. So the guard's own resources are applied before any + // orphan is removed, and if a removal then fails the guard is already up. A stale opening on an + // adopted node is removed as any orphan is. + var protecting, orphans []store.Applied + for _, orphan := range known.Orphans(declared, origin) { + if d.Adoption == nil && strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) { + protecting = append(protecting, orphan) + continue + } + orphans = append(orphans, orphan) + } + ordered := d.Resources + guardFirst := 0 + if d.Adoption != nil { + ordered = nil + for _, r := range d.Resources { + if strings.HasPrefix(r.Identity(), guardPrefix) { + ordered = append(ordered, r) + } + } + guardFirst = len(ordered) + for _, r := range d.Resources { + if !strings.HasPrefix(r.Identity(), guardPrefix) { + ordered = append(ordered, r) + } + } + } + orphansRemoved := false + removeOrphans := func() error { + orphansRemoved = true + for _, orphan := range orphans { + if err := removeOrphan(orphan); err != nil { + return err + } + } + return nil + } + + // **A hold whose resource is no longer declared is let go, and nothing on disk is touched.** + // What was found stays as it was; only the host's note that it holds it for a module goes, so + // the node stops reporting a hold for a module no longer assigned. Should the module come back, + // what is there is present with no record and is found, and held, again — its first kept + // original is never overwritten (novox/hq ADR 0100). Only a declaration from the mesh says + // what is assigned: a carried bundle's silence is not an unassignment. + if origin == store.OriginDeclared { + for _, h := range append([]store.Held{}, known.Held...) { + if declared[h.ID] { + continue + } + known.Release(h.ID) + report.Outcomes = append(report.Outcomes, Outcome{ID: h.ID, Type: h.Kind, Target: h.Target, + Action: "forgotten", Detail: "no longer declared; left as found"}) + log(fmt.Sprintf(" forgotten %s (%s): no longer declared; left as found", h.ID, h.Target)) + } } // What moved in this apply, so a service that must reflect a file can be told the file @@ -181,9 +288,60 @@ func Apply( // is a different thing — one is "this machine could not do it", the other is "this was never // a declaration", and they are fixed in different places. var failures []*Error - for _, resource := range d.Resources { + for i, resource := range ordered { + if !orphansRemoved && i == guardFirst { + // **Only a guard that is up may let the filter go.** Removing the derived filter's + // resources stops its unit, whose stop deletes the mesh's table; if a guard resource + // failed, doing that would leave the node with neither, and the store open until some + // later reconcile gets the guard up (novox/hq ADR 0103). + if len(failures) > 0 { + first := failures[0] + first.Done = report + first.Others = len(failures) - 1 + log(" kept " + guardPrefix + "*: the guard is not up, so what it replaces was left in force") + return report, known, first + } + if err := removeOrphans(); err != nil { + return report, known, err + } + } + // **On an adopted node, what is found is kept until its module is taken** (novox/hq ADR + // 0100, ADR 0103). Before anything is applied: whatever of a module not yet taken is + // present with no record of this host making it — or would reach what is — is held as it + // is and reported. Once held it stays held until its module is taken, and it is never + // recorded as applied, so it is never removed as an orphan either. + if d.Adoption != nil { + isHeld, news, outcome, err := holdOnAdopted(ctx, sys, resource, d, &known, before, run, keep, + changed, time.Now().UTC()) + if err != nil { + failures = append(failures, &Error{Resource: resource.Identity(), Err: err, Done: report}) + log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), resource.Target(), err)) + continue + } + if isHeld { + report.Outcomes = append(report.Outcomes, outcome) + if news { + log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) + } + continue + } + } + was, _ := known.Find(resource.Identity()) - outcome, err := applyOne(ctx, sys, resource, run, changed, declares, was, unseal) + var outcome Outcome + var err error + if o, isOpening := resource.(*declaration.Opening); isOpening { + outcome, err = applyOpening(ctx, o, run, fw) + } else { + // A file this host has no record of, under any id, is the machine's until the mesh + // writes over it — on any node, adopted or not: its original is kept first. + var keepFound Keep + if f, isFile := resource.(*declaration.File); isFile && was.ID == "" && + !known.Recorded(string(declaration.TypeFile), f.Path) { + keepFound = keep + } + outcome, err = applyOne(ctx, sys, resource, run, changed, declares, was, unseal, keepFound) + } if err != nil { failed := &Error{Resource: resource.Identity(), Err: err, Done: report} failures = append(failures, failed) @@ -227,8 +385,14 @@ func Apply( ID: resource.Identity(), Type: string(resource.Kind()), Target: outcome.Target, AppliedAt: time.Now().UTC(), Wrote: outcome.wrote, + Into: outcome.into, Holds: holds(resource), }) + // Its module has been taken, and what was held for it is now the mesh's. + if held, wasHeld := known.HeldAt(resource.Identity()); wasHeld { + known.Release(held.ID) + outcome.Detail = takenDetail(held) + } report.Outcomes = append(report.Outcomes, outcome) if outcome.Action != "unchanged" { changed[resource.Identity()] = true @@ -236,6 +400,25 @@ func Apply( } } + if !orphansRemoved { + if err := removeOrphans(); err != nil { + return report, known, err + } + } + + // A converged node whose found firewall was in force retires it only now, once everything — + // the mesh's derived filter among it — applied cleanly (novox/hq ADR 0100). + if len(failures) == 0 { + if err := retireFirewall(ctx, d, origin, &known, run, log); err != nil { + return report, known, &Error{Resource: "the firewall found on this machine", Err: err, Done: report} + } + for _, orphan := range protecting { + if err := removeOrphan(orphan); err != nil { + return report, known, err + } + } + } + if len(failures) > 0 { // The first, carrying everything that did happen. One error is what the caller reports // and what a person reads first; the rest are in the report, which is what the mesh @@ -248,18 +431,22 @@ func Apply( return report, known, nil } +// guardPrefix is the ids of the mesh's guard on an adopted node: its package, table, unit and +// service (novox/hq ADR 0100). +const guardPrefix = declaration.AdoptionPrefix + "guard" + // Unseal opens a value the mesh sealed to this node. Nil when the node has no sealing key, which // makes every sealed file an error rather than a silently skipped one. type Unseal func(sealed string) ([]byte, error) func applyOne(ctx context.Context, sys system.System, r declaration.Resource, run Runner, changed map[string]bool, declares map[string]string, previous store.Applied, - unseal Unseal) (Outcome, error) { + unseal Unseal, keepFound Keep) (Outcome, error) { switch res := r.(type) { case *declaration.Directory: return applyDirectory(res) case *declaration.File: - return applyFile(res, previous, unseal) + return applyFile(res, previous, unseal, keepFound) case *declaration.Service: return applyService(ctx, sys, res, run, changed) case *declaration.Package: @@ -404,7 +591,12 @@ func applyAccess(r *declaration.Access) (Outcome, error) { return out, nil } -func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outcome, error) { +// keepFound, when not nil, is where the original of a file this host has no record of is kept +// before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome. +func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) { + if r.Into != "" { + return applyInto(r, previous) + } out := begin(r) // What actually goes on disk. For a sealed file the mesh never had this, and neither did @@ -498,7 +690,15 @@ func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outc drifted := existed && previous.Wrote != "" && digestOf(string(existing)) != previous.Wrote modeSame := existed && beforeMode == mode.Perm() + kept := "" if !contentSame { + if existed && keepFound != nil { + // Before anything is written: a keep that fails stops the write, since the + // original could not be had back otherwise. + if kept, err = keepFound(r.Path, existing, beforeMode); err != nil { + return out, fmt.Errorf("keeping the original of %s before writing over it: %w", r.Path, err) + } + } if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil { return out, err } @@ -562,6 +762,12 @@ func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outc default: out.Action = "unchanged" } + if kept != "" { + if out.Detail != "" { + out.Detail += "; " + } + out.Detail += "the file found here, which the mesh had no record of, was kept at " + kept + } return out, nil } @@ -604,11 +810,32 @@ func reflected(r *declaration.Service, changed map[string]bool) []string { return restartedBy(r.RestartOn, changed) } +// serviceReloader is a service manager that can tell a running unit to read its configuration again. +type serviceReloader interface { + ReloadService(ctx context.Context, run system.Runner, unit string) error +} + +// unitReloader is a service manager that caches unit files and must be told to read them again. +type unitReloader interface { + ReloadUnits(ctx context.Context, run system.Runner) error +} + func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner, changed map[string]bool) (Outcome, error) { out := begin(r) var changes []string + // A file the service reflects changed, and it may be the unit's own file or a drop-in: the + // service manager reads those again only when told to, and a restart without it runs the unit + // it had already loaded. + if reflects(r, changed) { + if u, ok := sys.(unitReloader); ok { + if err := u.ReloadUnits(ctx, run); err != nil { + return out, fmt.Errorf("reloading the service manager's units for %s: %w", r.Unit, err) + } + } + } + // Boot first. A unit asked to be running and enabled should survive this apply failing // half way in the more useful direction: enabled-and-stopped comes back at the next boot, // where running-and-disabled does not. @@ -674,6 +901,25 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service "%s was restarted to pick up a change and is %s", r.Unit, after) } changes = append(changes, "restarted for "+strings.Join(reflected(r, changed), ", ")) + } else if r.State == "running" && len(restartedBy(r.ReloadOn, changed)) > 0 { + // Told to read its configuration again, not stopped: for a service whose restart would + // stop what it runs — every container, for the container runtime (novox/hq ADR 0102). + reloader, ok := sys.(serviceReloader) + if !ok { + return out, fmt.Errorf("%s must be reloaded for %s and this machine's service manager "+ + "cannot reload a unit", r.Unit, strings.Join(restartedBy(r.ReloadOn, changed), ", ")) + } + if err := reloader.ReloadService(ctx, run, r.Unit); err != nil { + return out, fmt.Errorf("reloading %s: %w", r.Unit, err) + } + after, err := sys.ServiceState(ctx, run, r.Unit) + if err != nil { + return out, err + } + if after != "running" { + return out, fmt.Errorf("%s was reloaded to pick up a change and is %s", r.Unit, after) + } + changes = append(changes, "reloaded for "+strings.Join(restartedBy(r.ReloadOn, changed), ", ")) } if len(changes) == 0 { @@ -728,6 +974,9 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner) return "removed", "no longer declared, and empty", nil case declaration.TypeFile: + if a.Into != nil { + return removeInto(a) + } if err := os.RemoveAll(a.Target); err != nil { return "", "", err } diff --git a/internal/apply/apply_test.go b/internal/apply/apply_test.go index 52a179c..ba5c5aa 100644 --- a/internal/apply/apply_test.go +++ b/internal/apply/apply_test.go @@ -1114,6 +1114,18 @@ func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) { if !stopped || !started { t.Errorf("the file changed and the service was not restarted; commands were %v", commands) } + reloaded, stop := -1, -1 + for i, c := range commands { + if strings.Contains(c, "daemon-reload") && reloaded < 0 { + reloaded = i + } + if strings.Contains(c, "stop thing.service") && stop < 0 { + stop = i + } + } + if reloaded < 0 || reloaded > stop { + t.Errorf("the service was restarted without the unit files being read again first; commands were %v", commands) + } } } diff --git a/internal/apply/hold.go b/internal/apply/hold.go new file mode 100644 index 0000000..3776c41 --- /dev/null +++ b/internal/apply/hold.go @@ -0,0 +1,637 @@ +package apply + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// Keep records the original of a file found on an adopted node, before anything else happens to +// it, and says where (novox/hq ADR 0100). It never overwrites an original it already kept: the +// first copy is the one that was there before the mesh. +type Keep func(path string, content []byte, mode os.FileMode) (string, error) + +// KeepIn keeps originals under dir/kept, each named for the path it came from AND for what was in +// it, readable by root alone — a predecessor's configuration may carry its credentials. +// +// **By content as well as path, because a path has more than one original.** A file held, let go +// when its module was unassigned, rewritten by the predecessor and found again is a second +// original; named by path alone the second copy was silently discarded while the report said it +// was kept (novox/hq ADR 0100). The same content at the same path is kept once. +func KeepIn(dir string) Keep { + return func(path string, content []byte, _ os.FileMode) (string, error) { + where := sha256.Sum256([]byte(path)) + what := sha256.Sum256(content) + kept := filepath.Join(dir, "kept", hex.EncodeToString(where[:])[:12]+"-"+ + hex.EncodeToString(what[:])[:12]+"-"+filepath.Base(path)) + if _, err := os.Lstat(kept); err == nil { + return kept, nil + } + if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil { + return "", err + } + if err := writeAtomically(kept, content, 0o600); err != nil { + return "", err + } + back, err := os.ReadFile(kept) + if err != nil || string(back) != string(content) { + return "", fmt.Errorf("kept the original of %s at %s and cannot read it back", path, kept) + } + return kept, nil + } +} + +// foundBefore is what an adopted apply finds on the machine before it changes anything: each +// directory, service unit and container mount source of an untaken module that is present with no +// record (novox/hq ADR 0103). Looked at first, because the apply itself makes such things — a +// file's parent directory, a unit file a module writes, a package that brings its unit — and what +// the mesh made in this apply was not found. +type foundBefore struct { + is map[string]bool + // trouble is what could not be asked about, by the same key, so the resource that would need + // the answer fails rather than proceeding as if the machine had nothing there. + trouble map[string]string +} + +func (f foundBefore) has(key string) bool { return f.is[key] } + +// why is the reason a key could not be settled, and empty when there was none. +func (f foundBefore) why(key string) string { return f.trouble[key] } + +func lookBefore(ctx context.Context, sys system.System, d *declaration.Declaration, known store.State, + run Runner) foundBefore { + seen := foundBefore{is: map[string]bool{}, trouble: map[string]string{}} + if d.Adoption == nil { + return seen + } + cri, asked := "", false + for _, r := range d.Resources { + if _, untaken := d.Adoption.UntakenModuleOf(r.Identity()); !untaken { + continue + } + if _, held := known.HeldAt(r.Identity()); held { + continue + } + switch res := r.(type) { + case *declaration.Directory: + if present(res.Path) && !recordedPath(known, res.Path) { + seen.is["path:"+res.Path] = true + } + case *declaration.Archive: + // Unpacking over it, and re-owning it recursively, would change the predecessor's + // files. + if present(res.Path) && !recordedPath(known, res.Path) { + seen.is["path:"+res.Path] = true + } + case *declaration.Process: + // Its unit would be written over and restarted. + if !known.Recorded(string(declaration.TypeProcess), res.Name) && + present(filepath.Join(unitDir, res.Name+".service")) { + seen.is["unit-file:"+res.Name] = true + } + case *declaration.User: + // Its shell and groups would be changed. + if !known.Recorded(string(declaration.TypeUser), res.Name) { + if _, exists, err := system.LookUpUser(ctx, run, res.Name); err == nil && exists { + seen.is["user:"+res.Name] = true + } + } + case *declaration.Service: + if known.Recorded(string(declaration.TypeService), res.Unit) { + continue + } + // **Found is a unit somebody put on this machine, or one the machine uses.** + // + // Where it comes from first: a unit the service manager loads from outside /usr — + // /etc/systemd/system or /run/systemd/system — was installed by an administrator, so + // it is a predecessor's whatever state it is in, and one deliberately stopped and + // disabled must stay that way (novox/hq ADR 0103). + // + // A unit a package ships, under /usr, is not held by its mere presence: the private + // network's own wg-quick@mesh0 is an instance of a template the tunnel package ships, + // nothing had ever run it, and holding it kept the private network from ever coming up + // (found by the adoption bed). Such a unit is held only if the machine actually uses + // it — running, or started at boot. + state, err := sys.ServiceState(ctx, run, res.Unit) + if err != nil { + continue + } + if from, ok := sys.(unitFiles); ok { + if path, err := from.ServiceUnitFile(ctx, run, res.Unit); err == nil && installedByHand(path) { + seen.is["unit:"+res.Unit] = true + continue + } + } + boot, _ := sys.ServiceBoot(ctx, run, res.Unit) + if state == "running" || boot == "enabled" { + seen.is["unit:"+res.Unit] = true + } + case *declaration.Container: + if known.Recorded(string(declaration.TypeContainer), res.Name) { + continue + } + for _, v := range res.Volumes { + src := mountSource(v) + switch { + case src == "": + case strings.HasPrefix(src, "/"): + if !systemPath(src) && present(src) && !recordedPath(known, src) { + seen.is["path:"+src] = true + } + default: + if !asked { + cri, _ = containerRuntime(ctx, run) + asked = true + } + if cri == "" { + continue + } + if _, err := run(ctx, cri, "volume", "inspect", src); err == nil { + seen.is["volume:"+src] = true + } else if !absent(err) { + seen.trouble["volume:"+src] = fmt.Sprintf( + "the container runtime could not say whether the volume %s is here: %v", src, err) + } + } + } + } + } + return seen +} + +// unitFiles is a service manager that can say where it loads a unit from. +type unitFiles interface { + ServiceUnitFile(ctx context.Context, run Runner, unit string) (string, error) +} + +// installedByHand is whether a unit file is one somebody put on this machine rather than one a +// package ships: anywhere but /usr, where distributions keep what they install. +func installedByHand(path string) bool { + if path == "" { + return false + } + return !strings.HasPrefix(filepath.Clean(path), "/usr/") +} + +func present(path string) bool { + _, err := os.Lstat(path) + return err == nil +} + +// recordedPath is whether this host has a record of MAKING something at a path — a directory it +// created, a file it wrote, an archive it unpacked. An access record is not one of those: it says +// the mesh set permissions on a path it does not own, which is exactly what it does to a path +// somebody else's software made, so a path it only has access for is still found (novox/hq ADR 0103). +func recordedPath(known store.State, path string) bool { + for _, kind := range []declaration.Type{declaration.TypeDirectory, declaration.TypeFile, + declaration.TypeArchive} { + if known.Recorded(string(kind), path) { + return true + } + } + return false +} + +// systemPath is whether a bind-mount source is the machine's own plumbing — the runtime's socket, +// the kernel's filesystems, the devices, the clock — which every machine has and no predecessor's +// data lives in. Mounting it shares nothing that was found. +func systemPath(src string) bool { + clean := filepath.Clean(src) + for _, exact := range []string{"/etc/localtime", "/etc/timezone", "/etc/hosts", "/etc/resolv.conf", + "/etc/machine-id", "/etc/passwd", "/etc/group"} { + if clean == exact { + return true + } + } + for _, under := range []string{"/run", "/var/run", "/sys", "/proc", "/dev", "/usr/share/zoneinfo", + "/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/lib/modules", "/usr/lib/modules"} { + if clean == under || strings.HasPrefix(clean, under+"/") { + return true + } + } + return false +} + +// mountSource is what a volume mapping mounts: a path on the machine, or a named volume. Empty for +// an anonymous volume, which mounts nothing that could already be there. +func mountSource(mapping string) string { + src, _, ok := strings.Cut(mapping, ":") + if !ok { + return "" + } + return src +} + +// runsIn is the container a resource runs inside, if any: an action's `in`, or a run-once step +// sharing a container's namespace. An action with no `in` runs on the machine itself and is not +// held for a container: it reaches nothing a predecessor holds by running there, and holding every +// action of an untaken module would stop a module preparing itself before its cutover. +func runsIn(r declaration.Resource) string { + switch res := r.(type) { + case *declaration.Action: + return res.In + case *declaration.Container: + if res.RunOnce { + if name, ok := strings.CutPrefix(res.Network, "container:"); ok { + return name + } + } + } + return "" +} + +// heldContainer is what is held under a container's name. +func heldContainer(known store.State, name string) (store.Held, bool) { + for _, h := range known.Held { + if h.Kind == string(declaration.TypeContainer) && h.Target == name { + return h, true + } + } + return store.Held{}, false +} + +// holdOnAdopted decides whether a resource of an adopted node is held rather than applied, and +// holds it (novox/hq ADR 0100, ADR 0103). For a module not yet taken, what is present with no +// record is kept as it is: a file or a container under its name, a directory, a service's unit, +// and a container that would mount a path or a volume found there. An action or a run-once step +// run inside a held container is held with it. Once held, a resource stays held — changed or gone +// — until its module is taken, and it is never recorded as applied, so never removed as an orphan. +// +// Held is false for a resource to apply as usual. News is whether the hold is new or changed, +// which is what is worth a line in the log. +func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resource, d *declaration.Declaration, + known *store.State, before foundBefore, run Runner, keep Keep, changed map[string]bool, + now time.Time) (held, news bool, out Outcome, err error) { + was, already := known.HeldAt(r.Identity()) + + if in := runsIn(r); in != "" { + if container, isHeld := heldContainer(*known, in); isHeld { + module, untaken := d.Adoption.UntakenModuleOf(r.Identity()) + if !untaken { + module = container.Module + } + h := was + if !already { + h = store.Held{ID: r.Identity(), Kind: string(r.Kind()), Target: r.Target(), Since: now} + } + h.Module, h.Why = module, "runs in "+in + known.RecordHeld(h) + out = begin(r) + out.Action = "held" + out.Detail = fmt.Sprintf("runs in %s, which is held as found; not run until %s is taken", in, module) + return true, !already, out, nil + } + } + + // A file written into replaces nothing that was found, so it is never held (novox/hq ADR + // 0102) — and a hold from when it was declared whole must not keep the mesh's keys out. + if f, ok := r.(*declaration.File); ok && f.Into != "" { + if already { + known.Release(r.Identity()) + } + return false, false, out, nil + } + + module, untaken := d.Adoption.UntakenModuleOf(r.Identity()) + if !untaken { + return false, false, out, nil + } + why := was.Why + isFound := already + if !already { + switch res := r.(type) { + case *declaration.File: + if res.Into == "" { + if isFound, err = found(ctx, r, run, *known); err != nil { + return false, false, begin(r), err + } + } + case *declaration.Container: + if known.Recorded(string(declaration.TypeContainer), res.Name) { + break + } + _, exists, err := inspectFound(ctx, res.Name, run) + if err != nil { + return false, false, begin(r), err + } + if exists { + if isFound, err = found(ctx, r, run, *known); err != nil { + return false, false, begin(r), err + } + break + } + // Not there under its name, and still it would share what was found: created, it + // would mount the predecessor's data beside the predecessor's own container. + for _, v := range res.Volumes { + src := mountSource(v) + key := "volume:" + src + if strings.HasPrefix(src, "/") { + key = "path:" + src + } + if src == "" { + continue + } + if trouble := before.why(key); trouble != "" { + return false, false, begin(r), fmt.Errorf( + "%s, so it is not safe to create a container that would mount it", trouble) + } + if before.has(key) { + isFound, why = true, "would mount "+src+", found on the machine" + break + } + } + case *declaration.Directory: + isFound = before.has("path:" + res.Path) + case *declaration.Archive: + isFound = before.has("path:" + res.Path) + case *declaration.Process: + isFound = before.has("unit-file:" + res.Name) + case *declaration.User: + isFound = before.has("user:" + res.Name) + case *declaration.Service: + isFound = before.has("unit:" + res.Unit) + } + } + if !isFound { + return false, false, out, nil + } + + out, h, err := hold(ctx, sys, r, module, was, already, why, run, keep, now) + if err != nil { + return true, false, out, err + } + // A held service is not started, stopped, enabled or restarted — but a reload stops nothing, + // so one the module names still happens (novox/hq ADR 0102, ADR 0103). + if svc, ok := r.(*declaration.Service); ok && svc.State == "running" { + if which := restartedBy(svc.ReloadOn, changed); len(which) > 0 { + if state, err := sys.ServiceState(ctx, run, svc.Unit); err == nil && state == "running" { + reloader, can := sys.(serviceReloader) + if !can { + return true, false, out, fmt.Errorf("%s must be reloaded for %s and this machine's "+ + "service manager cannot reload a unit", svc.Unit, strings.Join(which, ", ")) + } + if err := reloader.ReloadService(ctx, run, svc.Unit); err != nil { + return true, false, out, fmt.Errorf("reloading the held %s: %w", svc.Unit, err) + } + out.Detail += "; reloaded for " + strings.Join(which, ", ") + ", which stops nothing" + } + } + } + known.RecordHeld(h) + return true, !already || h.Changed != was.Changed, out, nil +} + +// found is whether a declared file or container is present on the machine with no record of this +// host making it (novox/hq ADR 0100). A container carrying the host's own spec label was made by +// a host, whatever this store says, so it is never found. +func found(ctx context.Context, r declaration.Resource, run Runner, known store.State) (bool, error) { + if known.Recorded(string(r.Kind()), r.Target()) { + return false, nil + } + switch res := r.(type) { + case *declaration.File: + _, err := os.Lstat(res.Path) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return err == nil, err + case *declaration.Container: + seen, exists, err := inspectFound(ctx, res.Name, run) + if err != nil || !exists { + return false, err + } + return seen.spec == "", nil + } + return false, nil +} + +type foundContainer struct { + id string + running bool + spec string +} + +// inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and +// whether a host made it. +func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, bool, error) { + cri, err := containerRuntime(ctx, run) + if err != nil { + return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name) + } + out, err := run(ctx, cri, "inspect", "--format", + "{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name) + if err != nil { + if absent(err) { + return foundContainer{}, false, nil + } + // **A runtime that could not answer is not a machine with nothing there.** Read as + // absence, a daemon that is down or a permission denied would let the mesh create its own + // container over a predecessor's — the one thing an adopted node must never do + // (novox/hq ADR 0100). + return foundContainer{}, false, fmt.Errorf( + "the container runtime could not say whether %s is here, so it is not safe to make one: %w", + name, err) + } + parts := strings.Split(strings.TrimSpace(out), "\t") + for len(parts) < 3 { + parts = append(parts, "") + } + spec := strings.TrimSpace(parts[2]) + if spec == "" { + spec = "" + } + return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil +} + +// absent is whether a runtime said the thing is not there, rather than failing to answer. Its own +// words: docker and podman both say "No such object", "No such container" or "No such volume". +func absent(err error) bool { + said := strings.ToLower(err.Error()) + for _, missing := range []string{"no such object", "no such container", "no such volume", + "no such image"} { + if strings.Contains(said, missing) { + return true + } + } + return false +} + +// hold keeps a found file or container as it is, and reports it — the first time by recording +// what was found, every time after by comparing against that. Nothing is reverted, restarted or +// created: a held target that disappears stays held and gone until its module is taken. +func hold(ctx context.Context, sys system.System, r declaration.Resource, module string, was store.Held, + already bool, why string, run Runner, keep Keep, now time.Time) (Outcome, store.Held, error) { + out := begin(r) + h := was + if !already { + h = store.Held{ID: r.Identity(), Module: module, Kind: string(r.Kind()), + Target: r.Target(), Since: now, Why: why} + } + h.Module = module + detail := "found on the machine; kept until " + module + " is taken" + + var changed string + switch res := r.(type) { + case *declaration.File: + info, err := os.Lstat(res.Path) + switch { + case errors.Is(err, os.ErrNotExist): + if !already { + return out, h, fmt.Errorf("%s was found and is gone before it could be kept", res.Path) + } + changed = "gone" + case err != nil: + return out, h, err + default: + content, err := os.ReadFile(res.Path) + if err != nil { + return out, h, fmt.Errorf("%s was found and cannot be read to keep it: %w", res.Path, err) + } + if !already { + // The original first, before anything is recorded: a hold with no kept copy + // would be a promise the host cannot keep. + if keep == nil { + return out, h, fmt.Errorf( + "%s was found on this adopted node and this host has nowhere to keep its original", res.Path) + } + kept, err := keep(res.Path, content, info.Mode().Perm()) + if err != nil { + return out, h, fmt.Errorf("keeping the original of %s: %w", res.Path, err) + } + h.Kept = kept + h.Digest = digestOf(string(content)) + h.Mode = fmt.Sprintf("%04o", info.Mode().Perm()) + if st, ok := info.Sys().(*syscall.Stat_t); ok { + h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid) + } + } else if digestOf(string(content)) != h.Digest { + changed = "rewritten" + } + } + case *declaration.Directory: + info, err := os.Lstat(res.Path) + switch { + case errors.Is(err, os.ErrNotExist): + if !already { + return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path) + } + changed = "gone" + case err != nil: + return out, h, err + case !already: + // Its mode and owner as found, which the mesh leaves: a database refuses to start + // on a data directory whose mode changed. + h.Mode = fmt.Sprintf("%04o", info.Mode().Perm()) + if st, ok := info.Sys().(*syscall.Stat_t); ok { + h.Owner = fmt.Sprintf("%d:%d", st.Uid, st.Gid) + } + } + detail = "found on the machine; its mode, owner and contents kept until " + module + " is taken" + case *declaration.Archive: + if _, err := os.Lstat(res.Path); errors.Is(err, os.ErrNotExist) { + if !already { + return out, h, fmt.Errorf("%s was found and is gone before it could be held", res.Path) + } + changed = "gone" + } else if err != nil { + return out, h, err + } + detail = "something is already at " + res.Path + "; nothing unpacked over it or re-owned until " + + module + " is taken" + case *declaration.Process: + unit := filepath.Join(unitDir, res.Name+".service") + if !present(unit) { + if !already { + return out, h, fmt.Errorf("%s was found and is gone before it could be held", unit) + } + changed = "gone" + } + detail = "its unit " + unit + " was found on the machine; not written over or restarted until " + + module + " is taken" + case *declaration.User: + _, exists, err := system.LookUpUser(ctx, run, res.Name) + switch { + case err != nil: + return out, h, err + case !exists && !already: + return out, h, fmt.Errorf("the user %s was found and is gone before it could be held", res.Name) + case !exists: + changed = "gone" + } + detail = "the user was found on the machine; its shell and groups are kept until " + module + " is taken" + case *declaration.Service: + state, err := sys.ServiceState(ctx, run, res.Unit) + switch { + case err != nil && !already: + return out, h, fmt.Errorf("the unit %s was found and cannot be read to hold it: %w", res.Unit, err) + case err != nil: + changed = "gone" + case !already: + h.Running = state == "running" + case h.Running && state != "running": + changed = "stopped" + } + detail = "its unit was found on the machine; its state and whether it starts at boot are " + + "kept until " + module + " is taken" + case *declaration.Container: + if h.Why != "" && h.Container == "" { + // Held for what it would mount, never created: there is nothing of it to compare. + detail = "not created: it " + h.Why + "; kept until " + module + " is taken" + break + } + seen, exists, err := inspectFound(ctx, res.Name, run) + if err != nil { + return out, h, err + } + switch { + case !exists && !already: + return out, h, fmt.Errorf("container %s was found and is gone before it could be held", res.Name) + case !already: + h.Container, h.Running = seen.id, seen.running + case !exists: + changed = "gone" + case seen.id != h.Container: + changed = "replaced" + case h.Running && !seen.running: + changed = "stopped" + } + default: + return out, h, fmt.Errorf("a %s cannot be held", r.Kind()) + } + + if changed != h.Changed { + h.Changed = changed + h.ChangedAt = now + if changed == "" { + h.ChangedAt = time.Time{} + } + } + out.Action = "held" + out.Detail = detail + if h.Changed != "" { + out.Detail += "; " + h.Changed + " by something other than the mesh since it was found, and not reverted" + } + return out, h, nil +} + +// takenDetail is what an outcome says when a module's cutover replaced what was held for it. +func takenDetail(h store.Held) string { + if h.Kind == string(declaration.TypeAction) || (h.Why != "" && h.Container == "") { + return "taken: no longer held (" + h.Why + ")" + } + if h.Kept != "" { + return "taken: replaced what was found; original kept at " + h.Kept + } + return "taken: replaced what was found" +} diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go new file mode 100644 index 0000000..0651f4f --- /dev/null +++ b/internal/apply/hold_test.go @@ -0,0 +1,1043 @@ +package apply + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0100: on an adopted node, what is found is kept until its module is taken. + +// machine is a fake container runtime holding containers by name: id, running, and the host's spec +// label when a host made it. Every command it is asked is written down. +type machine struct { + containers map[string]*fakeContainer + asked []string + + // units are service units by name, as systemd would report them; volumes are the runtime's + // named volumes. + units map[string]*fakeUnit + volumes map[string]bool + users map[string]bool +} + +type fakeUnit struct { + active, enabled string + // fragment is where systemd loads the unit from; empty means /etc/systemd/system, where an + // administrator installs one. + fragment string +} + +// systemctl answers as systemd does for the units the machine has, and "not-found" for any other. +func (m *machine) systemctl(args []string) (string, error) { + unit := args[len(args)-1] + if args[0] == "show" { + unit = args[1] + } + u, ok := m.units[unit] + switch args[0] { + case "show": + if !ok { + if len(args) > 2 && strings.Contains(args[2], "FragmentPath") { + return "FragmentPath=\n", nil + } + return "LoadState=not-found\nActiveState=inactive\nType=simple\n", nil + } + if len(args) > 2 && strings.Contains(args[2], "FragmentPath") { + from := u.fragment + if from == "" { + from = "/etc/systemd/system/" + unit + } + return "FragmentPath=" + from + "\n", nil + } + return "LoadState=loaded\nActiveState=" + u.active + "\nType=simple\nRemainAfterExit=no\n", nil + case "is-enabled": + if !ok { + return "", errors.New("not found") + } + return u.enabled + "\n", nil + case "start": + u.active = "active" + case "stop": + u.active = "inactive" + case "enable": + u.enabled = "enabled" + case "disable": + u.enabled = "disabled" + } + return "", nil +} + +func (m *machine) did(prefix string) bool { + for _, a := range m.asked { + if strings.HasPrefix(a, prefix) { + return true + } + } + return false +} + +type fakeContainer struct { + id string + running bool + spec string +} + +func (m *machine) run(_ context.Context, name string, args ...string) (string, error) { + m.asked = append(m.asked, name+" "+strings.Join(args, " ")) + if name == "systemctl" { + return m.systemctl(args) + } + if name == "getent" { + if m.users[args[len(args)-1]] { + return args[len(args)-1] + ":x:1500:1500::/home/" + args[len(args)-1] + ":/bin/bash\n", nil + } + return "", errors.New("exit status 2") + } + if name != "docker" { + return "", nil + } + switch args[0] { + case "volume": + if m.volumes[args[len(args)-1]] { + return "[]\n", nil + } + return "", errors.New("no such volume") + case "info": + return "27.0\n", nil + case "inspect": + c, ok := m.containers[args[len(args)-1]] + if !ok { + return "", errors.New("no such container") + } + running := "false" + if c.running { + running = "true" + } + if strings.HasPrefix(args[2], "{{.Id}}") { + return c.id + "\t" + running + "\t" + c.spec + "\n", nil + } + return running + "\t" + c.spec + "\n", nil + case "rm": + delete(m.containers, args[len(args)-1]) + return "", nil + case "run": + var name, spec string + for i, a := range args { + if a == "--name" { + name = args[i+1] + } + if a == "--label" && strings.HasPrefix(args[i+1], specLabel+"=") { + spec = strings.TrimPrefix(args[i+1], specLabel+"=") + } + } + m.containers[name] = &fakeContainer{id: "made-by-host", running: true, spec: spec} + return "made-by-host\n", nil + } + return "", nil +} + +func (m *machine) removed(name string) bool { + for _, a := range m.asked { + if strings.HasPrefix(a, "docker rm") && strings.HasSuffix(a, " "+name) { + return true + } + } + return false +} + +func adopted(t *testing.T, adoption, resources string) *declaration.Declaration { + t.Helper() + return parse(t, `{"declaration":1,"adoption":`+adoption+`,"resources":[`+resources+`]}`) +} + +const untakenWeb = `{"taken":[],"untaken":{"hello-web":["hello-web.page","hello-web.server"]}}` +const takenWeb = `{"taken":["hello-web"]}` + +func webResources(page string) string { + return `{"id":"hello-web.page","type":"file","path":"` + page + `","content":"the mesh's page\n"}, + {"id":"hello-web.server","type":"container","name":"hello-web","image":"` + pinned + `"}` +} + +func applyAdopted(t *testing.T, d *declaration.Declaration, known store.State, m *machine, keepDir string) (Report, store.State) { + t.Helper() + report, state, err := ApplyKeeping(context.Background(), archHost(t), d, known, + store.OriginDeclared, m.run, nil, nil, KeepIn(keepDir)) + if err != nil { + t.Fatalf("apply failed: %v", err) + } + return report, state +} + +func outcomeOf(r Report, id string) Outcome { + for _, o := range r.Outcomes { + if o.ID == id { + return o + } + } + return Outcome{} +} + +func predecessor(t *testing.T) (dir, page string, m *machine) { + t.Helper() + dir = t.TempDir() + page = filepath.Join(dir, "index.html") + if err := os.WriteFile(page, []byte("the predecessor's page\n"), 0o640); err != nil { + t.Fatal(err) + } + return dir, page, &machine{containers: map[string]*fakeContainer{ + "hello-web": {id: "predecessor-id", running: true}, + }} +} + +func TestAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) { + dir, page, m := predecessor(t) + report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + + got, _ := os.ReadFile(page) + if string(got) != "the predecessor's page\n" { + t.Fatalf("a found file was changed: %q", got) + } + info, _ := os.Stat(page) + if info.Mode().Perm() != 0o640 { + t.Errorf("a found file's mode was changed to %o", info.Mode().Perm()) + } + if o := outcomeOf(report, "hello-web.page"); o.Action != "held" || + !strings.Contains(o.Detail, "kept until hello-web is taken") { + t.Errorf("the found file was not reported held: %+v", o) + } + h, ok := state.HeldAt("hello-web.page") + if !ok || h.Module != "hello-web" || h.Mode != "0640" { + t.Fatalf("the hold was not recorded: %+v", h) + } + kept, err := os.ReadFile(h.Kept) + if err != nil || string(kept) != "the predecessor's page\n" { + t.Fatalf("the original was not kept: %q %v", kept, err) + } + if info, _ := os.Stat(h.Kept); info.Mode().Perm() != 0o600 { + t.Errorf("the kept original is mode %o", info.Mode().Perm()) + } + if _, recorded := state.Find("hello-web.page"); recorded { + t.Error("a held file was recorded as applied, so it would be removed as an orphan") + } + if report.Changed() { + t.Errorf("holding was reported as changing the machine: %+v", report.Outcomes) + } +} + +func TestAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) { + dir, page, m := predecessor(t) + report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + + if m.removed("hello-web") { + t.Fatal("a found container was removed") + } + for _, a := range m.asked { + if strings.HasPrefix(a, "docker run") { + t.Fatalf("a container was started over a found one: %s", a) + } + } + if outcomeOf(report, "hello-web.server").Action != "held" { + t.Errorf("the found container was not held: %+v", report.Outcomes) + } + if h, _ := state.HeldAt("hello-web.server"); h.Container != "predecessor-id" || !h.Running { + t.Errorf("the container as found was not recorded: %+v", h) + } +} + +func TestWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) { + // Assigning prepares: what the module declares that is not there is made. + dir := t.TempDir() + page := filepath.Join(dir, "index.html") + m := &machine{containers: map[string]*fakeContainer{}} + report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.page"); o.Action != "created" { + t.Errorf("an absent file of an untaken module was not created: %+v", o) + } + if o := outcomeOf(report, "hello-web.server"); o.Action != "created" { + t.Errorf("an absent container of an untaken module was not created: %+v", o) + } + if len(state.Held) != 0 { + t.Errorf("something was held that was not found: %+v", state.Held) + } +} + +func TestAFileThisHostWroteIsNotFound(t *testing.T) { + // Found means present with no record. A record of any origin — carried or declared, this life + // of the node or an earlier one — means this host wrote it. + for _, origin := range []string{store.OriginCarried, store.OriginDeclared} { + dir, page, m := predecessor(t) + known := store.State{Resources: []store.Applied{ + {ID: "earlier-name", Type: "file", Target: page, Origin: origin}}} + report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), known, m, dir) + if o := outcomeOf(report, "hello-web.page"); o.Action == "held" { + t.Errorf("%s: a file this host has a record of was held: %+v", origin, o) + } + if _, held := state.HeldAt("hello-web.page"); held { + t.Errorf("%s: a recorded file was held", origin) + } + } +} + +func TestAContainerAHostMadeIsNotFound(t *testing.T) { + dir, page, m := predecessor(t) + m.containers["hello-web"].spec = "some-spec" + report, _ := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.server"); o.Action == "held" { + t.Errorf("a container carrying the host's spec label was held: %+v", o) + } +} + +func TestTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) { + // ADR 0078: the foundation's store, raised from the bundle and recorded as carried, is adopted + // as a module by name. It is the mesh's own and must never read as a predecessor's. + dir := t.TempDir() + m := &machine{containers: map[string]*fakeContainer{"mesh-store": {id: "x", running: true}}} + known := store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}} + d := adopted(t, `{"taken":[],"untaken":{"postgres":["postgres.server"]}}`, + `{"id":"postgres.server","type":"container","name":"mesh-store","image":"`+pinned+`"}`) + report, state := applyAdopted(t, d, known, m, dir) + if o := outcomeOf(report, "postgres.server"); o.Action == "held" { + t.Errorf("the carried store was held: %+v", o) + } + if len(state.Held) != 0 { + t.Errorf("the carried store was held: %+v", state.Held) + } +} + +func TestTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) { + dir, page, m := predecessor(t) + _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + h, _ := state.HeldAt("hello-web.page") + + report, state := applyAdopted(t, adopted(t, takenWeb, webResources(page)), state, m, dir) + got, _ := os.ReadFile(page) + if string(got) != "the mesh's page\n" { + t.Fatalf("taking the module did not converge the file: %q", got) + } + if !m.removed("hello-web") || m.containers["hello-web"].id != "made-by-host" { + t.Fatal("taking the module did not replace the found container") + } + if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "original kept at "+h.Kept) { + t.Errorf("the cutover does not say where the original is: %+v", o) + } + if len(state.Held) != 0 { + t.Errorf("what was taken is still held: %+v", state.Held) + } + if _, recorded := state.Find("hello-web.page"); !recorded { + t.Error("a taken file was not recorded as applied") + } + kept, err := os.ReadFile(h.Kept) + if err != nil || string(kept) != "the predecessor's page\n" { + t.Errorf("the kept original did not survive the cutover: %q %v", kept, err) + } +} + +func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) { + dir, page, m := predecessor(t) + _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + + other := filepath.Join(dir, "other") + _, state = applyAdopted(t, adopted(t, `{"taken":[]}`, + `{"id":"x.other","type":"file","path":"`+other+`","content":"x"}`), state, m, dir) + if got, _ := os.ReadFile(page); string(got) != "the predecessor's page\n" { + t.Fatalf("a held file was touched when its module left: %q", got) + } + if m.removed("hello-web") { + t.Fatal("a held container was removed when its module left") + } + if _, still := state.HeldAt("hello-web.page"); still { + t.Error("a hold outlived its resource leaving the declaration, so the node reports it for ever") + } + if _, recorded := state.Find("hello-web.page"); recorded { + t.Error("a file let go was recorded as the mesh's") + } +} + +func TestAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing.T) { + dir, page, m := predecessor(t) + _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + first, _ := state.HeldAt("hello-web.page") + + report, state := applyAdopted(t, adopted(t, `{"taken":[]}`, withConf(dir)), state, m, dir) + if o := outcomeOf(report, "hello-web.page"); o.Action != "forgotten" || o.Detail != "no longer declared; left as found" { + t.Errorf("letting a hold go was not reported: %+v", o) + } + if len(state.Held) != 0 { + t.Errorf("holds outlived their resources: %+v", state.Held) + } + + // The module comes back: what is there is found again, and the original first kept stays. + if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil { + t.Fatal(err) + } + report, state = applyAdopted(t, adopted(t, untakenWeb, webResources(page)), state, m, dir) + if o := outcomeOf(report, "hello-web.page"); o.Action != "held" { + t.Fatalf("a returning module's found file was not held again: %+v", o) + } + again, _ := state.HeldAt("hello-web.page") + // The predecessor rewrote it while nothing held it, so that is a second original, kept beside + // the first rather than in place of it (novox/hq ADR 0100). + if kept, _ := os.ReadFile(again.Kept); string(kept) != "the predecessor wrote again\n" { + t.Errorf("what is named as kept is %q", kept) + } + if kept, _ := os.ReadFile(first.Kept); string(kept) != "the predecessor's page\n" { + t.Errorf("the first kept original was lost: %q", kept) + } + if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" { + t.Errorf("the found file was touched: %q", got) + } +} + +func TestACarriedApplyLetsNoHoldGo(t *testing.T) { + dir, page, m := predecessor(t) + _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) + carried := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) + _, state, err := ApplyKeeping(context.Background(), archHost(t), carried, state, store.OriginCarried, + m.run, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if len(state.Held) != 2 { + t.Errorf("a carried apply let holds go: %+v", state.Held) + } +} + +func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) { + dir, page, m := predecessor(t) + d := adopted(t, untakenWeb, webResources(page)) + _, state := applyAdopted(t, d, store.State{}, m, dir) + + if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil { + t.Fatal(err) + } + report, state := applyAdopted(t, d, state, m, dir) + if got, _ := os.ReadFile(page); string(got) != "the predecessor wrote again\n" { + t.Fatalf("a held file was reverted: %q", got) + } + if h, _ := state.HeldAt("hello-web.page"); h.Changed != "rewritten" || h.ChangedAt.IsZero() { + t.Errorf("a rewrite was not recorded: %+v", h) + } + if o := outcomeOf(report, "hello-web.page"); !strings.Contains(o.Detail, "rewritten") { + t.Errorf("a rewrite was not reported: %+v", o) + } + h, _ := state.HeldAt("hello-web.page") + if kept, _ := os.ReadFile(h.Kept); string(kept) != "the predecessor's page\n" { + t.Errorf("the kept original was overwritten by a later write: %q", kept) + } +} + +func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) { + for _, c := range []struct { + change func(*machine) + want string + }{ + {func(m *machine) { m.containers["hello-web"].running = false }, "stopped"}, + {func(m *machine) { m.containers["hello-web"].id = "another" }, "replaced"}, + {func(m *machine) { delete(m.containers, "hello-web") }, "gone"}, + } { + dir, page, m := predecessor(t) + d := adopted(t, untakenWeb, webResources(page)) + _, state := applyAdopted(t, d, store.State{}, m, dir) + c.change(m) + m.asked = nil + _, state = applyAdopted(t, d, state, m, dir) + if h, _ := state.HeldAt("hello-web.server"); h.Changed != c.want { + t.Errorf("%s: recorded as %q", c.want, h.Changed) + } + for _, a := range m.asked { + if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") || + strings.HasPrefix(a, "docker start") { + t.Errorf("%s: the held container was acted on: %s", c.want, a) + } + } + } +} + +func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) { + dir, page, m := predecessor(t) + d := adopted(t, untakenWeb, webResources(page)) + _, state := applyAdopted(t, d, store.State{}, m, dir) + if err := os.Remove(page); err != nil { + t.Fatal(err) + } + _, state = applyAdopted(t, d, state, m, dir) + if _, err := os.Stat(page); !errors.Is(err, os.ErrNotExist) { + t.Fatal("a held file that vanished was created before its module was taken") + } + if h, _ := state.HeldAt("hello-web.page"); h.Changed != "gone" { + t.Errorf("a vanished held file was not reported gone: %+v", h) + } +} + +func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) { + // No adoption, no holds: byte for byte what a converged node did before ADR 0100. + dir, page, m := predecessor(t) + d := parse(t, `{"declaration":1,"resources":[`+webResources(page)+`]}`) + report, state := applyAdopted(t, d, store.State{}, m, dir) + if got, _ := os.ReadFile(page); string(got) != "the mesh's page\n" { + t.Errorf("a converged node kept a found file: %q", got) + } + if !m.removed("hello-web") { + t.Error("a converged node kept a found container") + } + if len(state.Held) != 0 || outcomeOf(report, "hello-web.page").Action == "held" { + t.Errorf("a converged node held something: %+v", state.Held) + } + // What it writes over that it has no record of, it keeps first — on any node. + o := outcomeOf(report, "hello-web.page") + kept := o.Detail[strings.Index(o.Detail, "kept at ")+len("kept at "):] + if got, err := os.ReadFile(kept); err != nil || string(got) != "the predecessor's page\n" { + t.Errorf("the file written over was not kept, or not named: %q (%s) %v", got, o.Detail, err) + } +} + +func TestAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt(t *testing.T) { + dir := t.TempDir() + conf := filepath.Join(dir, "nftables.conf") + _ = os.WriteFile(conf, []byte("# the distribution's own\n"), 0o644) + decl := func(content string) *declaration.Declaration { + return parse(t, `{"declaration":1,"resources":[{"id":"nftables.config","type":"file","path":"`+conf+ + `","content":"`+content+`"}]}`) + } + m := &machine{containers: map[string]*fakeContainer{}} + report, state := applyAdopted(t, decl("table inet mesh {}\\n"), store.State{}, m, dir) + o := outcomeOf(report, "nftables.config") + if !strings.Contains(o.Detail, "had no record of, was kept at ") { + t.Fatalf("writing over an unrecorded file did not keep it: %+v", o) + } + kept := o.Detail[strings.Index(o.Detail, "kept at ")+len("kept at "):] + if got, _ := os.ReadFile(kept); string(got) != "# the distribution's own\n" { + t.Errorf("the kept original is %q", got) + } + + // Now the mesh's: a later change keeps nothing more, and the first original stays. + report, _ = applyAdopted(t, decl("table inet mesh { }\\n"), state, m, dir) + if o := outcomeOf(report, "nftables.config"); o.Action != "updated" || strings.Contains(o.Detail, "kept at") { + t.Errorf("a file the mesh wrote was kept again: %+v", o) + } + if got, _ := os.ReadFile(kept); string(got) != "# the distribution's own\n" { + t.Errorf("the first original was overwritten: %q", got) + } +} + +// Defends novox/hq ADR 0103: found covers every kind that can reach what the machine already has. + +// untaken is an adoption with hello-web untaken, listing these of its resources. +func untaken(ids ...string) string { + return `{"taken":[],"untaken":{"hello-web":["` + strings.Join(ids, `","`) + `"]}}` +} + +func TestAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing.T) { + dir := t.TempDir() + data := filepath.Join(dir, "data") + if err := os.Mkdir(data, 0o700); err != nil { + t.Fatal(err) + } + inside := filepath.Join(data, "PG_VERSION") + if err := os.WriteFile(inside, []byte("16\n"), 0o600); err != nil { + t.Fatal(err) + } + m := &machine{containers: map[string]*fakeContainer{}} + report, state := applyAdopted(t, adopted(t, untaken("hello-web.data"), + `{"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0755"}`), store.State{}, m, dir) + + if info, _ := os.Stat(data); info.Mode().Perm() != 0o700 { + t.Errorf("a found directory was re-moded to %o", info.Mode().Perm()) + } + if got, _ := os.ReadFile(inside); string(got) != "16\n" { + t.Errorf("what is inside a found directory was touched: %q", got) + } + if o := outcomeOf(report, "hello-web.data"); o.Action != "held" || !strings.Contains(o.Detail, "mode, owner and contents") { + t.Errorf("the found directory was not held: %+v", o) + } + if h, ok := state.HeldAt("hello-web.data"); !ok || h.Mode != "0700" { + t.Errorf("the directory as found was not recorded: %+v", h) + } + if _, recorded := state.Find("hello-web.data"); recorded { + t.Error("a held directory was recorded as applied") + } +} + +func TestADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) { + // A file's parent is made as the file is written; what the mesh made is not found. + dir := t.TempDir() + data := filepath.Join(dir, "data") + m := &machine{containers: map[string]*fakeContainer{}} + report, state := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.data"), + `{"id":"hello-web.conf","type":"file","path":"`+filepath.Join(data, "conf")+`","content":"x\n"}, + {"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0750"}`), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.data"); o.Action == "held" { + t.Errorf("a directory the apply itself made was held: %+v", o) + } + if info, _ := os.Stat(data); info.Mode().Perm() != 0o750 { + t.Errorf("the mesh's own directory was not converged: %o", info.Mode().Perm()) + } + if len(state.Held) != 0 { + t.Errorf("held: %+v", state.Held) + } +} + +func TestAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t *testing.T) { + dir := t.TempDir() + conf := filepath.Join(dir, "hello.conf") + m := &machine{containers: map[string]*fakeContainer{}, + // The predecessor's unit: stopped just now, but it starts at boot, so it is the machine's. + units: map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "enabled"}}} + report, state := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"), + `{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"x\n"}, + {"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled", + "restart-on":["hello-web.conf"]}`), store.State{}, m, dir) + + for _, verb := range []string{"systemctl start", "systemctl stop", "systemctl enable", "systemctl disable", "systemctl restart"} { + if m.did(verb) { + t.Errorf("a found service was changed: %s (%v)", verb, m.asked) + } + } + if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" || !strings.Contains(o.Detail, "starts at boot") { + t.Errorf("the found service was not held: %+v", o) + } + if h, ok := state.HeldAt("hello-web.unit"); !ok || h.Running { + t.Errorf("the service as found was not recorded: %+v", h) + } +} + +func TestAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) { + // A reload stops nothing (novox/hq ADR 0102); a restart would stop the predecessor's service. + dir := t.TempDir() + conf := filepath.Join(dir, "daemon.json") + m := &machine{containers: map[string]*fakeContainer{}, + units: map[string]*fakeUnit{"docker.service": {active: "active", enabled: "enabled"}}} + report, _ := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"), + `{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"{}\n"}, + {"id":"hello-web.unit","type":"service","unit":"docker.service","state":"running","boot":"enabled", + "reload-on":["hello-web.conf"]}`), store.State{}, m, dir) + if !m.did("systemctl reload docker.service") { + t.Errorf("a held service was not reloaded for what it re-reads: %v", m.asked) + } + if m.did("systemctl stop") || m.did("systemctl start") { + t.Errorf("a held service was restarted: %v", m.asked) + } + if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" || !strings.Contains(o.Detail, "reloaded for hello-web.conf") { + t.Errorf("the reload was not reported on the hold: %+v", o) + } +} + +func TestAUnitAPackageOnlyShipsIsNotFound(t *testing.T) { + // The adoption bed found this: the private network's wg-quick@mesh0 is an instance of a unit + // the tunnel package ships. Nothing had ever run it, yet it was held as a predecessor's, and + // the private network never came up. Found is what the machine runs. + dir := t.TempDir() + m := &machine{containers: map[string]*fakeContainer{}, + units: map[string]*fakeUnit{"wg-quick@mesh0.service": {active: "inactive", enabled: "disabled", + fragment: "/usr/lib/systemd/system/wg-quick@.service"}}} + report, state := applyAdopted(t, adopted(t, untaken("mesh-wireguard.overlay-up"), + `{"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0.service","state":"running","boot":"enabled"}`), + store.State{}, m, dir) + if o := outcomeOf(report, "mesh-wireguard.overlay-up"); o.Action == "held" { + t.Fatalf("a unit nothing runs was held as a predecessor's: %+v", o) + } + if !m.did("systemctl start wg-quick@mesh0.service") || !m.did("systemctl enable wg-quick@mesh0.service") { + t.Errorf("the unit was not started and enabled: %v", m.asked) + } + if len(state.Held) != 0 { + t.Errorf("held: %+v", state.Held) + } +} + +func TestAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) { + // No unit before the apply: nothing of a predecessor's to hold. + dir := t.TempDir() + m := &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{}} + d := adopted(t, untaken("hello-web.unit"), `{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running"}`) + _, _, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, + m.run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "does not exist") { + t.Errorf("an absent unit was held rather than applied: %v", err) + } +} + +func TestAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) { + dir := t.TempDir() + data := filepath.Join(dir, "predecessor-data") + if err := os.Mkdir(data, 0o700); err != nil { + t.Fatal(err) + } + for _, c := range []struct { + name, volume string + m *machine + }{ + {"a path", data + ":/var/lib/postgresql/data", &machine{containers: map[string]*fakeContainer{}}}, + {"a named volume", "predecessor-pgdata:/var/lib/postgresql/data", + &machine{containers: map[string]*fakeContainer{}, volumes: map[string]bool{"predecessor-pgdata": true}}}, + } { + report, state := applyAdopted(t, adopted(t, untaken("hello-web.server"), + `{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`", + "volumes":["`+c.volume+`"]}`), store.State{}, c.m, dir) + if c.m.did("docker run") { + t.Errorf("%s: a container mounting found data was created: %v", c.name, c.m.asked) + } + o := outcomeOf(report, "hello-web.server") + if o.Action != "held" || !strings.Contains(o.Detail, "would mount") { + t.Errorf("%s: not held: %+v", c.name, o) + } + if h, ok := state.HeldAt("hello-web.server"); !ok || !strings.Contains(h.Why, "would mount") { + t.Errorf("%s: the hold does not say why: %+v", c.name, h) + } + // Held, it stays held on the next pass, and is still not created. + c.m.asked = nil + _, state = applyAdopted(t, adopted(t, untaken("hello-web.server"), + `{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`", + "volumes":["`+c.volume+`"]}`), state, c.m, dir) + if c.m.did("docker run") || len(state.Held) != 1 { + t.Errorf("%s: a held container was created on the next pass: %v", c.name, c.m.asked) + } + } +} + +func TestAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) { + dir := t.TempDir() + data := filepath.Join(dir, "data") + m := &machine{containers: map[string]*fakeContainer{}} + report, _ := applyAdopted(t, adopted(t, untaken("hello-web.data", "hello-web.server"), + `{"id":"hello-web.data","type":"directory","path":"`+data+`"}, + {"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`", + "volumes":["`+data+`:/data"]}`), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.server"); o.Action != "created" { + t.Errorf("a container mounting only what the mesh made was not created: %+v", o) + } +} + +func TestARunOnceStepInAHeldContainerIsHeld(t *testing.T) { + dir, page, m := predecessor(t) + step := `{"id":"hello-web.migrate","type":"container","name":"hello-web-migrate","image":"` + pinned + `", + "run-once":true,"network":"container:hello-web"}` + report, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server", "hello-web.migrate"), webResources(page)+","+step), store.State{}, m, dir) + if m.did("docker run") { + t.Errorf("a step was run inside a held container: %v", m.asked) + } + o := outcomeOf(report, "hello-web.migrate") + if o.Action != "held" || !strings.Contains(o.Detail, "runs in hello-web") { + t.Errorf("the step was not held: %+v", o) + } + if _, ok := state.HeldAt("hello-web.migrate"); !ok { + t.Error("the held step is not reported held") + } +} + +func TestAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) { + // An action cannot arrive over the link today, and a bundle cannot say a node is adopted; the + // host holds one anyway, since what it would run in is the predecessor's. + dir, page, m := predecessor(t) + d := adopted(t, untakenWeb, webResources(page)) + d.Resources = append(d.Resources, &declaration.Action{ID: "hello-web.seed", Type: declaration.TypeAction, + In: "hello-web", Command: []string{"seed"}, Verify: []string{"seeded"}}) + report, state := applyAdopted(t, d, store.State{}, m, dir) + if m.did("docker exec") { + t.Errorf("an action was run inside a held container: %v", m.asked) + } + if o := outcomeOf(report, "hello-web.seed"); o.Action != "held" || !strings.Contains(o.Detail, "not run until hello-web is taken") { + t.Errorf("the action was not held: %+v", o) + } + if h, ok := state.HeldAt("hello-web.seed"); !ok || h.Module != "hello-web" { + t.Errorf("the held action is not its module's: %+v", h) + } + + // Taken: the container is the mesh's, and the action runs in it. + taken := adopted(t, takenWeb, webResources(page)) + taken.Resources = append(taken.Resources, d.Resources[len(d.Resources)-1]) + report, state = applyAdopted(t, taken, state, m, dir) + if !m.did("docker exec hello-web ") { + t.Errorf("the action did not run once its module was taken: %v", m.asked) + } + if _, still := state.HeldAt("hello-web.seed"); still || len(state.Held) != 0 { + t.Errorf("holds outlived the take: %+v", state.Held) + } +} + +const sixtyFourZeros = "0000000000000000000000000000000000000000000000000000000000000000" + +func TestAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) { + dir := t.TempDir() + at := filepath.Join(dir, "site") + if err := os.Mkdir(at, 0o750); err != nil { + t.Fatal(err) + } + theirs := filepath.Join(at, "index.html") + _ = os.WriteFile(theirs, []byte("the predecessor's site\n"), 0o640) + m := &machine{containers: map[string]*fakeContainer{}} + // The source is unreachable: fetching it would fail the apply, so a pass means it was not tried. + report, state := applyAdopted(t, adopted(t, untaken("hello-web.site"), + `{"id":"hello-web.site","type":"archive","source":"http://192.0.2.1/site.tar.gz", + "digest":"sha256:`+sixtyFourZeros+`","path":"`+at+`","owner":"root"}`), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.site"); o.Action != "held" || !strings.Contains(o.Detail, "nothing unpacked") { + t.Errorf("an archive over found files was not held: %+v", o) + } + if got, _ := os.ReadFile(theirs); string(got) != "the predecessor's site\n" { + t.Errorf("the found files were touched: %q", got) + } + if _, ok := state.HeldAt("hello-web.site"); !ok { + t.Error("the hold was not recorded") + } +} + +func TestAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) { + dir := t.TempDir() + was := unitDir + unitDir = dir + t.Cleanup(func() { unitDir = was }) + unit := filepath.Join(dir, "hello-daemon.service") + _ = os.WriteFile(unit, []byte("[Service]\nExecStart=/opt/predecessor/hello\n"), 0o644) + m := &machine{containers: map[string]*fakeContainer{}} + report, state := applyAdopted(t, adopted(t, untaken("hello-web.daemon"), + `{"id":"hello-web.daemon","type":"process","name":"hello-daemon","source":"http://192.0.2.1/d.tar.gz", + "digest":"sha256:`+sixtyFourZeros+`","run":["hello"]}`), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.daemon"); o.Action != "held" || !strings.Contains(o.Detail, "not written over or restarted") { + t.Errorf("a process whose unit was found was not held: %+v", o) + } + if got, _ := os.ReadFile(unit); string(got) != "[Service]\nExecStart=/opt/predecessor/hello\n" { + t.Errorf("the found unit was written over: %q", got) + } + if m.did("systemctl") { + t.Errorf("the found unit was touched: %v", m.asked) + } + if _, ok := state.HeldAt("hello-web.daemon"); !ok { + t.Error("the hold was not recorded") + } +} + +func TestAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) { + dir := t.TempDir() + m := &machine{containers: map[string]*fakeContainer{}, users: map[string]bool{"hello": true}} + report, _ := applyAdopted(t, adopted(t, untaken("hello-web.user"), + `{"id":"hello-web.user","type":"user","name":"hello","shell":"/bin/zsh","groups":["docker"]}`), + store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.user"); o.Action != "held" || !strings.Contains(o.Detail, "shell and groups") { + t.Errorf("a found user was not held: %+v", o) + } + for _, a := range m.asked { + if strings.HasPrefix(a, "usermod") || strings.HasPrefix(a, "useradd") { + t.Errorf("a found user was changed: %s", a) + } + } +} + +func TestMountingTheMachinesOwnPlumbingIsNotFoundData(t *testing.T) { + // The runtime's socket, the kernel's filesystems and the clock are on every machine; a + // container mounting them shares nothing a predecessor kept (novox/hq ADR 0103). + dir := t.TempDir() + m := &machine{containers: map[string]*fakeContainer{}} + report, state := applyAdopted(t, adopted(t, untaken("hello-web.server"), + `{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`", + "volumes":["/var/run/docker.sock:/var/run/docker.sock","/etc/localtime:/etc/localtime:ro", + "/proc/cpuinfo:/host/cpuinfo:ro","/dev/null:/data/null"]}`), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.server"); o.Action != "created" { + t.Errorf("a container mounting only system paths was not created: %+v", o) + } + if len(state.Held) != 0 { + t.Errorf("held: %+v", state.Held) + } +} + +// Defends novox/hq ADR 0100: a runtime that cannot answer is not a machine with nothing there. + +// unreachable is a machine whose container runtime answers everything with a daemon that is down. +type unreachable struct{ asked []string } + +func (u *unreachable) run(_ context.Context, name string, args ...string) (string, error) { + u.asked = append(u.asked, name+" "+strings.Join(args, " ")) + if name == "docker" && args[0] == "info" { + return "27.0\n", nil + } + if name == "docker" { + return "", errors.New("docker exited 1: Cannot connect to the Docker daemon at unix:///var/run/docker.sock") + } + return "", nil +} + +func TestARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *testing.T) { + dir := t.TempDir() + u := &unreachable{} + d := adopted(t, untaken("hello-web.server"), + `{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`"}`) + _, state, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, + u.run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "not safe to make one") { + t.Fatalf("a runtime that could not answer was read as nothing there: %v", err) + } + for _, a := range u.asked { + if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") { + t.Errorf("the mesh acted on a container it could not ask about: %s", a) + } + } + if len(state.Held) != 0 { + t.Errorf("something was held on an answer the machine never gave: %+v", state.Held) + } +} + +func TestAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) { + dir, page, m := predecessor(t) + d := adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)) + _, state := applyAdopted(t, d, store.State{}, m, dir) + if _, held := state.HeldAt("hello-web.server"); !held { + t.Fatal("the found container was not held to begin with") + } + u := &unreachable{} + _, state, err := ApplyKeeping(context.Background(), archHost(t), d, state, store.OriginDeclared, + u.run, nil, nil, KeepIn(dir)) + if err == nil { + t.Fatal("a runtime that could not answer reported success") + } + if h, held := state.HeldAt("hello-web.server"); !held || h.Changed != "" { + t.Errorf("a hold was let go or called changed on an answer the machine never gave: %+v", h) + } +} + +func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) { + dir := t.TempDir() + run := func(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "docker" && args[0] == "info": + return "27.0\n", nil + case name == "docker" && args[0] == "volume": + return "", errors.New("docker exited 1: Cannot connect to the Docker daemon") + case name == "docker" && args[0] == "inspect": + return "", errors.New("Error: No such object: hello-web") + case name == "docker": + return "", errors.New("docker run must not happen") + } + return "", nil + } + d := adopted(t, untaken("hello-web.server"), + `{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`", + "volumes":["predecessor-data:/data"]}`) + _, _, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, + run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "could not say whether the volume") { + t.Fatalf("a volume the runtime could not be asked about did not stop the container: %v", err) + } +} + +func TestAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) { + // A predecessor's unit under /etc, deliberately stopped and disabled: starting it would put + // back a service somebody took down on purpose (novox/hq ADR 0103). + dir := t.TempDir() + m := &machine{containers: map[string]*fakeContainer{}, + units: map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "disabled", + fragment: "/etc/systemd/system/hello.service"}}} + report, state := applyAdopted(t, adopted(t, untaken("hello-web.unit"), + `{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled"}`), + store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" { + t.Fatalf("a unit an administrator installed was not held: %+v", o) + } + if m.did("systemctl start") || m.did("systemctl enable") { + t.Errorf("a unit somebody had stopped and disabled was started: %v", m.asked) + } + if _, ok := state.HeldAt("hello-web.unit"); !ok { + t.Error("the hold was not recorded") + } +} + +func TestAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) { + // The predecessor's own service from a package, running: not the mesh's to restart. + dir := t.TempDir() + conf := filepath.Join(dir, "hello.conf") + m := &machine{containers: map[string]*fakeContainer{}, + units: map[string]*fakeUnit{"nginx.service": {active: "active", enabled: "enabled", + fragment: "/usr/lib/systemd/system/nginx.service"}}} + report, _ := applyAdopted(t, adopted(t, untaken("hello-web.conf", "hello-web.unit"), + `{"id":"hello-web.conf","type":"file","path":"`+conf+`","content":"x\n"}, + {"id":"hello-web.unit","type":"service","unit":"nginx.service","state":"running","boot":"enabled", + "restart-on":["hello-web.conf"]}`), store.State{}, m, dir) + if o := outcomeOf(report, "hello-web.unit"); o.Action != "held" { + t.Fatalf("a packaged unit the machine runs was not held: %+v", o) + } + if m.did("systemctl stop") || m.did("systemctl restart") { + t.Errorf("the predecessor's service was restarted: %v", m.asked) + } +} + +func TestASecondOriginalAtTheSamePathIsKeptToo(t *testing.T) { + // Held, let go when the module was unassigned, rewritten by the predecessor, found again: + // both originals are kept, and the report names the one it kept (novox/hq ADR 0100). + dir := t.TempDir() + page := filepath.Join(dir, "index.html") + keep := KeepIn(dir) + first, err := keep(page, []byte("the predecessor's page\n"), 0o640) + if err != nil { + t.Fatal(err) + } + same, err := keep(page, []byte("the predecessor's page\n"), 0o640) + if err != nil || same != first { + t.Errorf("the same original was kept twice: %s %s %v", first, same, err) + } + second, err := keep(page, []byte("the predecessor wrote again\n"), 0o640) + if err != nil { + t.Fatal(err) + } + if second == first { + t.Fatal("a second original was kept under the first's name") + } + if got, _ := os.ReadFile(first); string(got) != "the predecessor's page\n" { + t.Errorf("the first original is %q", got) + } + if got, _ := os.ReadFile(second); string(got) != "the predecessor wrote again\n" { + t.Errorf("the second original is %q", got) + } +} + +func TestAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) { + dir, page, m := predecessor(t) + _, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)), + store.State{}, m, dir) + first, _ := state.HeldAt("hello-web.page") + + // Unassigned, then the predecessor writes again, then assigned once more. + _, state = applyAdopted(t, adopted(t, `{"taken":[]}`, withConf(dir)), state, m, dir) + if err := os.WriteFile(page, []byte("the predecessor wrote again\n"), 0o640); err != nil { + t.Fatal(err) + } + report, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)), + state, m, dir) + again, _ := state.HeldAt("hello-web.page") + if again.Kept == first.Kept { + t.Fatalf("the second original was kept under the first's name: %s", again.Kept) + } + if got, _ := os.ReadFile(again.Kept); string(got) != "the predecessor wrote again\n" { + t.Errorf("what the report names as kept is %q", got) + } + if got, _ := os.ReadFile(first.Kept); string(got) != "the predecessor's page\n" { + t.Errorf("the first original was lost: %q", got) + } + if o := outcomeOf(report, "hello-web.page"); o.Action != "held" { + t.Errorf("the file found again was not held: %+v", o) + } +} + +func TestAPathTheMeshOnlySetAccessOnIsStillFound(t *testing.T) { + // An access record says the mesh set permissions on a path it does not own — which is what it + // does to somebody else's directory. It is not a record of making it (novox/hq ADR 0103). + dir := t.TempDir() + data := filepath.Join(dir, "data") + if err := os.Mkdir(data, 0o700); err != nil { + t.Fatal(err) + } + known := store.State{Resources: []store.Applied{ + {ID: "hello-web.readable", Type: "access", Target: data, Origin: store.OriginDeclared}}} + m := &machine{containers: map[string]*fakeContainer{}} + report, _ := applyAdopted(t, adopted(t, untaken("hello-web.data"), + `{"id":"hello-web.data","type":"directory","path":"`+data+`","mode":"0755"}`), known, m, dir) + if o := outcomeOf(report, "hello-web.data"); o.Action != "held" { + t.Errorf("a directory the mesh only has access for was not held: %+v", o) + } + if info, _ := os.Stat(data); info.Mode().Perm() != 0o700 { + t.Errorf("it was re-moded to %o", info.Mode().Perm()) + } +} diff --git a/internal/apply/into.go b/internal/apply/into.go new file mode 100644 index 0000000..eefcbeb --- /dev/null +++ b/internal/apply/into.go @@ -0,0 +1,388 @@ +package apply + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "sort" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// A file written into, never over (novox/hq ADR 0102). +// +// **The file is the machine's; the mesh owns keys in it.** The container runtime's configuration +// is the case that needed it: the mesh states one fact there — its registry is trusted over the +// private network — and writing the file whole replaced everything the machine had set, down to +// where the runtime keeps its data. So the host reads what is there, sets only the declared keys, +// keeps every other key as it found it, and records what each of its keys held before. Undeclared, +// each key goes back, and a file the mesh created goes only if nothing but its keys is left. + +// applyInto writes a file's declared keys into the object already at its path. +func applyInto(r *declaration.File, previous store.Applied) (Outcome, error) { + out := begin(r) + if r.Into != declaration.IntoJSON { + return out, fmt.Errorf("%s: into %q is not a format this host writes into", r.Path, r.Into) + } + var declared map[string]json.RawMessage + if err := json.Unmarshal([]byte(r.Content), &declared); err != nil { + return out, fmt.Errorf("%s: the keys to write are not a JSON object: %w", r.Path, err) + } + + existing, err := os.ReadFile(r.Path) + existed := err == nil + if err != nil && !errors.Is(err, os.ErrNotExist) { + return out, err + } + object := map[string]json.RawMessage{} + if existed && len(bytes.TrimSpace(existing)) > 0 { + if err := json.Unmarshal(existing, &object); err != nil || object == nil { + // Refused, never replaced: a file the host cannot read as an object is a file it + // cannot write into without losing whatever it is. + return out, fmt.Errorf("%s is not a JSON object, so the mesh cannot write its keys into it "+ + "without replacing what is there; it was left as it is", r.Path) + } + } + + rec := store.Into{Format: declaration.IntoJSON, Before: map[string]json.RawMessage{}, + Added: map[string][]json.RawMessage{}} + if previous.Into != nil { + rec.Created = previous.Into.Created + for k, v := range previous.Into.Before { + rec.Before[k] = v + } + rec.Absent = slices.Clone(previous.Into.Absent) + for k, v := range previous.Into.Added { + rec.Added[k] = slices.Clone(v) + } + } else { + rec.Created = !existed + } + tracked := func(k string) bool { + _, before := rec.Before[k] + _, added := rec.Added[k] + return before || added || slices.Contains(rec.Absent, k) + } + + // Drift: the machine no longer holds what this host last set in its keys. + drifted := previous.Wrote != "" && existed && digestOf(viewOf(object, rec, keysTracked(rec))) != previous.Wrote + + // Keys the mesh set before and no longer declares go back to what they held. + for _, k := range keysTracked(rec) { + if _, still := declared[k]; still { + continue + } + giveBack(object, &rec, k) + } + // Declared keys: remember what each held the first time, then set it. A list is the + // machine's too — a predecessor's own trusted registries, say — so the mesh adds its members + // to it rather than replacing it, and remembers exactly which it added. + for _, k := range keysIn(declared) { + _, scalar := rec.Before[k] + if isList(declared[k]) && !scalar { + current, had := object[k] + if had && !isList(current) { + return out, fmt.Errorf("%s: the mesh adds to the list %q, and the machine holds something "+ + "other than a list there; it was left as it is", r.Path, k) + } + if !tracked(k) && !had { + rec.Absent = append(rec.Absent, k) + } + merged, added, err := addMembers(current, declared[k], rec.Added[k], previous.Into == nil) + if err != nil { + return out, fmt.Errorf("%s: %q: %w", r.Path, k, err) + } + rec.Added[k] = added + object[k] = merged + continue + } + if !tracked(k) { + v, had := object[k] + // **What the host may have written itself is not the machine's.** With no record of + // this file — the first apply, or a host that wrote and died before saving its state — + // a key already holding exactly what the mesh declares cannot be told from one the + // mesh set a moment ago. Remembered as the machine's, it would never be given back: + // undeclaring would leave the mesh's own value behind for ever. So it is the mesh's, + // and undeclaring takes it out (novox/hq ADR 0102). + if had && !(previous.Into == nil && canonical(v) == canonical(declared[k])) { + rec.Before[k] = v + } else { + rec.Absent = append(rec.Absent, k) + } + } + object[k] = declared[k] + } + + want, err := render(object) + if err != nil { + return out, err + } + same := existed && canonical(existing) == canonical(want) + if !same { + mode := os.FileMode(0o644) + if info, err := os.Stat(r.Path); err == nil { + mode = info.Mode().Perm() // the machine's file keeps the machine's mode + } else if r.Mode != "" { + if m, err := modeOf(r.Mode, mode); err == nil { + mode = m + } + } + if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil { + return out, err + } + if err := writeAtomically(r.Path, want, mode); err != nil { + return out, err + } + } + // Read back: every declared key holds what was declared. + written, err := os.ReadFile(r.Path) + if err != nil { + return out, fmt.Errorf("wrote into %s and cannot read it back: %w", r.Path, err) + } + var check map[string]json.RawMessage + if err := json.Unmarshal(written, &check); err != nil { + return out, fmt.Errorf("%s is not a JSON object after writing into it: %w", r.Path, err) + } + for k, v := range declared { + if _, list := rec.Added[k]; list { + members, _ := membersOf(v) + have, err := membersOf(check[k]) + if err != nil { + return out, fmt.Errorf("%s does not hold a list at %q after writing into it", r.Path, k) + } + for _, m := range members { + if !hasMember(have, m) { + return out, fmt.Errorf("%s does not hold the declared %s in %q after writing into it", r.Path, m, k) + } + } + continue + } + if canonical(check[k]) != canonical(v) { + return out, fmt.Errorf("%s does not hold the declared %q after writing into it", r.Path, k) + } + } + + if len(rec.Before) == 0 { + rec.Before = nil + } + if len(rec.Added) == 0 { + rec.Added = nil + } + out.into = &rec + out.wrote = digestOf(viewOf(check, rec, keysIn(declared))) + switch { + case !existed: + out.Action = "created" + out.Detail = "written into; the file was not there" + case same: + out.Action = "unchanged" + case drifted: + out.Action = "corrected" + out.Detail = "the mesh's keys had been changed on the machine; the rest of the file was kept" + default: + out.Action = "updated" + out.Detail = "the mesh's keys written in; every other key kept as it was" + } + return out, nil +} + +// removeInto gives back what a file written into held before the mesh's keys. +func removeInto(a store.Applied) (string, string, error) { + existing, err := os.ReadFile(a.Target) + if errors.Is(err, os.ErrNotExist) { + return "forgotten", "no longer there", nil + } + if err != nil { + return "", "", err + } + object := map[string]json.RawMessage{} + if len(bytes.TrimSpace(existing)) > 0 { + if err := json.Unmarshal(existing, &object); err != nil || object == nil { + return "kept", "no longer a JSON object, so the mesh's keys were left in it; " + + "remove them by hand", nil + } + } + rec := *a.Into + for _, k := range keysTracked(rec) { + giveBack(object, &rec, k) + } + if a.Into.Created && len(object) == 0 { + if err := os.Remove(a.Target); err != nil { + return "", "", err + } + return "removed", "no longer declared; the mesh had created it and nothing else was in it", nil + } + want, err := render(object) + if err != nil { + return "", "", err + } + info, err := os.Stat(a.Target) + if err != nil { + return "", "", err + } + if err := writeAtomically(a.Target, want, info.Mode().Perm()); err != nil { + return "", "", err + } + return "restored", "no longer declared; the mesh's keys were given back what they held", nil +} + +func giveBack(object map[string]json.RawMessage, rec *store.Into, k string) { + if added, list := rec.Added[k]; list { + // Only the members the mesh added go; the list and everything else in it stay, unless + // the mesh made the key and nothing is left in it. + wasAbsent := slices.Contains(rec.Absent, k) + if current, had := object[k]; had && isList(current) { + have, _ := membersOf(current) + have = slices.DeleteFunc(have, func(m json.RawMessage) bool { return hasMember(added, m) }) + if len(have) == 0 && wasAbsent { + delete(object, k) + } else { + object[k] = listOf(have) + } + } + delete(rec.Added, k) + rec.Absent = slices.DeleteFunc(rec.Absent, func(a string) bool { return a == k }) + return + } + if v, had := rec.Before[k]; had { + object[k] = v + delete(rec.Before, k) + return + } + delete(object, k) + rec.Absent = slices.DeleteFunc(rec.Absent, func(a string) bool { return a == k }) +} + +func keysTracked(rec store.Into) []string { + var keys []string + for k := range rec.Before { + keys = append(keys, k) + } + for k := range rec.Added { + keys = append(keys, k) + } + keys = append(keys, rec.Absent...) + sort.Strings(keys) + return slices.Compact(keys) +} + +// viewOf is what the mesh holds itself to in a file written into: each scalar key's value, and for +// a list only whether each member the mesh added is still there — what the machine keeps beside +// them is not the mesh's to judge. +func viewOf(object map[string]json.RawMessage, rec store.Into, keys []string) string { + var b bytes.Buffer + for _, k := range keys { + if added, list := rec.Added[k]; list { + have, _ := membersOf(object[k]) + b.WriteString(k + " holds") + for _, m := range added { + fmt.Fprintf(&b, " %s=%v", canonical(m), hasMember(have, m)) + } + b.WriteString("\n") + continue + } + b.WriteString(k + "=" + canonical(object[k]) + "\n") + } + return b.String() +} + +func isList(raw json.RawMessage) bool { + t := bytes.TrimSpace(raw) + return len(t) > 0 && t[0] == '[' +} + +func membersOf(raw json.RawMessage) ([]json.RawMessage, error) { + if len(bytes.TrimSpace(raw)) == 0 { + return nil, nil + } + var members []json.RawMessage + if err := json.Unmarshal(raw, &members); err != nil { + return nil, err + } + return members, nil +} + +func hasMember(list []json.RawMessage, m json.RawMessage) bool { + for _, have := range list { + if canonical(have) == canonical(m) { + return true + } + } + return false +} + +func listOf(members []json.RawMessage) json.RawMessage { + if members == nil { + members = []json.RawMessage{} + } + raw, _ := json.Marshal(members) + return raw +} + +// addMembers adds the declared members to the machine's list, dropping only members the mesh +// added before and no longer declares. It returns the list and exactly which members the mesh +// added — a declared member the machine already had is the machine's, and is never recorded. +// unrecorded says there is no record of this file yet, in which case a declared member already in +// the list may be one the host itself wrote before it could save its state, and is taken as the +// mesh's. +func addMembers(current, declared json.RawMessage, addedBefore []json.RawMessage, + unrecorded bool) (json.RawMessage, []json.RawMessage, error) { + have, err := membersOf(current) + if err != nil { + return nil, nil, err + } + want, err := membersOf(declared) + if err != nil { + return nil, nil, err + } + added := []json.RawMessage{} + for _, a := range addedBefore { + if hasMember(want, a) { + added = append(added, a) + continue + } + have = slices.DeleteFunc(have, func(m json.RawMessage) bool { return canonical(m) == canonical(a) }) + } + for _, m := range want { + if !hasMember(have, m) { + have = append(have, m) + } else if !unrecorded { + continue // the machine's own, and never the mesh's to take out + } + if !hasMember(added, m) { + added = append(added, m) + } + } + return listOf(have), added, nil +} + +func keysIn(m map[string]json.RawMessage) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// canonical is a JSON value compacted, so formatting is not mistaken for a change. +func canonical(raw []byte) string { + var b bytes.Buffer + if err := json.Compact(&b, raw); err != nil { + return string(raw) + } + return b.String() +} + +func render(object map[string]json.RawMessage) ([]byte, error) { + b, err := json.MarshalIndent(object, "", " ") + if err != nil { + return nil, err + } + return append(b, '\n'), nil +} diff --git a/internal/apply/into_test.go b/internal/apply/into_test.go new file mode 100644 index 0000000..c5f1362 --- /dev/null +++ b/internal/apply/into_test.go @@ -0,0 +1,342 @@ +package apply + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0102: a file the mesh shares with software it did not install is written +// into, never over, and a service that re-reads its configuration is reloaded, not restarted. + +func intoDecl(t *testing.T, path, keys string) string { + t.Helper() + return fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q} + ]}`, path, keys) +} + +func readObject(t *testing.T, path string) map[string]any { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var o map[string]any + if err := json.Unmarshal(raw, &o); err != nil { + t.Fatalf("%s is not a JSON object: %v\n%s", path, err, raw) + } + return o +} + +// The machine's own runtime settings, the way a predecessor leaves them. +const machinesOwn = `{"data-root":"/srv/docker","log-opts":{"max-size":"10m"},"insecure-registries":["192.0.2.7:5000"]}` + +func TestWritingIntoKeepsEveryKeyTheMachineHad(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + if err := os.WriteFile(path, []byte(machinesOwn), 0o600); err != nil { + t.Fatal(err) + } + d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + o := readObject(t, path) + if o["data-root"] != "/srv/docker" { + t.Errorf("the machine's data directory was not kept: %v", o) + } + if fmt.Sprint(o["log-opts"]) != "map[max-size:10m]" { + t.Errorf("the machine's logging settings were not kept: %v", o) + } + if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" { + t.Errorf("the mesh's member was not added beside the machine's own: %v", o) + } + if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 { + t.Errorf("the machine's file mode was changed to %o", info.Mode().Perm()) + } + if got := report.Outcomes[0].Action; got != "updated" { + t.Errorf("writing into was reported as %q", got) + } + + // Again, with nothing changed: nothing to do. + report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != "unchanged" { + t.Errorf("a second apply was %q", got) + } + + // Undeclared: the key goes back to what the machine had, and the file stays. + empty := somethingElse(t) + report, _, err = Apply(context.Background(), archHost(t), empty, state, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + o = readObject(t, path) + if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" || o["data-root"] != "/srv/docker" { + t.Errorf("undeclaring did not give the machine back what it had: %v", o) + } + if got := report.Outcomes[0].Action; got != "restored" { + t.Errorf("undeclaring was reported as %q", got) + } +} + +func TestAFileWrittenIntoThatWasNotThereIsRemovedWhenOnlyTheMeshsKeysAreLeft(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != "created" { + t.Errorf("writing into a file that was not there was %q", got) + } + // Somebody else adds a key of their own: the file is no longer only the mesh's. + o := readObject(t, path) + o["debug"] = true + raw, _ := json.Marshal(o) + _ = os.WriteFile(path, raw, 0o644) + + empty := somethingElse(t) + if _, _, err := Apply(context.Background(), archHost(t), empty, state, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + o = readObject(t, path) + if _, still := o["insecure-registries"]; still || o["debug"] != true { + t.Errorf("undeclaring should remove the mesh's key and keep the other: %v", o) + } + + // Without the other key, the file the mesh created goes. + path2 := filepath.Join(t.TempDir(), "daemon.json") + d2 := parse(t, intoDecl(t, path2, `{"insecure-registries":["10.42.0.1:5000"]}`)) + _, state2, err := Apply(context.Background(), archHost(t), d2, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if _, _, err := Apply(context.Background(), archHost(t), empty, state2, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path2); !os.IsNotExist(err) { + t.Errorf("a file the mesh created, holding only its keys, was left behind") + } +} + +func TestAKeyNoLongerDeclaredGoesBackAndANewOneIsRemembered(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte(machinesOwn), 0o644) + first := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + _, state, err := Apply(context.Background(), archHost(t), first, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + second := parse(t, intoDecl(t, path, `{"registry-mirrors":["http://10.42.0.1:5000"]}`)) + if _, _, err := Apply(context.Background(), archHost(t), second, state, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + o := readObject(t, path) + if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" { + t.Errorf("a key the mesh stopped declaring was not given back: %v", o) + } + if fmt.Sprint(o["registry-mirrors"]) != "[http://10.42.0.1:5000]" { + t.Errorf("the newly declared key was not written: %v", o) + } +} + +func TestAFileThatIsNotAnObjectIsRefusedAndLeftAlone(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte("# not json at all\n"), 0o644) + d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil); err == nil { + t.Fatal("writing into a file that is not a JSON object was not refused") + } + raw, _ := os.ReadFile(path) + if string(raw) != "# not json at all\n" { + t.Errorf("a file the mesh could not write into was changed: %q", raw) + } +} + +func TestAFileWrittenIntoIsNeverHeldOnAnAdoptedNode(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte(machinesOwn), 0o644) + d := adopted(t, `{"taken":[],"untaken":{"networking":["networking.registry-trust"]}}`, + fmt.Sprintf(`{"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q}`, + path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + m := &machine{} + report, state := applyAdopted(t, d, store.State{}, m, t.TempDir()) + if got := outcomeOf(report, "networking.registry-trust").Action; got == "held" { + t.Fatal("a file written into was held, though it replaces nothing that was found") + } + if len(state.Held) != 0 { + t.Errorf("something was held: %+v", state.Held) + } + o := readObject(t, path) + if o["data-root"] != "/srv/docker" || fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" { + t.Errorf("the adopted node's file was not written into: %v", o) + } +} + +func TestAServiceIsReloadedNotRestartedForWhatItReloadsOn(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"trust","type":"file","path":%q,"into":"json","content":%q}, + {"id":"runtime","type":"service","unit":"docker.service","state":"running","reload-on":["trust"]} + ]}`, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + var commands []string + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, + recordingServices(&commands), nil, nil); err != nil { + t.Fatal(err) + } + joined := strings.Join(commands, "\n") + if !strings.Contains(joined, "systemctl reload docker.service") { + t.Errorf("the runtime was not reloaded; commands were %v", commands) + } + if strings.Contains(joined, "stop docker.service") || strings.Contains(joined, "restart docker.service") { + t.Errorf("the runtime was stopped, which stops every container on the machine; commands were %v", commands) + } +} + +// somethingElse is a declaration that no longer holds the file: only an unrelated directory. +func somethingElse(t *testing.T) *declaration.Declaration { + t.Helper() + return parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"other","type":"directory","path":%q} + ]}`, filepath.Join(t.TempDir(), "other"))) +} + +func TestAListIsAddedToNeverReplaced(t *testing.T) { + // The predecessor's own trusted registries are kept; the mesh adds its own and, undeclared, + // takes back only what it added (novox/hq ADR 0102). + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte(`{"insecure-registries":["192.0.2.7:5000","10.42.0.9:5000"]}`), 0o644) + // First the mesh's own member alone, so there is a record of this file. + first := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + _, state, err := Apply(context.Background(), archHost(t), first, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + // Now 10.42.0.9 is declared too, and was already the machine's: it is never the mesh's to remove. + d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000","10.42.0.9:5000"]}`)) + _, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := fmt.Sprint(readObject(t, path)["insecure-registries"]); got != "[192.0.2.7:5000 10.42.0.9:5000 10.42.0.1:5000]" { + t.Fatalf("the list after writing into it: %s", got) + } + rec, _ := state.Find("networking.registry-trust") + if added := rec.Into.Added["insecure-registries"]; len(added) != 1 || canonical(added[0]) != `"10.42.0.1:5000"` { + t.Errorf("recorded as added: %s", added) + } + + // The predecessor adds a member of its own: not the mesh's drift. + o := readObject(t, path) + o["insecure-registries"] = append(o["insecure-registries"].([]any), "198.51.100.3:5000") + raw, _ := json.Marshal(o) + _ = os.WriteFile(path, raw, 0o644) + report, state, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != "unchanged" { + t.Errorf("a member the machine added was taken for drift: %q", got) + } + + // Somebody takes the mesh's member out: that is drift, and it is put back. + o = readObject(t, path) + o["insecure-registries"] = []any{"192.0.2.7:5000", "10.42.0.9:5000", "198.51.100.3:5000"} + raw, _ = json.Marshal(o) + _ = os.WriteFile(path, raw, 0o644) + report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != "corrected" { + t.Errorf("the mesh's member removed by hand was %q", got) + } + + // Undeclared: only the member the mesh added goes. + if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + if got := fmt.Sprint(readObject(t, path)["insecure-registries"]); got != "[192.0.2.7:5000 10.42.0.9:5000 198.51.100.3:5000]" { + t.Errorf("undeclaring took more than the mesh added: %s", got) + } +} + +func TestAListTheMeshCreatedGoesWhenEmptied(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte(`{"data-root":"/srv/docker"}`), 0o644) + d := parse(t, intoDecl(t, path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + if o := readObject(t, path); fmt.Sprint(o) != "map[data-root:/srv/docker]" { + t.Errorf("the key the mesh created was not removed: %v", o) + } +} + +func TestAHoldFromAWholeFileDoesNotKeepOutAnIntoWrite(t *testing.T) { + // Declared whole before, the runtime's file was held; declared into now, it is written into. + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte(machinesOwn), 0o644) + known := store.State{Held: []store.Held{{ID: "networking.registry-trust", Module: "networking", + Kind: "file", Target: path}}} + d := adopted(t, `{"taken":[],"untaken":{"networking":["networking.registry-trust"]}}`, + fmt.Sprintf(`{"id":"networking.registry-trust","type":"file","path":%q,"into":"json","content":%q}`, + path, `{"insecure-registries":["10.42.0.1:5000"]}`)) + report, state := applyAdopted(t, d, known, &machine{}, t.TempDir()) + if got := outcomeOf(report, "networking.registry-trust").Action; got != "updated" { + t.Errorf("the file was %q, not written into", got) + } + if len(state.Held) != 0 { + t.Errorf("the old hold outlived the into declaration: %+v", state.Held) + } + if fmt.Sprint(readObject(t, path)["insecure-registries"]) != "[192.0.2.7:5000 10.42.0.1:5000]" { + t.Errorf("the mesh's member was not written in: %v", readObject(t, path)) + } +} + +func TestAWriteWithNoRecordOfItIsTheMeshsOwn(t *testing.T) { + // A host that wrote into the file and died before saving its state comes back with no record + // of it. What is there is then exactly what the mesh declares — and remembered as the + // machine's it would never be given back (novox/hq ADR 0102). + path := filepath.Join(t.TempDir(), "daemon.json") + _ = os.WriteFile(path, []byte(`{"data-root":"/srv/docker","insecure-registries":["192.0.2.7:5000"]}`), 0o644) + d := parse(t, intoDecl(t, path, `{"live-restore":true,"insecure-registries":["10.42.0.1:5000"]}`)) + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + + // The crash: the state was never saved, so the next apply knows nothing of this file. + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + rec, _ := state.Find("networking.registry-trust") + if _, asTheMachines := rec.Into.Before["live-restore"]; asTheMachines { + t.Error("the mesh's own key was remembered as the machine's") + } + if _, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared, nil, nil, nil); err != nil { + t.Fatal(err) + } + o := readObject(t, path) + if _, still := o["live-restore"]; still { + t.Errorf("undeclaring left the mesh's key behind: %v", o) + } + if fmt.Sprint(o["insecure-registries"]) != "[192.0.2.7:5000]" || o["data-root"] != "/srv/docker" { + t.Errorf("the machine did not get its file back: %v", o) + } +} diff --git a/internal/apply/opening.go b/internal/apply/opening.go new file mode 100644 index 0000000..216fc7c --- /dev/null +++ b/internal/apply/opening.go @@ -0,0 +1,138 @@ +package apply + +import ( + "context" + "fmt" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/firewall" + "github.com/novox/mesh-host/internal/store" +) + +// foundFirewall settles, before anything else in an apply, which firewall this node has — and on +// an adopted node that the mesh had converged, puts it back in force first (novox/hq ADR 0100). +// +// Only an adopted node asks. It is detected on every apply rather than remembered, so a firewall +// switched on after adoption is spoken to from the next reconcile; what is remembered is what was +// found first, and whether the mesh retired it. An unsupported firewall refuses the whole +// declaration: the mesh could neither open what it needs through it nor say what it would close. +func foundFirewall(ctx context.Context, d *declaration.Declaration, known *store.State, run Runner, + log func(string)) (firewall.Kind, error) { + if d.Adoption == nil { + return "", nil + } + rec := known.Firewall + if rec != nil && rec.DisabledByMesh && rec.Kind == string(firewall.UFW) { + // Returned to adopted: the found firewall is enabled again before the openings are + // converged through it, and the derived filter is gone with this declaration. + if err := firewall.Enable(ctx, run); err != nil { + return "", err + } + rec.DisabledByMesh = false + rec.Forward = nil + log(" enabled ufw again: this node is adopted, and the firewall found on it is in force") + } + kind, name, err := firewall.Detect(ctx, run) + if err != nil { + return "", err + } + if kind == firewall.Unsupported { + return "", fmt.Errorf( + "this machine is filtered by %s, and no host speaks that firewall yet. An adopted node "+ + "keeps the firewall it was found with, so the mesh could neither open what it needs "+ + "through it nor say what it would close; this declaration is refused whole", name) + } + if rec == nil { + rec = &store.FoundFirewall{Kind: string(kind), WasActive: kind == firewall.UFW, + FoundAt: time.Now().UTC()} + } else { + rec.Kind = string(kind) + rec.WasActive = rec.WasActive || kind == firewall.UFW + } + known.Firewall = rec + return kind, nil +} + +// retireFirewall disables the found firewall once a converged declaration has applied cleanly, +// which is when the mesh's derived filter has taken its place. Disabled, never flushed: its +// configuration stays on disk for a return to adopted, and the container runtime's rules are not +// its to take. +// +// Only a declaration from the mesh converges a node. A carried bundle never says a node is adopted +// — it cannot — so its silence is not the controller's word that the node was converged, and an +// adopted node re-applying its bundle keeps the firewall it was found with. +func retireFirewall(ctx context.Context, d *declaration.Declaration, origin string, known *store.State, + run Runner, log func(string)) error { + rec := known.Firewall + if origin != store.OriginDeclared || d.Adoption != nil || rec == nil || rec.Kind != string(firewall.UFW) || !rec.WasActive || + rec.DisabledByMesh { + return nil + } + // **Nothing is retired until what replaces it is in force** (novox/hq ADR 0100). The flip + // loads the mesh's derived filter in ufw's place; disabling ufw before that table is actually + // loaded — a filter module not assigned, or a unit that did not load — leaves the machine with + // no filter at all. + loaded, err := firewall.MeshTableLoaded(ctx, run) + if err != nil { + return err + } + if !loaded { + return fmt.Errorf("this node is converged and the mesh's own filter (table %s) is not loaded on "+ + "this machine, so ufw was left in force: retiring it would leave the machine filtering "+ + "nothing. Assign a filter module to this node, or return it to adopted", firewall.MeshTable) + } + if rec.Forward == nil { + // Recorded before ufw is touched: disabling it opens the forward policy, and a retry + // must know what it was (novox/hq ADR 0100). + rec.Forward = firewall.ForwardPolicies(ctx, run) + } + if err := firewall.Disable(ctx, run, rec.Forward); err != nil { + return err + } + rec.DisabledByMesh = true + log(" disabled ufw: this node is converged and filtered by the mesh; ufw's configuration is left on disk") + return nil +} + +// applyOpening makes one opening true through the firewall found here. +func applyOpening(ctx context.Context, o *declaration.Opening, run Runner, kind firewall.Kind) (Outcome, error) { + out := begin(o) + switch kind { + case firewall.None: + out.Action = "unchanged" + out.Detail = "no firewall found; nothing filters this port" + return out, nil + case firewall.UFW: + done, err := firewall.Converge(ctx, run, o) + if err != nil { + return out, err + } + out.Action = done.Action + out.Detail = "through ufw, marked " + firewall.Mark(o) + if done.SatisfiedBy != "" { + // ufw would take a rule differing only in its comment for the same one, so the + // mesh's is not added beside it (novox/hq ADR 0103). + out.Detail = "satisfied by a rule found in ufw (" + done.SatisfiedBy + + "); the mesh added nothing and will remove nothing" + } + return out, nil + } + return out, fmt.Errorf("no firewall is known for this node, so %s cannot be opened", o.Target()) +} + +// removeOpening deletes the rules the mesh marked for an opening no longer declared, and nothing +// the machine had before. +func removeOpening(ctx context.Context, a store.Applied, run Runner, rec *store.FoundFirewall) (string, string, error) { + if rec == nil || rec.Kind != string(firewall.UFW) { + return "forgotten", "no firewall held a rule for it", nil + } + n, err := firewall.Remove(ctx, run, a.ID) + if err != nil { + return "", "", err + } + if n == 0 { + return "forgotten", "ufw held no rule marked for it", nil + } + return "removed", fmt.Sprintf("%d ufw rule(s) marked as the mesh's deleted", n), nil +} diff --git a/internal/apply/opening_test.go b/internal/apply/opening_test.go new file mode 100644 index 0000000..5f976ee --- /dev/null +++ b/internal/apply/opening_test.go @@ -0,0 +1,492 @@ +package apply + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force; converging the +// node retires it by disabling it, and returning the node to adopted enables it again. + +type ufwMachine struct { + installed, active bool + rules []string + ruleset string + asked []string + + // forward is iptables' forward policy when set; empty is a machine without iptables. failP + // is how many -P calls fail before one succeeds. + forward string + failP int +} + +func (u *ufwMachine) iptables(args []string) (string, error) { + if len(args) == 3 && args[0] == "-P" { + if u.failP > 0 { + u.failP-- + return "", errors.New("iptables: resource temporarily unavailable") + } + u.forward = args[2] + return "", nil + } + return "-P FORWARD " + u.forward + "\n-A FORWARD -j DOCKER-USER\n", nil +} + +func (u *ufwMachine) run(_ context.Context, name string, args ...string) (string, error) { + u.asked = append(u.asked, name+" "+strings.Join(args, " ")) + switch name { + case "nft": + return u.ruleset, nil + case "iptables": + if u.forward == "" { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + return u.iptables(args) + case "ufw": + if !u.installed { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + default: + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + switch args[0] { + case "status": + if u.active { + return "Status: active\n", nil + } + return "Status: inactive\n", nil + case "show": + out := "Added user rules (see 'ufw status' for running firewall):\n" + for _, r := range u.rules { + out += "ufw " + r + "\n" + } + return out, nil + case "--force": + u.active = true + return "", nil + case "disable": + u.active = false + if u.forward != "" { + u.forward = "ACCEPT" // as measured: ufw disable opens the forward policy + } + return "", nil + case "delete": + want := strings.Join(args[1:], " ") + for i, r := range u.rules { + if strings.ReplaceAll(r, "'", "") == want { + u.rules = append(u.rules[:i], u.rules[i+1:]...) + return "", nil + } + } + return "", errors.New("Could not delete non-existent rule") + default: + // Printed back the way it was given, with the comment quoted as ufw does. + line := strings.Join(args[:len(args)-1], " ") + " '" + args[len(args)-1] + "'" + u.rules = append(u.rules, line) + return "", nil + } +} + +func (u *ufwMachine) index(prefix string) int { + for i, a := range u.asked { + if strings.HasPrefix(a, prefix) { + return i + } + } + return -1 +} + +const busOpening = `{"id":"adoption.opening-tcp-5671-incoming","type":"opening","port":5671,"protocol":"tcp","from":"everywhere","path":"incoming"}` + +func withConf(dir string) string { + return `{"id":"x.conf","type":"file","path":"` + filepath.Join(dir, "x.conf") + `","content":"x\n"}` +} + +func applyWith(t *testing.T, d *declaration.Declaration, known store.State, run Runner) (Report, store.State, error) { + t.Helper() + return ApplyKeeping(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil, + KeepIn(t.TempDir())) +} + +func TestAnOpeningOnAMachineWithNoFirewallChangesNothing(t *testing.T) { + dir := t.TempDir() + u := &ufwMachine{} + report, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + o := outcomeOf(report, "adoption.opening-tcp-5671-incoming") + if o.Action != "unchanged" || !strings.Contains(o.Detail, "nothing filters this port") { + t.Errorf("an opening with no firewall: %+v", o) + } + if state.Firewall == nil || state.Firewall.Kind != "none" { + t.Errorf("the firewall found was not recorded: %+v", state.Firewall) + } +} + +func TestAnUnsupportedFirewallRefusesTheWholeDeclaration(t *testing.T) { + dir := t.TempDir() + u := &ufwMachine{ruleset: "table inet filter {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t}\n}\n"} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err == nil || !strings.Contains(err.Error(), "no host speaks that firewall") { + t.Fatalf("an unsupported firewall was not refused: %v", err) + } + if _, statErr := os.Stat(filepath.Join(dir, "x.conf")); !errors.Is(statErr, os.ErrNotExist) { + t.Error("part of a refused declaration was applied") + } + if state.Firewall != nil { + t.Errorf("an unsupported firewall was recorded: %+v", state.Firewall) + } +} + +func TestConvergingRetiresTheFoundFirewallAndReturningRestoresIt(t *testing.T) { + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} + + // Adopted: the opening goes through ufw. + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + if len(u.rules) != 2 || !u.active { + t.Fatalf("adopted: rules %v, active %v", u.rules, u.active) + } + if state.Firewall == nil || state.Firewall.Kind != "ufw" || !state.Firewall.WasActive { + t.Fatalf("adopted: firewall recorded as %+v", state.Firewall) + } + + // Converged: the derived filter is loaded, the opening's rule goes, and only then is ufw + // disabled — never reset. + u.asked = nil + u.ruleset = "table inet mesh\n" + converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) + _, state, err = applyWith(t, converged, state, u.run) + if err != nil { + t.Fatal(err) + } + if u.active || !state.Firewall.DisabledByMesh { + t.Fatalf("converged: ufw still active (%v) or not recorded as retired (%+v)", u.active, state.Firewall) + } + if len(u.rules) != 1 || u.rules[0] != "allow 22/tcp" { + t.Errorf("converged: the operator's rules were touched, or the mesh's left: %v", u.rules) + } + // What protected the adopted node goes last: after the derived filter applied and ufw was + // retired (novox/hq ADR 0103). + if del, dis := u.index("ufw delete"), u.index("ufw disable"); dis < 0 || del < dis { + t.Errorf("converged: the opening was removed before ufw was retired: %v", u.asked) + } + for _, a := range u.asked { + if strings.Contains(a, "reset") { + t.Errorf("converged: ufw was reset: %s", a) + } + } + + // Converged again: nothing more to retire. + u.asked = nil + if _, state, err = applyWith(t, converged, state, u.run); err != nil { + t.Fatal(err) + } + if u.index("ufw") >= 0 { + t.Errorf("a converged node kept talking to a retired ufw: %v", u.asked) + } + + // Returned to adopted: ufw is enabled before the opening is converged through it. + u.asked = nil + _, state, err = applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), state, u.run) + if err != nil { + t.Fatal(err) + } + if !u.active || state.Firewall.DisabledByMesh { + t.Fatalf("returned: ufw active %v, record %+v", u.active, state.Firewall) + } + if en, add := u.index("ufw --force enable"), u.index("ufw allow"); en < 0 || add < en { + t.Errorf("returned: ufw was not enabled before the opening was added: %v", u.asked) + } + if len(u.rules) != 2 { + t.Errorf("returned: the opening was not converged again: %v", u.rules) + } +} + +func TestAConvergedNodeThatWasNeverAdoptedNeverAsksAboutAFirewall(t *testing.T) { + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true} + if _, _, err := applyWith(t, parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`), store.State{}, u.run); err != nil { + t.Fatal(err) + } + if len(u.asked) != 0 { + t.Errorf("a converged apply asked the machine about its firewall: %v", u.asked) + } +} + +func TestAnOpeningOnAConvergedNodeIsRefused(t *testing.T) { + if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` + busOpening + `]}`)); err == nil { + t.Error("an opening was accepted on a node the declaration does not say is adopted") + } +} + +func TestACarriedApplyOnAnAdoptedNodeLeavesItsFirewallInForce(t *testing.T) { + // The bundle, re-applied by the installer or the one-shot CLI, never says a node is adopted. + // That is not the controller converging it, so ufw must stay enabled (novox/hq ADR 0100). + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + u.asked = nil + carried := parse(t, `{"declaration":1,"resources":[`+withConf(filepath.Join(dir, "bundle"))+`]}`) + _, state, err = ApplyKeeping(context.Background(), archHost(t), carried, state, store.OriginCarried, + u.run, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if !u.active || state.Firewall.DisabledByMesh || u.index("ufw disable") >= 0 { + t.Fatalf("a carried apply retired the found firewall: active %v, record %+v, asked %v", + u.active, state.Firewall, u.asked) + } +} + +func TestAFlipThatFailsKeepsTheGuardAndTheOpenings(t *testing.T) { + // Converging a node removes its openings and its guard only once everything else applied and + // the found firewall is retired. A flip that fails part-way keeps them, so the store is never + // left unguarded behind a filter that did not load (novox/hq ADR 0103). + dir := t.TempDir() + guard := filepath.Join(dir, "guard.nft") + guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}` + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+guardFile+","+withConf(dir)), + store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + + // The derived filter cannot be written: its path is under a file. + blocked := filepath.Join(dir, "not-a-directory") + if err := os.WriteFile(blocked, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + filter := `{"id":"nftables.config","type":"file","path":"` + filepath.Join(blocked, "nftables.conf") + `","content":"table inet mesh {}\n"}` + converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`,`+filter+`]}`) + u.asked = nil + u.ruleset = "table inet mesh\n" // what the filter's unit loads once the file is written + _, state, err = applyWith(t, converged, state, u.run) + if err == nil { + t.Fatal("the failing flip reported success") + } + if !u.active || u.index("ufw disable") >= 0 { + t.Errorf("the found firewall was retired by a flip that failed: %v", u.asked) + } + if len(u.rules) != 2 || u.index("ufw delete") >= 0 { + t.Errorf("the opening was removed by a flip that failed: %v", u.rules) + } + if _, statErr := os.Stat(guard); statErr != nil { + t.Errorf("the guard was removed by a flip that failed: %v", statErr) + } + for _, id := range []string{"adoption.guard", "adoption.opening-tcp-5671-incoming"} { + if _, ok := state.Find(id); !ok { + t.Errorf("%s was forgotten, so the next flip would never remove it", id) + } + } + + // Fixed, the next flip completes: filter, retire, and only then the guard and the openings. + if err := os.Remove(blocked); err != nil { + t.Fatal(err) + } + u.asked = nil + _, state, err = applyWith(t, converged, state, u.run) + if err != nil { + t.Fatal(err) + } + if u.active || len(u.rules) != 1 { + t.Errorf("the completed flip left ufw active %v, rules %v", u.active, u.rules) + } + if _, statErr := os.Stat(guard); !errors.Is(statErr, os.ErrNotExist) { + t.Errorf("the guard outlived the completed flip: %v", statErr) + } + if _, ok := state.Find("adoption.guard"); ok { + t.Error("the guard is still recorded after the completed flip") + } +} + +func TestAnOpeningAFoundRuleAnswersIsReportedSatisfied(t *testing.T) { + // novox/hq ADR 0103: the mesh adds nothing beside a rule ufw would take for the same one. + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 5671/tcp"}} + report, _, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + o := outcomeOf(report, "adoption.opening-tcp-5671-incoming") + if o.Action != "unchanged" || !strings.Contains(o.Detail, "satisfied by a rule found in ufw (allow 5671/tcp)") { + t.Errorf("the opening was not reported satisfied: %+v", o) + } + if len(u.rules) != 2 || u.index("ufw allow") >= 0 { + t.Errorf("a rule was added beside the found one: %v", u.rules) + } +} + +// Defends novox/hq ADR 0103: returned to adopted, the guard is up before the derived filter's +// orphans go, and stays up if removing them fails. +func TestReturningToAdoptedLoadsTheGuardBeforeRemovingTheFilter(t *testing.T) { + dir := t.TempDir() + guard := filepath.Join(dir, "guard.nft") + guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}` + for _, stopFails := range []bool{false, true} { + _ = os.Remove(guard) + guardUpAtStop := false + run := func(_ context.Context, name string, args ...string) (string, error) { + if name != "systemctl" { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + switch args[0] { + case "show": + return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\n", nil + case "stop": + _, err := os.Stat(guard) + guardUpAtStop = err == nil + if stopFails { + return "", errors.New("the filter would not stop") + } + } + return "", nil + } + converged := store.State{Resources: []store.Applied{ + {ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run) + if !guardUpAtStop { + t.Errorf("stop fails %v: the derived filter was stopped before the guard was written", stopFails) + } + if stopFails { + if err == nil { + t.Error("a failed removal was not reported") + } + if _, statErr := os.Stat(guard); statErr != nil { + t.Error("the guard is not up after the filter's removal failed") + } + if _, ok := state.Find("adoption.guard"); !ok { + t.Error("the guard applied before the failure was not recorded") + } + if _, still := state.Find("nftables.load"); !still { + t.Error("the filter that would not stop was forgotten, so nothing would stop it later") + } + } else if err != nil { + t.Fatal(err) + } + } +} + +func TestAStaleOpeningOnAnAdoptedNodeIsRemovedAsAnyOrphan(t *testing.T) { + // Only the flip defers the adoption's own orphans; an adopted node drops a stale opening at + // once, before what replaces it is applied. + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + u.asked = nil + other := `{"id":"adoption.opening-tcp-5000-incoming","type":"opening","port":5000,"protocol":"tcp","from":"everywhere","path":"incoming"}` + if _, _, err = applyWith(t, adopted(t, `{"taken":[]}`, other+","+withConf(dir)), state, u.run); err != nil { + t.Fatal(err) + } + if del, add := u.index("ufw delete"), u.index("ufw allow"); del < 0 || add < 0 || del > add { + t.Errorf("the stale opening was not removed before the new one was added: %v", u.asked) + } +} + +func TestARetiredFirewallRetriedStillPutsBackTheForwardPolicy(t *testing.T) { + // The forward policy is recorded before ufw is disabled, so a retirement that failed after + // the disable restores what the machine had, not what the disable left (novox/hq ADR 0100). + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, forward: "DROP", failP: 1, ruleset: "table inet mesh\n"} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) + if _, state, err = applyWith(t, converged, state, u.run); err == nil { + t.Fatal("the failed restore was not reported") + } + if u.active || u.forward != "ACCEPT" || state.Firewall.Forward["iptables"] != "DROP" { + t.Fatalf("after the failed attempt: active %v, forward %s, recorded %+v", u.active, u.forward, state.Firewall) + } + if _, state, err = applyWith(t, converged, state, u.run); err != nil { + t.Fatal(err) + } + if u.forward != "DROP" || !state.Firewall.DisabledByMesh { + t.Errorf("the retry did not put the forward policy back: %s, %+v", u.forward, state.Firewall) + } +} + +func TestAGuardThatFailsLeavesTheDerivedFilterInForce(t *testing.T) { + // Returning to adopted: if the guard cannot be raised, the filter it replaces must not be + // stopped — its stop deletes the mesh's table, and the node would have neither (novox/hq ADR 0103). + dir := t.TempDir() + blocked := filepath.Join(dir, "not-a-directory") + if err := os.WriteFile(blocked, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + guardFile := `{"id":"adoption.guard","type":"file","path":"` + filepath.Join(blocked, "guard.nft") + + `","content":"table inet mesh_guard {}\n"}` + stopped := false + run := func(_ context.Context, name string, args ...string) (string, error) { + if name != "systemctl" { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + if args[0] == "show" { + return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\n", nil + } + if args[0] == "stop" { + stopped = true + } + return "", nil + } + converged := store.State{Resources: []store.Applied{ + {ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run) + if err == nil { + t.Fatal("a guard that could not be written reported success") + } + if stopped { + t.Error("the derived filter was stopped though the guard is not up") + } + if _, gone := state.Find("nftables.load"); !gone { + t.Error("the filter was forgotten, so nothing would ever stop it") + } +} + +func TestUfwIsNotRetiredUntilTheMeshsOwnFilterIsLoaded(t *testing.T) { + // The flip retires the found firewall because the mesh's derived filter takes its place. If + // that table is not loaded, retiring would leave the machine filtering nothing (novox/hq ADR 0100). + dir := t.TempDir() + u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} + _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) + if err != nil { + t.Fatal(err) + } + converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) + _, state, err = applyWith(t, converged, state, u.run) + if err == nil || !strings.Contains(err.Error(), "table inet mesh") { + t.Fatalf("ufw was retired with nothing in its place: %v", err) + } + if !u.active || state.Firewall.DisabledByMesh { + t.Errorf("ufw was disabled: active %v, %+v", u.active, state.Firewall) + } + + // Once the table is loaded, the same declaration retires it. + u.ruleset = "table inet mesh\n" + if _, state, err = applyWith(t, converged, state, u.run); err != nil { + t.Fatal(err) + } + if u.active || !state.Firewall.DisabledByMesh { + t.Errorf("ufw was not retired once the mesh's filter was loaded: active %v, %+v", u.active, state.Firewall) + } +} diff --git a/internal/apply/process.go b/internal/apply/process.go index 686b7f1..d9956e7 100644 --- a/internal/apply/process.go +++ b/internal/apply/process.go @@ -32,8 +32,9 @@ import ( // owners end up disagreeing about one path. const daemonRoot = "/var/lib/mesh/daemons" -// unitDir is where the mesh writes the units it owns. -const unitDir = "/etc/systemd/system" +// unitDir is where the mesh writes the units it owns. A variable only so a test can point it at a +// directory of its own. +var unitDir = "/etc/systemd/system" func applyProcess(ctx context.Context, r *declaration.Process, run Runner, changed map[string]bool, previous store.Applied) (Outcome, error) { diff --git a/internal/apply/schedule.go b/internal/apply/schedule.go index 3057559..ccede49 100644 --- a/internal/apply/schedule.go +++ b/internal/apply/schedule.go @@ -86,7 +86,11 @@ func NewScheduler(clock Clock, run Runner, log func(string)) *Scheduler { // A job whose declaration is unchanged keeps its place in the cadence — its next due time and // whether a run is in flight — so an ordinary reconcile every few minutes does not keep resetting // the clock out from under a schedule and prevent it ever firing. -func (s *Scheduler) Sync(d *declaration.Declaration) { +// held is the ids this node holds as found — what an adopted node keeps until its module is taken +// (novox/hq ADR 0100). A step of a module not yet taken is not armed: run on its cadence it would +// work on the predecessor's data, under the predecessor's service, which is the one thing an +// adopted node must not do. Nil on a converged node, where nothing is held. +func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) { s.mu.Lock() defer s.mu.Unlock() @@ -96,6 +100,14 @@ func (s *Scheduler) Sync(d *declaration.Declaration) { if !ok || c.Schedule == "" { continue } + if module, untaken := d.Adoption.UntakenModuleOf(c.Identity()); untaken || held[c.Identity()] { + if module == "" { + module = "its module" + } + s.log(fmt.Sprintf("scheduled step %s: not armed while %s is held as found on this node", + c.Identity(), module)) + continue + } cron, err := declaration.ParseCron(c.Schedule) if err != nil { // The declaration parser already refused a malformed cron before this runs, so a diff --git a/internal/apply/schedule_test.go b/internal/apply/schedule_test.go index fcca78c..715b0db 100644 --- a/internal/apply/schedule_test.go +++ b/internal/apply/schedule_test.go @@ -244,7 +244,7 @@ func TestAScheduledStepRunsWhenDueAndNotBefore(t *testing.T) { d := &declaration.Declaration{Version: 1, Resources: []declaration.Resource{ scheduledContainer(t, "* * * * *"), // every minute; next due 12:01:00 }} - s.Sync(d) + s.Sync(d, nil) // Not yet due: 12:00:45 is before 12:01:00, so nothing runs. s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 0, 45, 0, time.UTC)) @@ -306,7 +306,7 @@ func TestAFailedRunIsRecordedAndDoesNotFailAnything(t *testing.T) { s := NewScheduler(clock, run, log) s.Sync(&declaration.Declaration{Version: 1, Resources: []declaration.Resource{ scheduledContainer(t, "* * * * *"), - }}) + }}, nil) // Fire a run that exits non-zero. Advance returns nothing — there is no error to fail an apply, // because the run happens outside any apply and outside the store. @@ -371,7 +371,7 @@ func TestASlowRunSkipsTheNextDueRunRatherThanStacking(t *testing.T) { s := NewScheduler(clock, run, log) s.Sync(&declaration.Declaration{Version: 1, Resources: []declaration.Resource{ scheduledContainer(t, "* * * * *"), // every minute - }}) + }}, nil) // First occurrence: 12:01 is due — starts a run that blocks in the runner. s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 1, 5, 0, time.UTC)) @@ -414,7 +414,7 @@ func TestSyncForgetsAScheduleTheDeclarationNoLongerNames(t *testing.T) { s.Sync(&declaration.Declaration{Version: 1, Resources: []declaration.Resource{ scheduledContainer(t, "* * * * *"), - }}) + }}, nil) s.mu.Lock() have := len(s.jobs) s.mu.Unlock() @@ -427,10 +427,55 @@ func TestSyncForgetsAScheduleTheDeclarationNoLongerNames(t *testing.T) { parseTrusted(t, `{"declaration":1,"resources":[ {"id":"web","type":"container","name":"web","image":"`+pinned+`"} ]}`).Resources[0], - }}) + }}, nil) s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 5, 5, 0, time.UTC)) s.Wait() if n := rec.fireCount(); n != 0 { t.Errorf("a schedule the declaration no longer names still fired (%d run(s))", n) } } + +// Defends novox/hq ADR 0103: a scheduled step of a module not yet taken is not armed — run on its +// cadence it would work on the predecessor's data. +func TestAScheduledStepOfAnUntakenModuleIsNotArmed(t *testing.T) { + clock := &fixedClock{now: time.Date(2026, 9, 7, 12, 0, 30, 0, time.UTC)} + rec := &recordingRun{} + var said []string + s := NewScheduler(clock, rec.run, func(line string) { said = append(said, line) }) + + d := &declaration.Declaration{Version: 1, + Adoption: &declaration.Adoption{Untaken: map[string][]string{"backups": {"sync"}}}, + Resources: []declaration.Resource{ + scheduledContainer(t, "* * * * *"), + }} + s.Sync(d, nil) + s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 1, 5, 0, time.UTC)) + s.Wait() + if rec.fireCount() != 0 { + t.Errorf("a held module's scheduled step ran %d time(s)", rec.fireCount()) + } + if len(said) == 0 || !strings.Contains(said[0], "held as found") { + t.Errorf("nothing said why the step was not armed: %v", said) + } + + // Held by id — a step whose own container was found on the machine. + s2 := NewScheduler(clock, rec.run, nil) + s2.Sync(&declaration.Declaration{Version: 1, Resources: []declaration.Resource{ + scheduledContainer(t, "* * * * *"), + }}, map[string]bool{"sync": true}) + s2.Advance(context.Background(), time.Date(2026, 9, 7, 12, 1, 5, 0, time.UTC)) + s2.Wait() + if rec.fireCount() != 0 { + t.Errorf("a held scheduled step ran %d time(s)", rec.fireCount()) + } + + // Taken: the same step is armed and runs. + taken := &declaration.Declaration{Version: 1, Adoption: &declaration.Adoption{Taken: []string{"backups"}}, + Resources: []declaration.Resource{scheduledContainer(t, "* * * * *")}} + s.Sync(taken, nil) + s.Advance(context.Background(), time.Date(2026, 9, 7, 12, 2, 5, 0, time.UTC)) + s.Wait() + if rec.fireCount() == 0 { + t.Error("a taken module's scheduled step never ran") + } +} diff --git a/internal/bootstrap/adopted.go b/internal/bootstrap/adopted.go new file mode 100644 index 0000000..ed7207c --- /dev/null +++ b/internal/bootstrap/adopted.go @@ -0,0 +1,202 @@ +package bootstrap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + + "github.com/novox/mesh-host/internal/declaration" +) + +// What an adopted genesis changes about the foundation (novox/hq ADR 0100). +// +// **The firewall found on the machine stays in force.** The foundation's own filter drops by +// default, and every base chain at a hook runs; an accept ends only its own chain and a drop in any +// is final — so loading it would close whatever the machine serves. On an adopted machine it is +// not loaded. Its duty, the store never reachable from outside, passes to the mesh's guard: a table +// of the mesh's own that only refuses, and only the foundation's own ports, which genesis has just +// checked free — so it cannot close anything the machine serves. + +// The guard, as the controller declares it: the same ids, paths and text, so the first push +// finds it already there and takes it over unchanged. +const ( + guardID = declaration.AdoptionPrefix + "guard" + guardUnitID = declaration.AdoptionPrefix + "guard-unit" + guardRunningID = declaration.AdoptionPrefix + "guard-running" + guardPath = "/etc/mesh/guard.nft" + guardUnit = "mesh-guard.service" + guardUnitPath = "/etc/systemd/system/" + guardUnit +) + +// AsGuard renders the mesh's refusal-only table for the given machine ports. It passes everything +// by default; it refuses the ports except from the machine itself — its loopback and the container +// runtime's own networks — and from the private network, known by the interface a packet arrives +// on and never by its source address; at prerouting, ahead of the runtime's destination +// translation, in the inet family so both address families. It matches only packets addressed to +// this machine: what the machine routes for others is never its business (novox/hq ADR 0103). +// +// Character for character the controller's (mesh-controller internal/catalogue AsGuard); a test +// on each side holds its copy to the same golden text. +func AsGuard(ports []int) string { + sorted := append([]int{}, ports...) + sort.Ints(sorted) + listed := make([]string, len(sorted)) + for i, p := range sorted { + listed[i] = strconv.Itoa(p) + } + var b strings.Builder + b.WriteString("table inet mesh_guard {}\n") + b.WriteString("delete table inet mesh_guard\n") + b.WriteString("table inet mesh_guard {\n") + b.WriteString("\tchain prerouting {\n") + b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n") + fmt.Fprintf(&b, "\t\tfib daddr type local iifname != \"lo\" iifname != \"docker0\" iifname != \"br-*\" "+ + "iifname != \"mesh0\" tcp dport { %s } drop\n", strings.Join(listed, ", ")) + b.WriteString("\t}\n") + b.WriteString("}\n") + return b.String() +} + +// guardUnitText is the unit that loads the guard. Stopping it deletes only its own table — never a +// flush, which would take the container runtime's rules and the found firewall with it. +func guardUnitText() string { + return "[Unit]\n" + + "Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" + + "DefaultDependencies=no\n" + + "Wants=network-pre.target\n" + + "Before=network-pre.target shutdown.target\n" + + "Conflicts=shutdown.target\n" + + "\n" + + "[Service]\n" + + "Type=oneshot\n" + + "RemainAfterExit=yes\n" + + "ExecStart=nft -f " + guardPath + "\n" + + "ExecReload=nft -f " + guardPath + "\n" + + "ExecStop=nft delete table inet mesh_guard\n" + + "\n" + + "[Install]\n" + + "WantedBy=multi-user.target\n" +} + +// guardResources are the guard as three resources of kinds the host already has. +func guardResources(ports []int) []map[string]any { + return []map[string]any{ + {"id": guardID, "type": "file", "path": guardPath, "content": AsGuard(ports), "mode": "0644"}, + {"id": guardUnitID, "type": "file", "path": guardUnitPath, "content": guardUnitText(), "mode": "0644"}, + // A changed table is reloaded — the unit's ExecReload loads it in one transaction, so the + // ports are never unguarded — and only a changed unit restarts it. As the controller + // declares it, so the first push finds nothing different. + {"id": guardRunningID, "type": "service", "unit": guardUnit, "state": "running", + "boot": "enabled", "reload-on": []any{guardID}, "restart-on": []any{guardUnitID}}, + } +} + +// guardAfter is where the guard goes: once the container runtime runs, before anything publishes +// a port. +const guardAfter = "container-runtime-running" + +// AdoptedRewrite says what RewriteAdopted did. +type AdoptedRewrite struct { + Removed []string + Guarded []int +} + +// RewriteAdopted makes the produced bundle one for an adopted machine: the foundation's own filter +// taken out, and the mesh's guard put in its place, guarding on this node the store's port, the +// broker's management port and the broker's plaintext port. The last is published on every +// interface and the foundation's filter admits it from the private network only, so a found +// firewall that filters only incoming traffic would leave it reachable from anywhere (novox/hq ADR +// 0103). Every one is a port of a module genesis takes. The nftables package stays: the guard is loaded with it, and +// installing a package loads no table. Openings are not the bundle's — the first push declares +// them, once there is a controller to derive them. +func RewriteAdopted(r *Rewritten, p FoundationPorts) (AdoptedRewrite, error) { + var out AdoptedRewrite + p = p.orDefaults() + bundle := r.Bundle + var err error + for _, id := range []string{"base-filter-loaded", "base-filter"} { + if !r.declares(id) { + continue + } + if bundle, err = removeResource(bundle, id); err != nil { + return out, err + } + out.Removed = append(out.Removed, id) + } + + out.Guarded = []int{p.Store, p.Management, p.AMQP} + var text bytes.Buffer + text.WriteString(",\n // The mesh's guard (novox/hq ADR 0100): this machine is adopted, so its own firewall\n" + + " // stays in force and the foundation's filter is not loaded. The guard only refuses: the\n" + + " // store's port and the broker's management and plaintext ports, except from the machine and\n" + + " // the private network.") + for _, res := range guardResources(out.Guarded) { + var one bytes.Buffer + enc := json.NewEncoder(&one) + enc.SetEscapeHTML(false) + if err := enc.Encode(res); err != nil { + return out, err + } + text.WriteString("\n ") + text.Write(bytes.TrimSpace(one.Bytes())) + text.WriteString(",") + } + insert := bytes.TrimSuffix(text.Bytes(), []byte(",")) + + _, _, to, err := resourceAt(bundle, guardAfter) + if err != nil { + return out, fmt.Errorf("the guard goes after %q, and %w", guardAfter, err) + } + rest := bundle[to:] + joined := make([]byte, 0, len(bundle)+len(insert)) + joined = append(joined, bundle[:to]...) + joined = append(joined, insert...) + // What followed the resource — its own comma, or the end of the list — now follows the guard. + if trimmed := bytes.TrimLeft(rest, " \t\r\n"); len(trimmed) > 0 && trimmed[0] != ',' && trimmed[0] != ']' { + return out, fmt.Errorf("the bundle does not separate %q from what follows it the way a list does", guardAfter) + } + joined = append(joined, rest...) + + parsed, err := declaration.ParseFileTrusted(joined) + if err != nil { + return out, fmt.Errorf("the bundle stopped being a declaration once it was made an adopted one, which is this installer's fault: %w", err) + } + r.Bundle, r.Declaration, r.Resources = joined, parsed, len(parsed.Resources) + return out, nil +} + +// declares is whether the produced bundle names a resource. +func (r Rewritten) declares(id string) bool { + for _, res := range r.Declaration.Resources { + if res.Identity() == id { + return true + } + } + return false +} + +// genesisTakes are the modules an adopted genesis takes as it installs them: the foundation's and +// the mesh's own, whose names genesis checked free, so taking them replaces nothing a predecessor +// ran. The private network is not among them — it rewrites the machine's hosts file and the +// container runtime's configuration whole — and neither is anything the operator installs later. +var genesisTakes = map[string]bool{ + RegistryModule: true, ControlPlaneModule: true, BuilderModule: true, + "postgres": true, "lavinmq": true, "mesh-vault": true, "mesh-catalog": true, +} + +// takeIfAdopted takes one of genesis's own modules on an adopted node, once it is assigned and +// before the push that raises it. +func takeIfAdopted(ctx context.Context, o Options, control controlPlane, module string, say func(string)) error { + if !o.Adopted || !genesisTakes[module] { + return nil + } + if _, err := control.tell(ctx, "take", o.Node, module); err != nil { + return err + } + say(" taken " + module + " on " + o.Node + " — the mesh's own, its name checked free") + return nil +} diff --git a/internal/bootstrap/adopted_test.go b/internal/bootstrap/adopted_test.go new file mode 100644 index 0000000..f52e0e1 --- /dev/null +++ b/internal/bootstrap/adopted_test.go @@ -0,0 +1,233 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Defends novox/hq ADR 0100: an adopted genesis loads no table that drops by default or holds an +// accept; the mesh guards its own ports in a table that only refuses; and genesis takes the mesh's +// own modules as it installs them, and nothing else. + +// The same golden text the controller's test holds its AsGuard to. +const goldenGuard = `table inet mesh_guard {} +delete table inet mesh_guard +table inet mesh_guard { + chain prerouting { + type filter hook prerouting priority raw; policy accept; + fib daddr type local iifname != "lo" iifname != "docker0" iifname != "br-*" iifname != "mesh0" tcp dport { 5432, 15672 } drop + } +} +` + +func TestTheGuardIsExactlyThisTable(t *testing.T) { + if got := AsGuard([]int{15672, 5432}); got != goldenGuard { + t.Fatalf("the guard changed:\n%s", got) + } +} + +// The same golden unit the controller's test holds its guard unit to. It is loaded before the +// network is up, so it carries no default dependencies, and it is stopped only at shutdown. +const goldenGuardUnit = `[Unit] +Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100) +DefaultDependencies=no +Wants=network-pre.target +Before=network-pre.target shutdown.target +Conflicts=shutdown.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=nft -f /etc/mesh/guard.nft +ExecReload=nft -f /etc/mesh/guard.nft +ExecStop=nft delete table inet mesh_guard + +[Install] +WantedBy=multi-user.target +` + +func TestTheGuardUnitIsExactlyThisUnit(t *testing.T) { + if got := guardUnitText(); got != goldenGuardUnit { + t.Fatalf("the guard's unit changed:\n%s", got) + } +} + +func TestTheGuardRefusesOnlyWhatIsAddressedToThisMachine(t *testing.T) { + // A machine that routes for others — a predecessor's private-network hub — must not have a + // packet for another machine's database port refused (novox/hq ADR 0103). + for _, line := range strings.Split(AsGuard([]int{5432}), "\n") { + if strings.Contains(line, " drop") && !strings.HasPrefix(strings.TrimSpace(line), "fib daddr type local ") { + t.Errorf("a refusal matches packets not addressed to this machine: %q", line) + } + } +} + +func TestAnAdoptedBundleLoadsNoDroppingTableAndExactlyTheGuard(t *testing.T) { + r := producedBundle(t) + p := FoundationPorts{Store: 5433, Management: 15673, AMQP: 5773} + if _, err := RewritePorts(&r, p, ""); err != nil { + t.Fatal(err) + } + got, err := RewriteAdopted(&r, p) + if err != nil { + t.Fatal(err) + } + if strings.Join(got.Removed, ",") != "base-filter-loaded,base-filter" { + t.Errorf("removed %v", got.Removed) + } + at := map[string]int{} + var guards []*declaration.File + for i, res := range r.Declaration.Resources { + at[res.Identity()] = i + if f, ok := res.(*declaration.File); ok { + if strings.Contains(f.Content, "policy drop") || strings.Contains(f.Content, " accept\n") && + !strings.Contains(f.Content, "policy accept") { + t.Errorf("%s loads a table that drops or accepts: %q", f.ID, f.Content) + } + if f.Path == guardPath { + guards = append(guards, f) + } + } + if s, ok := res.(*declaration.Service); ok && s.Unit == "nftables.service" { + t.Errorf("the foundation's filter is still loaded by %s", s.ID) + } + } + if len(guards) != 1 { + t.Fatalf("%d guard table(s)", len(guards)) + } + // The store's, the broker's plaintext and its management port: each one the filter admits + // from the private network only (novox/hq ADR 0103). + if !strings.Contains(guards[0].Content, "tcp dport { 5433, 5773, 15673 } drop") { + t.Errorf("the guard does not refuse this node's ports: %s", guards[0].Content) + } + if strings.Count(guards[0].Content, "accept") != 1 || !strings.Contains(guards[0].Content, "policy accept") { + t.Errorf("the guard holds an accept of its own: %s", guards[0].Content) + } + for _, id := range []string{guardID, guardUnitID, guardRunningID} { + if _, ok := at[id]; !ok { + t.Errorf("the bundle has no %s", id) + } + } + if !(at["container-runtime-running"] < at[guardID] && at[guardRunningID] < at["store"]) { + t.Errorf("the guard is not between the runtime and the store: %v", at) + } + if _, kept := at["base-filter-package"]; !kept { + t.Error("nft, which loads the guard, is no longer installed") + } + unit := r.Declaration.Resources[at[guardRunningID]].(*declaration.Service) + // A changed table is reloaded, never restarted: a restart deletes the table before loading + // it again, leaving the ports unguarded in between. + if unit.Unit != guardUnit || unit.State != "running" || strings.Join(unit.RestartOn, ",") != guardUnitID || + strings.Join(unit.ReloadOn, ",") != guardID { + t.Errorf("the guard's service: %+v", unit) + } + stop := r.Declaration.Resources[at[guardUnitID]].(*declaration.File).Content + if !strings.Contains(stop, "ExecStop=nft delete table inet mesh_guard") || strings.Contains(stop, "flush") { + t.Errorf("stopping the guard does not delete only its own table: %s", stop) + } +} + +func TestAConvergedBundleIsNotMadeAnAdoptedOne(t *testing.T) { + // The converged genesis keeps the foundation's filter, byte for byte (novox/hq ADR 0088). + r := producedBundle(t) + for _, res := range r.Declaration.Resources { + if strings.HasPrefix(res.Identity(), declaration.AdoptionPrefix) { + t.Errorf("a converged bundle carries %s", res.Identity()) + } + } + if !r.declares("base-filter-loaded") { + t.Error("a converged bundle lost its filter") + } +} + +func TestAnAdoptedGenesisTakesTheMeshsOwnModulesBeforePushingThem(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + for _, c := range []struct { + module string + takes bool + }{{RegistryModule, true}, {ControlPlaneModule, true}, {BuilderModule, true}, {"gitea", false}} { + rec := &controlRecorder{settings: map[string]string{}} + control := controlPlane{container: "temp-mesh-controller", run: rec.run, timeout: time.Second} + o := Options{Node: "anchor", Adopted: true, Wait: time.Second} + if _, err := installModule(context.Background(), o, control, c.module, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + assign, take, push := rec.index("assign anchor "+c.module), rec.index("take anchor "+c.module), rec.index("push anchor") + if !c.takes { + if take >= 0 { + t.Errorf("%s was taken at genesis", c.module) + } + continue + } + if !(assign >= 0 && assign < take && take < push) { + t.Errorf("%s: assign %d, take %d, push %d: %v", c.module, assign, take, push, rec.told) + } + } +} + +func TestAConvergedGenesisTakesNothing(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + rec := &controlRecorder{settings: map[string]string{}} + control := controlPlane{container: "temp-mesh-controller", run: rec.run, timeout: time.Second} + if _, err := installModule(context.Background(), Options{Node: "anchor", Wait: time.Second}, control, + RegistryModule, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + if rec.index("take") >= 0 { + t.Errorf("a converged genesis took a module: %v", rec.told) + } +} + +func TestAnAdoptedGenesisOpensTheRegistryFromAnywhere(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + rec := &controlRecorder{settings: map[string]string{}} + control := controlPlane{container: "temp-mesh-controller", run: rec.run, timeout: time.Second} + o := Options{Node: "anchor", Adopted: true, Ports: FoundationPorts{Registry: 5100}, Wait: time.Second} + if _, err := installModule(context.Background(), o, control, RegistryModule, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + if got := rec.settings["distribution-settings.json"]; got != `{"expose":{"5000":"anywhere"},"ports":{"5000":5100}}` { + t.Errorf("the registry was told %s", got) + } +} + +func TestAnAdoptedGenesisChoosesTheFilterAndLoadsNone(t *testing.T) { + rec := &controlRecorder{settings: map[string]string{}} + control := controlPlane{container: "mesh-controller", run: rec.run, timeout: time.Second} + o := Options{Node: "anchor", Adopted: true, Answers: map[string]string{"packet-filter": "nftables"}} + filter, err := ChooseAndInstallFilter(context.Background(), o, control, quietly) + if err != nil || filter != "nftables" { + t.Fatalf("%q %v", filter, err) + } + if len(rec.told) != 0 { + t.Errorf("an adopted genesis installed a filter: %v", rec.told) + } +} + +func TestAnAdoptedNodeIsRecordedAdopted(t *testing.T) { + stop := errors.New("stop here") + runtime := &asked{answer: func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "node list"): + return "", nil + case strings.Contains(joined, "node add"): + return "", nil + } + return "", fmt.Errorf("%w: %s %v", stop, name, args) + }} + _, _ = Enrol(context.Background(), Options{ + Node: "anchor", Adopted: true, State: t.TempDir() + "/state.json", Timeout: time.Second, + Host: "/usr/local/bin/mesh-host", HostInBackground: true, + }, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second}, + func(string) {}) + if !runtime.ran("node add anchor --adopted") { + t.Errorf("the node was not added adopted: %v", runtime.commands) + } +} diff --git a/internal/bootstrap/apply.go b/internal/bootstrap/apply.go index 372793d..82c6a10 100644 --- a/internal/bootstrap/apply.go +++ b/internal/bootstrap/apply.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "strings" "github.com/novox/mesh-host/internal/apply" @@ -46,8 +47,13 @@ func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declarati return apply.Report{}, err } - report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried, run, - func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed) + // The bundle writes over whatever the machine has at the paths the foundation needs — a + // distribution's own /etc/nftables.conf among them — so it keeps the original of each file it + // has no record of, beside the node's state, exactly as a declaration from the mesh does + // (novox/hq ADR 0100). + report, updated, applyErr := apply.ApplyKeeping(ctx, sys, d, known, store.OriginCarried, run, + func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed, + apply.KeepIn(filepath.Dir(o.State))) // Saved whichever way it went, for the reason `mesh-host` gives: what was applied before a // failure is on the machine either way, and a host that did not record it would believe it diff --git a/internal/bootstrap/apply_test.go b/internal/bootstrap/apply_test.go index e3f45e6..486b452 100644 --- a/internal/bootstrap/apply_test.go +++ b/internal/bootstrap/apply_test.go @@ -3,8 +3,13 @@ package bootstrap import ( "context" "errors" + "os" + "path/filepath" "strings" "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/system" ) // `mesh-host` is built for one operating system and pins it at link time. An installer run by hand @@ -89,3 +94,36 @@ func TestASealedFileInAFoundationIsRefusedWithAReason(t *testing.T) { t.Errorf("the refusal does not say why there is no key: %v", err) } } + +// Defends novox/hq ADR 0100: the carried bundle keeps the original of a file it writes over that +// the host has no record of — the distribution's own ruleset, say. +func TestTheBundleKeepsTheOriginalOfWhatItWritesOver(t *testing.T) { + dir := t.TempDir() + conf := filepath.Join(dir, "nftables.conf") + if err := os.WriteFile(conf, []byte("# the distribution's own\n"), 0o644); err != nil { + t.Fatal(err) + } + d, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[ + {"id":"base-filter","type":"file","path":"` + conf + `","content":"table inet mesh {}\n"}]}`)) + if err != nil { + t.Fatal(err) + } + sys, err := system.For("arch") + if err != nil { + t.Fatal(err) + } + o := Options{State: filepath.Join(dir, "state.json")} + report, err := ApplyBundle(context.Background(), o, sys, d, nil, quietly) + if err != nil { + t.Fatal(err) + } + detail := report.Outcomes[0].Detail + at := strings.Index(detail, "kept at ") + if at < 0 { + t.Fatalf("the bundle wrote over a file it had no record of and kept nothing: %q", detail) + } + if got, err := os.ReadFile(detail[at+len("kept at "):]); err != nil || + string(got) != "# the distribution's own\n" { + t.Errorf("the kept original is %q (%v)", got, err) + } +} diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 9d3d787..4349bcc 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -34,6 +34,8 @@ import ( "fmt" "strings" "time" + + "github.com/novox/mesh-host/internal/firewall" ) // Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence @@ -185,6 +187,20 @@ type Options struct { Prompt func(Choice) (string, error) // Extras are catalogue modules beyond the floor, asked for by name. Extras []string + + // Ports are the ports the foundation binds on this machine (novox/hq ADR 0100). Inputs to + // genesis, each checked free before anything is raised, and then the node's settings for the + // foundation's modules — so adopting the foundation as modules leaves it where it was raised. + // Zero means the catalogue's defaults. + Ports FoundationPorts + // OverlayRange is the private network's address range, checked against every interface and + // route the machine already has. Empty means the mesh's default. + OverlayRange string + + // Adopted raises this machine as an adopted node (novox/hq ADR 0100): what is on it is kept + // until each module is taken, its firewall stays in force, and the mesh guards its own ports + // in a table that only refuses. Without it, a machine in use is refused. + Adopted bool } // pivots reports whether this run goes past the foundation. @@ -287,6 +303,14 @@ type Result struct { // Stopped names why a run went no further. Empty on a run that pivoted. Stopped string `json:"stopped,omitempty"` + + // Adopted, the firewall found, and the ports the foundation was raised on (novox/hq ADR 0100). + Adopted bool `json:"adopted,omitempty"` + Firewall string `json:"firewall,omitempty"` + Ports FoundationPorts `json:"ports"` + // Filter is the packet filter chosen for when the node converges; an adopted genesis loads + // none, and the flip assigns this one. + Filter string `json:"filter-on-converge,omitempty"` } // Run performs the bootstrap, saying what it is doing as it goes. @@ -339,7 +363,12 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro if say == nil { say = func(string) {} } - result := Result{DryRun: o.DryRun} + result := Result{DryRun: o.DryRun, Adopted: o.Adopted} + o.Ports = o.Ports.orDefaults() + result.Ports = o.Ports + if err := o.Ports.Check(); err != nil { + return result, failed(StepPreflight, err) + } // ---- 1. preflight ------------------------------------------------------------------- say("preflight — what has to be true before anything is changed") @@ -350,6 +379,23 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // Which half of the host applies things here. Asked of the machine and proved, because // `mesh-host` pins this at link time and an installer run by hand has no link time. + // An adopted machine keeps the firewall it was found with, so the mesh must speak it; one no + // host speaks is refused here, before anything changes (novox/hq ADR 0100). + if o.Adopted { + kind, name, err := firewall.Detect(ctx, d.Run) + if err != nil { + return result, failed(StepPreflight, err) + } + if kind == firewall.Unsupported { + return result, failed(StepPreflight, fmt.Errorf( + "this machine is filtered by %s, and no host speaks that firewall yet. An adopted "+ + "machine keeps its firewall in force, so the mesh could neither open what it needs "+ + "through it nor say what it would close. Nothing was changed", name)) + } + result.Firewall = string(kind) + say(" adopted what is on this machine is kept; its firewall (" + string(kind) + ") stays in force") + } + sys, err := WorkOutSystem(ctx, d.Run, o.System) if err != nil { return result, failed(StepPreflight, err) @@ -399,12 +445,32 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro if err := RefuseExistingServers(ctx, d.Run, creds); err != nil { return result, failed(StepBundle, err) } + // The foundation's ports, its private network's range and its containers' names are checked + // free before anything is raised (novox/hq ADR 0100), each refusal naming what holds it. + if err := CheckTheMachine(ctx, o, d.Run, rewritten.Declaration, say); err != nil { + return result, failed(StepBundle, err) + } // From here on nothing this installer says contains the values it just made. say = Masking(say, creds) root, err := RewriteRoot(&rewritten, creds) if err != nil { return result, failed(StepBundle, err) } + moved, err := RewritePorts(&rewritten, o.Ports, o.OverlayRange) + if err != nil { + return result, failed(StepBundle, err) + } + if moved.Places > 0 { + say(fmt.Sprintf(" ports %d place(s) rewritten to this node's foundation ports", moved.Places)) + } + if o.Adopted { + adopted, err := RewriteAdopted(&rewritten, o.Ports) + if err != nil { + return result, failed(StepBundle, err) + } + say(fmt.Sprintf(" adopted bundle the foundation's filter is not loaded (%s); the mesh's guard refuses %v from outside", + strings.Join(adopted.Removed, ", "), adopted.Guarded)) + } for _, c := range []struct { what, path string made bool @@ -679,7 +745,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // ---- 17. filter ----------------------------------------------------------------------- say("filter — required, so the question is which, not whether") - if err := ChooseAndInstallFilter(ctx, o, permanentControl, say); err != nil { + filter, err := ChooseAndInstallFilter(ctx, o, permanentControl, say) + result.Filter = filter + if err != nil { return result, failed(StepFilter, err) } @@ -697,6 +765,12 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro return result, failed(StepExport, err) } + if o.Adopted { + say("\nthis machine is a mesh of one adopted node: it builds its own software, holds its graph " + + "and sits on its private network, and what it ran before is kept as it was, behind the firewall " + + "it was found with. Take each module on it once its data has moved; converge it when done.") + return result, nil + } say("\nthis machine is a mesh of one node: it builds its own software, holds its graph, " + "sits on its private network, and filters what modules declared.") say("what remains is somebody else's: adding nodes, and assigning what they should run.") diff --git a/internal/bootstrap/builder.go b/internal/bootstrap/builder.go index b86c1b1..e8ca324 100644 --- a/internal/bootstrap/builder.go +++ b/internal/bootstrap/builder.go @@ -1,8 +1,10 @@ package bootstrap import ( + "bytes" "context" "fmt" + "strconv" "strings" ) @@ -53,6 +55,9 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane "This is the manifest that makes the builder an ordinary module. Without it the mesh "+ "has the image and no way to run it, so nothing can be built here", err) } + if manifest, err = followPackagesPort(manifest, o.Ports.orDefaults().Packages); err != nil { + return out, err + } pinned, places, err := pinPlaceholder(manifest, published.Reference, BuilderModule) if err != nil { return out, err @@ -84,3 +89,25 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane out.Installed.Pushed, err = pushNode(ctx, o, control, say) return out, err } + +// packagesPortInBinding is the package registry's port as the builder's manifest names it, in the +// binding file it carries — JSON inside a JSON string, so its quotes are escaped. +const packagesPortInBinding = `\"port\": 3000` + +// followPackagesPort points the builder's package binding at the port the node gave the package +// registry (novox/hq ADR 0100). Genesis raises the registry by hand before gitea is a module, so no +// binding the controller resolves can say where it is; the builder carries the address in its own +// manifest, and a port given at genesis must reach it there or the base build dials a port nothing +// answers on. At the default it is left byte for byte as the catalogue has it. +func followPackagesPort(manifest []byte, port int) ([]byte, error) { + if port == defaultGiteaPort { + return manifest, nil + } + if n := bytes.Count(manifest, []byte(packagesPortInBinding)); n != 1 { + return nil, fmt.Errorf("the builder's manifest names the package registry's port %d time(s) where "+ + "this installer looks for it once (%s), so the port given with --packages-port cannot reach "+ + "it; nothing was changed", n, packagesPortInBinding) + } + return bytes.Replace(manifest, []byte(packagesPortInBinding), + []byte(`\"port\": `+strconv.Itoa(port)), 1), nil +} diff --git a/internal/bootstrap/builder_test.go b/internal/bootstrap/builder_test.go new file mode 100644 index 0000000..c2f9848 --- /dev/null +++ b/internal/bootstrap/builder_test.go @@ -0,0 +1,69 @@ +package bootstrap + +import ( + "encoding/json" + "strings" + "testing" +) + +// Defends novox/hq ADR 0100: a port given for the package registry at genesis reaches the one +// thing that dials it by a fixed number, the builder's package binding. + +// The builder's package binding exactly as the catalogue's manifest carries it. +const builderManifest = `{ + "module": "builder", + "resources": [ + { + "id": "package-binding", + "type": "file", + "path": "/var/lib/mesh/builder/package-registry.json", + "mode": "0600", + "content": "{\"provision\": \"package-registry\", \"from\": \"gitea\", \"at\": \"127.0.0.1\", \"as\": \"mesh-builder\", \"serves\": {\"scheme\": \"http\", \"port\": 3000, \"npm-path\": \"/api/packages/novox/npm/\"}}\n" + } + ] +}` + +func bindingPort(t *testing.T, manifest []byte) float64 { + t.Helper() + var m struct { + Resources []struct { + Content string `json:"content"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + t.Fatal(err) + } + var binding struct { + Serves struct { + Port float64 `json:"port"` + } `json:"serves"` + } + if err := json.Unmarshal([]byte(m.Resources[0].Content), &binding); err != nil { + t.Fatal(err) + } + return binding.Serves.Port +} + +func TestTheBuilderFollowsThePackageRegistrysGivenPort(t *testing.T) { + got, err := followPackagesPort([]byte(builderManifest), 3100) + if err != nil { + t.Fatal(err) + } + if p := bindingPort(t, got); p != 3100 { + t.Errorf("the builder's binding dials %v, not the port given", p) + } +} + +func TestTheBuilderOnTheDefaultPortIsUnchanged(t *testing.T) { + got, err := followPackagesPort([]byte(builderManifest), 3000) + if err != nil || string(got) != builderManifest { + t.Errorf("the default port changed the manifest: %v", err) + } +} + +func TestABuilderManifestThatNoLongerNamesThePortIsRefused(t *testing.T) { + moved := strings.Replace(builderManifest, `\"port\": 3000`, `\"port\": 3001`, 1) + if _, err := followPackagesPort([]byte(moved), 3100); err == nil || !strings.Contains(err.Error(), "--packages-port") { + t.Errorf("a manifest the port cannot reach was accepted: %v", err) + } +} diff --git a/internal/bootstrap/enrol.go b/internal/bootstrap/enrol.go index e64b86c..7ee3b71 100644 --- a/internal/bootstrap/enrol.go +++ b/internal/bootstrap/enrol.go @@ -72,7 +72,13 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla if mentions(nodes, o.Node) { say(" already a node " + o.Node) } else { - if _, err := control.tell(ctx, "node", "add", o.Node); err != nil { + add := []string{"node", "add", o.Node} + if o.Adopted { + // The controller records the node's mode; an adopted one keeps what it was found with + // until each module is taken (novox/hq ADR 0100). + add = append(add, "--adopted") + } + if _, err := control.tell(ctx, add...); err != nil { return out, err } out.Added = true diff --git a/internal/bootstrap/inuse.go b/internal/bootstrap/inuse.go new file mode 100644 index 0000000..c3cf613 --- /dev/null +++ b/internal/bootstrap/inuse.go @@ -0,0 +1,146 @@ +package bootstrap + +import ( + "context" + "fmt" + "net" + "strings" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/reachable" + "github.com/novox/mesh-host/internal/store" +) + +// quiet are the processes every fresh machine runs that serve nobody: name resolution (whose +// link-local resolver listens on TCP as well as UDP, on every address) and the network manager's +// address configuration. ss names a process by its first fifteen characters, so both spellings are +// here. +// +// **Only what the measurement found** (novox/hq ADR 0101): these two hold every listener on a +// freshly installed lab machine (testdata/fresh-machine-listeners.txt) and nothing else does. A +// daemon joins this list with a measurement of a fresh machine that holds it, never by guess — a +// time client or an address-configuration client listening on a machine that does not run one as +// standard is something somebody installed, and that is a machine in use. +var quiet = map[string]bool{ + "systemd-resolved": true, "systemd-resolve": true, + "systemd-networkd": true, "systemd-network": true, +} + +// InUse says what makes this machine a machine in use (novox/hq ADR 0100): every running container +// no host made, and every socket listening on an address other than loopback that is neither ssh's +// nor held by what every fresh machine runs. ours names +// what the mesh itself runs, which a re-run of genesis finds and does not count. +func InUse(ctx context.Context, run Runner, ours func(name string) bool) ([]string, []reachable.Reach, error) { + var containers []string + out, err := run(ctx, "docker", "ps", "--format", "{{.Names}}\t{{.Label \"mesh-host.spec\"}}") + if err != nil { + return nil, nil, fmt.Errorf("cannot ask the container runtime what is running here: %w", err) + } + for _, line := range strings.Split(out, "\n") { + name, label, _ := strings.Cut(strings.TrimSpace(line), "\t") + label = strings.TrimSpace(label) + if name == "" || (label != "" && label != "") || ours(name) { + continue + } + containers = append(containers, name) + } + + listening, err := run(ctx, "ss", "-Hltunp") + if err != nil { + return nil, nil, fmt.Errorf("cannot read what listens on this machine: %w", err) + } + var listeners []reachable.Reach + for _, r := range reachable.Sockets(listening) { + if counts(r) && !ours(r.By) { + listeners = append(listeners, r) + } + } + return containers, listeners, nil +} + +func counts(r reachable.Reach) bool { + if ip := net.ParseIP(r.Address); ip != nil && ip.IsLoopback() { + return false + } + switch r.Protocol { + case "tcp": + return r.By != "sshd" && !(r.By == "" && r.Port == 22) && !quiet[r.By] + case "udp": + return !quiet[r.By] + } + return false +} + +// RefuseAMachineInUse is the check a converged genesis makes before changing anything: a machine +// in use is refused, naming every container and listener counted, because raising the foundation's +// filter there would close what it serves — a forgotten --adopted must not close a working machine. +// An adopted genesis is told what it found, and goes on. +func RefuseAMachineInUse(ctx context.Context, o Options, run Runner, say func(string)) error { + known, err := store.Load(o.State) + if err != nil { + return err + } + if len(known.Resources) > 0 { + // **The machine says how it was raised** (novox/hq ADR 0103). A re-run must not change + // the node's mode by a flag forgotten or added: without --adopted the bundle would load + // the foundation's dropping filter over the found firewall, and with it on a converged + // machine the filter the node relies on would be removed as no longer carried. + switch adopted := RecordsAdoption(known); { + case adopted && !o.Adopted: + return fmt.Errorf("this machine was raised adopted, and genesis was run again without --adopted. " + + "Run it again the way it was raised: pass --adopted. Returning it to converged is the " + + "controller's act (converge), never genesis's; nothing was changed") + case !adopted && o.Adopted: + return fmt.Errorf("this machine was raised converged, and genesis was run again with --adopted, " + + "which would remove the foundation's filter it relies on. Run it again without --adopted; " + + "returning a node to adopted is the controller's act (adopt); nothing was changed") + } + // What genesis raised on an earlier run is the mesh's, and it is what the machine now + // serves; the question was answered the first time. + say(" in use not asked: this machine carries what an earlier genesis raised") + return nil + } + containers, listeners, err := InUse(ctx, run, func(string) bool { return false }) + if err != nil { + return err + } + if len(containers) == 0 && len(listeners) == 0 { + say(" in use no: no container runs and nothing listens beyond ssh") + return nil + } + var named []string + for _, c := range containers { + named = append(named, "container "+c) + } + for _, l := range listeners { + by := l.By + if by == "" { + by = "an unnamed process" + } + named = append(named, fmt.Sprintf("%s %s:%d by %s", l.Protocol, l.Address, l.Port, by)) + } + if o.Adopted { + say(fmt.Sprintf(" in use yes, and adopted: %d thing(s) found are kept", len(named))) + return nil + } + return fmt.Errorf("this machine is in use, and a converged genesis would close what it serves:\n - %s\n"+ + "If it is meant to join the mesh keeping what it runs, pass --adopted: its firewall stays in "+ + "force and every module is taken on it one at a time. Nothing was changed", + strings.Join(named, "\n - ")) +} + +// RecordsAdoption is whether this machine's state says it is an adopted node: it holds something +// of the mesh's that only an adopted node has — the guard or an opening, under the adoption prefix +// — or something it found and holds. A node the controller converged has neither any more; the +// record of the firewall it found outlives the flip, so it is not read as the mode. +func RecordsAdoption(known store.State) bool { + if len(known.Held) > 0 { + return true + } + for _, r := range known.Resources { + if strings.HasPrefix(r.ID, declaration.AdoptionPrefix) { + return true + } + } + return false +} diff --git a/internal/bootstrap/inuse_test.go b/internal/bootstrap/inuse_test.go new file mode 100644 index 0000000..2cc889d --- /dev/null +++ b/internal/bootstrap/inuse_test.go @@ -0,0 +1,179 @@ +package bootstrap + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0100: a converged genesis refuses a machine in use, naming every container +// and listener it counted. + +// Lines as `ss -Hltunp` prints them. The ssh, samba, loopback and proxy lines are captured from a +// real machine; the resolver and network-manager lines are written in the same shape. What a fresh +// machine actually runs is measured in testdata/fresh-machine-listeners.txt. +const inUseSockets = `tcp LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=1188536,fd=6)) +tcp LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=1188536,fd=7)) +tcp LISTEN 0 32 127.0.0.1:53 0.0.0.0:* users:(("dnsmasq",pid=1189392,fd=7)) +tcp LISTEN 0 4096 127.0.0.1:5432 0.0.0.0:* users:(("docker-proxy",pid=1854543,fd=7)) +udp UNCONN 0 0 0.0.0.0:5355 0.0.0.0:* users:(("systemd-resolve",pid=301,fd=11)) +udp UNCONN 0 0 192.0.2.10%eth0:68 0.0.0.0:* users:(("systemd-network",pid=280,fd=19)) +` + +const servingSockets = `tcp LISTEN 0 50 0.0.0.0:445 0.0.0.0:* users:(("smbd",pid=1248,fd=29)) +tcp LISTEN 0 4096 0.0.0.0:8080 0.0.0.0:* users:(("docker-proxy",pid=1920035,fd=7)) +udp UNCONN 0 0 0.0.0.0:123 0.0.0.0:* users:(("ntpd",pid=1070791,fd=17)) +` + +type inUseRunner struct{ ps, ss string } + +func (m inUseRunner) run(_ context.Context, name string, args ...string) (string, error) { + if name == "docker" { + return m.ps, nil + } + return m.ss, nil +} + +func TestAFreshMachineIsNotInUse(t *testing.T) { + containers, listeners, err := InUse(context.Background(), inUseRunner{ss: inUseSockets}.run, + func(string) bool { return false }) + if err != nil { + t.Fatal(err) + } + if len(containers) != 0 || len(listeners) != 0 { + t.Errorf("ssh, loopback and the daemons a fresh machine runs were counted: %v %v", containers, listeners) + } +} + +func TestAMachineServingIsInUse(t *testing.T) { + m := inUseRunner{ps: "hello-web\t\nmesh-store\tabc123\n", ss: inUseSockets + servingSockets} + containers, listeners, err := InUse(context.Background(), m.run, func(string) bool { return false }) + if err != nil { + t.Fatal(err) + } + if len(containers) != 1 || containers[0] != "hello-web" { + t.Errorf("containers counted: %v (one a host made is not a predecessor's)", containers) + } + var by []string + for _, l := range listeners { + by = append(by, l.By) + } + if strings.Join(by, " ") != "smbd docker-proxy ntpd" { + t.Errorf("listeners counted: %v", listeners) + } +} + +func TestAConvergedGenesisRefusesAMachineInUseNamingEverything(t *testing.T) { + o := Options{State: filepath.Join(t.TempDir(), "state.json")} + m := inUseRunner{ps: "hello-web\t\n", ss: inUseSockets + servingSockets} + err := RefuseAMachineInUse(context.Background(), o, m.run, quietly) + if err == nil { + t.Fatal("a machine in use was not refused") + } + for _, want := range []string{"container hello-web", "tcp 0.0.0.0:445 by smbd", "tcp 0.0.0.0:8080 by docker-proxy", + "udp 0.0.0.0:123 by ntpd", "--adopted"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not name %q: %v", want, err) + } + } + o.Adopted = true + if err := RefuseAMachineInUse(context.Background(), o, m.run, quietly); err != nil { + t.Errorf("an adopted genesis was refused a machine in use: %v", err) + } +} + +func TestARerunOfGenesisIsNotAMachineInUse(t *testing.T) { + o := Options{State: filepath.Join(t.TempDir(), "state.json")} + if err := store.Save(o.State, store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}}); err != nil { + t.Fatal(err) + } + m := inUseRunner{ps: "mesh-gitea-server\t\n", ss: servingSockets} + if err := RefuseAMachineInUse(context.Background(), o, m.run, quietly); err != nil { + t.Errorf("what an earlier genesis raised was counted as a machine in use: %v", err) + } +} + +func TestAFreshlyInstalledMachineAsMeasuredIsNotInUse(t *testing.T) { + // Captured with `ss -Hltunp` on a freshly installed lab machine: its resolver listens on TCP on + // every address, which the record's words alone would count. + raw, err := os.ReadFile("testdata/fresh-machine-listeners.txt") + if err != nil { + t.Fatal(err) + } + run := func(ctx context.Context, name string, args ...string) (string, error) { + if name == "ss" { + return string(raw), nil + } + return "", nil + } + containers, listeners, err := InUse(context.Background(), run, func(string) bool { return false }) + if err != nil { + t.Fatal(err) + } + if len(containers) != 0 || len(listeners) != 0 { + t.Errorf("a fresh machine read as in use: containers %v, listeners %v", containers, listeners) + } +} + +// Defends novox/hq ADR 0103: a machine raised adopted stays adopted if genesis is run again. The +// installer reads the mode from what the machine records, and refuses a flag that disagrees. +func TestARerunWithoutTheFlagOnAnAdoptedMachineIsRefused(t *testing.T) { + o := Options{State: filepath.Join(t.TempDir(), "state.json")} + adoptedState := store.State{Resources: []store.Applied{ + {ID: "store", Type: "container", Target: "mesh-store", Origin: store.OriginCarried}, + {ID: "adoption.guard", Type: "file", Target: "/etc/mesh/guard.nft", Origin: store.OriginCarried}, + }} + if err := store.Save(o.State, adoptedState); err != nil { + t.Fatal(err) + } + m := inUseRunner{ss: inUseSockets} + err := RefuseAMachineInUse(context.Background(), o, m.run, quietly) + if err == nil || !strings.Contains(err.Error(), "pass --adopted") { + t.Fatalf("a re-run without --adopted on an adopted machine was not refused: %v", err) + } + o.Adopted = true + if err := RefuseAMachineInUse(context.Background(), o, m.run, quietly); err != nil { + t.Errorf("a re-run with --adopted on an adopted machine was refused: %v", err) + } +} + +func TestARerunWithTheFlagOnAConvergedMachineIsRefused(t *testing.T) { + o := Options{State: filepath.Join(t.TempDir(), "state.json"), Adopted: true} + converged := store.State{Resources: []store.Applied{ + {ID: "store", Type: "container", Target: "mesh-store", Origin: store.OriginCarried}, + {ID: "base-filter", Type: "file", Target: "/etc/nftables.conf", Origin: store.OriginCarried}, + }, + // Converged by the controller from adopted: the firewall it found is still recorded. + Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, DisabledByMesh: true}} + if err := store.Save(o.State, converged); err != nil { + t.Fatal(err) + } + err := RefuseAMachineInUse(context.Background(), o, inUseRunner{ss: inUseSockets}.run, quietly) + if err == nil || !strings.Contains(err.Error(), "without --adopted") { + t.Fatalf("a re-run with --adopted on a converged machine was not refused: %v", err) + } + o.Adopted = false + if err := RefuseAMachineInUse(context.Background(), o, inUseRunner{ss: inUseSockets}.run, quietly); err != nil { + t.Errorf("a converged re-run of a converged machine was refused: %v", err) + } +} + +func TestOnlyTheDaemonsTheMeasurementFoundAreQuiet(t *testing.T) { + // novox/hq ADR 0101: the exempt daemons are the ones a fresh machine was measured to run — + // the resolver and the network manager. A time client or a DHCP client listening beyond + // loopback is something somebody put there, and that is a machine in use. + sockets := `udp UNCONN 0 0 0.0.0.0:123 0.0.0.0:* users:(("systemd-timesyn",pid=260,fd=9)) +udp UNCONN 0 0 0.0.0.0:68 0.0.0.0:* users:(("dhcpcd",pid=270,fd=9)) +` + _, listeners, err := InUse(context.Background(), inUseRunner{ss: sockets}.run, func(string) bool { return false }) + if err != nil { + t.Fatal(err) + } + if len(listeners) != 2 { + t.Errorf("counted %d listener(s), want the time client and the DHCP client: %+v", len(listeners), listeners) + } +} diff --git a/internal/bootstrap/module.go b/internal/bootstrap/module.go index 4ba2729..6352d4d 100644 --- a/internal/bootstrap/module.go +++ b/internal/bootstrap/module.go @@ -124,6 +124,9 @@ func registerAndAssign(ctx context.Context, o Options, control controlPlane, mod // the only place the reason appears. say(indent(refusal)) } + if err := prepareModule(ctx, o, control, module, say); err != nil { + return out, err + } return out, nil } @@ -209,3 +212,14 @@ func pinPlaceholder(manifest []byte, reference, module string) ([]byte, int, err } return pinned, places, nil } + +// prepareModule is what genesis tells the controller about a module on this node once it is +// assigned and before it is pushed: the ports this node gave it, and on an adopted node that the +// module is taken (novox/hq ADR 0100). +func prepareModule(ctx context.Context, o Options, control controlPlane, module string, + say func(string)) error { + if err := setFoundationSettings(ctx, o, control, module, say); err != nil { + return err + } + return takeIfAdopted(ctx, o, control, module, say) +} diff --git a/internal/bootstrap/phase2.go b/internal/bootstrap/phase2.go index 4eb21a3..ede6305 100644 --- a/internal/bootstrap/phase2.go +++ b/internal/bootstrap/phase2.go @@ -3,8 +3,10 @@ package bootstrap import ( "context" "encoding/json" + "errors" "fmt" "net" + "strconv" "strings" "time" ) @@ -87,6 +89,9 @@ func InstallFromCatalogue(ctx context.Context, o Options, control controlPlane, if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { return err } + if err := prepareModule(ctx, o, control, module, say); err != nil { + return err + } if _, err := pushNode(ctx, o, control, say); err != nil { return err } @@ -120,7 +125,7 @@ func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane, Name: "endpoint", Question: "Where do other machines reach this one for the private network? " + "(host:port; the host other machines dial)", - Default: derivedEndpoint(brokerAddress), + Default: derivedEndpoint(brokerAddress, o.Ports.orDefaults().Hub), }, o.Answers["endpoint"], o.Prompt, say) if err != nil { return err @@ -129,6 +134,10 @@ func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane, return fmt.Errorf("the private network needs an endpoint other machines can dial, and " + "nothing said one: pass --endpoint, or --broker-address so one can be derived") } + endpoint, err = endpointAgrees(endpoint, o.Ports.orDefaults().Hub) + if err != nil { + return err + } if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { return err @@ -146,16 +155,22 @@ func PlaceOnTheNetwork(ctx context.Context, o Options, control controlPlane, // ChooseAndInstallFilter picks the packet filter — required, so the question is which, not // whether — and installs it. -func ChooseAndInstallFilter(ctx context.Context, o Options, control controlPlane, say func(string)) error { +func ChooseAndInstallFilter(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { filter, err := decide(Choice{ Name: "packet-filter", Question: "Which packet filter should this machine run?", Options: []string{"nftables"}, }, o.Answers["packet-filter"], o.Prompt, say) if err != nil { - return err + return "", err } - return InstallFromCatalogue(ctx, o, control, filter, say) + if o.Adopted { + // The firewall found here stays in force until the node converges; the filter is + // chosen now and assigned by the flip (novox/hq ADR 0100). + say(" not installed " + filter + " — this machine is adopted; converging it assigns " + filter) + return filter, nil + } + return filter, InstallFromCatalogue(ctx, o, control, filter, say) } // InstallExtras installs what was asked for beyond the floor. @@ -200,14 +215,38 @@ func builds(manifest []byte) bool { return m.Build != nil && len(m.Build.Artifacts) > 0 } +// endpointAgrees holds the endpoint other machines dial to the port this node gave the private +// network's hub (novox/hq ADR 0100): the hub binds what --hub-port says, so an endpoint naming +// another port is an address nothing answers on. A host alone takes the hub's port. +func endpointAgrees(endpoint string, hub int) (string, error) { + _, portText, err := net.SplitHostPort(endpoint) + var missing *net.AddrError + if errors.As(err, &missing) && missing.Err == "missing port in address" { + return net.JoinHostPort(strings.Trim(endpoint, "[]"), strconv.Itoa(hub)), nil + } + if err != nil { + return "", fmt.Errorf("--endpoint %q is not host:port: %w", endpoint, err) + } + port, err := strconv.Atoi(portText) + if err != nil { + return "", fmt.Errorf("--endpoint %q does not end in a port", endpoint) + } + if port != hub { + return "", fmt.Errorf("--endpoint %s names port %d and the private network's hub binds %d "+ + "(--hub-port): other machines would dial a port nothing answers on. Give one port for the "+ + "hub; nothing was changed", endpoint, port, hub) + } + return endpoint, nil +} + // derivedEndpoint is the default place other machines dial for the private network: the same host // they already dial for the broker, on WireGuard's ordinary port. One fact, not two. -func derivedEndpoint(brokerAddress string) string { +func derivedEndpoint(brokerAddress string, hub int) string { host, _, err := net.SplitHostPort(brokerAddress) if err != nil || host == "" { return "" } - return net.JoinHostPort(host, "51820") + return net.JoinHostPort(host, strconv.Itoa(hub)) } func refOr(ref string) string { diff --git a/internal/bootstrap/phase2_test.go b/internal/bootstrap/phase2_test.go index cd212fc..b1b46f3 100644 --- a/internal/bootstrap/phase2_test.go +++ b/internal/bootstrap/phase2_test.go @@ -1,14 +1,17 @@ package bootstrap -import "testing" +import ( + "strings" + "testing" +) // The endpoint other machines dial defaults to the host they already dial — the broker's — on // WireGuard's port. One fact, not two that drift. func TestTheEndpointDerivesFromTheBrokerAddress(t *testing.T) { - if got := derivedEndpoint("192.0.2.10:5671"); got != "192.0.2.10:51820" { + if got := derivedEndpoint("192.0.2.10:5671", 51820); got != "192.0.2.10:51820" { t.Fatalf("derived %q", got) } - if got := derivedEndpoint(""); got != "" { + if got := derivedEndpoint("", 51820); got != "" { t.Fatalf("an endpoint was invented from nothing: %q", got) } } @@ -23,3 +26,21 @@ func TestOnlyAManifestWithArtifactsBuilds(t *testing.T) { t.Fatal("a manifest with nothing to build was built anyway") } } + +// Defends novox/hq ADR 0100: the hub's port is the node's, and the endpoint other machines dial +// must name it — an endpoint on another port is an address nothing answers on. +func TestTheEndpointAgreesWithTheHubsPort(t *testing.T) { + if got, err := endpointAgrees("192.0.2.10:51820", 51820); err != nil || got != "192.0.2.10:51820" { + t.Errorf("an endpoint on the hub's port: %q %v", got, err) + } + if got, err := endpointAgrees("192.0.2.10", 51821); err != nil || got != "192.0.2.10:51821" { + t.Errorf("a host alone did not take the hub's port: %q %v", got, err) + } + _, err := endpointAgrees("192.0.2.10:51820", 51821) + if err == nil || !strings.Contains(err.Error(), "--hub-port") { + t.Errorf("two ports for one hub were accepted: %v", err) + } + if _, err := endpointAgrees("192.0.2.10:not-a-port", 51820); err == nil { + t.Error("an endpoint whose port is not a number was accepted") + } +} diff --git a/internal/bootstrap/phase3.go b/internal/bootstrap/phase3.go index e39b9e3..2d60811 100644 --- a/internal/bootstrap/phase3.go +++ b/internal/bootstrap/phase3.go @@ -122,6 +122,9 @@ func installProvider(ctx context.Context, o Options, control controlPlane, modul if _, err := control.tell(ctx, "assign", o.Node, module); err != nil { return err } + if err := prepareModule(ctx, o, control, module, say); err != nil { + return err + } if beforePush != nil { if err := beforePush(); err != nil { return err diff --git a/internal/bootstrap/phase_packages.go b/internal/bootstrap/phase_packages.go index 1a4ceed..bb8414b 100644 --- a/internal/bootstrap/phase_packages.go +++ b/internal/bootstrap/phase_packages.go @@ -42,8 +42,9 @@ const ( // giteaDBRole/giteaDBName is gitea's own database in the foundation store. giteaDBRole = "mesh_gitea" giteaDBName = "mesh_gitea" - // giteaPort is where the raised server answers on the machine. - giteaPort = 3000 + // defaultGiteaPort is where the raised server answers on the machine unless the node gave the + // package registry another port (novox/hq ADR 0100). + defaultGiteaPort = 3000 ) // RaisePackageRegistry puts a working npm registry in front of the base build. It is idempotent: @@ -60,17 +61,18 @@ func RaisePackageRegistry(ctx context.Context, o Options, d Deps, control contro } say(" seeding gitea's database in the foundation store") + ports := o.Ports.orDefaults() if err := seedGiteaDatabase(ctx, run, o.Timeout, dbPassword, say); err != nil { return err } say(" raising the gitea server on that database") - if err := raiseGiteaServer(ctx, run, o.Timeout, dbPassword, say); err != nil { + if err := raiseGiteaServer(ctx, run, o.Timeout, dbPassword, ports, say); err != nil { return err } say(" waiting for gitea to answer") - base := fmt.Sprintf("http://127.0.0.1:%d", giteaPort) + base := fmt.Sprintf("http://127.0.0.1:%d", ports.Packages) if err := waitForGitea(ctx, d, o, base, say); err != nil { return err } @@ -146,11 +148,11 @@ func seedGiteaDatabase(ctx context.Context, run Runner, timeout time.Duration, p return nil } -// raiseGiteaServer starts the gitea server container against the foundation store. It joins the -// store's network namespace so `127.0.0.1:5432` reaches postgres, and publishes its own port on the -// machine so the builder and this installer can reach it. Started if absent, left alone if present. +// raiseGiteaServer starts the gitea server container against the foundation store. It runs on the +// machine's own network, so `127.0.0.1` reaches the store where it publishes its port, and it binds +// its own port there for the builder and this installer. Started if absent, left alone if present. func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, dbPassword string, - say func(string)) error { + ports FoundationPorts, say func(string)) error { asking, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -164,7 +166,7 @@ func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, db env := []string{ "-e", "GITEA__database__DB_TYPE=postgres", // The store is reached on the shared network namespace's loopback. - "-e", "GITEA__database__HOST=127.0.0.1:5432", + "-e", fmt.Sprintf("GITEA__database__HOST=127.0.0.1:%d", ports.Store), "-e", "GITEA__database__NAME=" + giteaDBName, "-e", "GITEA__database__USER=" + giteaDBRole, "-e", "GITEA__database__PASSWD=" + dbPassword, @@ -174,9 +176,14 @@ func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, db // package metadata hands npm a tarball URL built from ROOT_URL, and a client only sends its // stored credential to the host it was stored for. A default ROOT_URL of localhost is a // different host than the binding's 127.0.0.1, so the credential would not be sent. - "-e", fmt.Sprintf("GITEA__server__ROOT_URL=http://127.0.0.1:%d/", giteaPort), + "-e", fmt.Sprintf("GITEA__server__ROOT_URL=http://127.0.0.1:%d/", ports.Packages), "-e", "USER_UID=1000", "-e", "USER_GID=1000", } + if ports.Packages != defaultGiteaPort { + // On the machine's network the server binds its own port, so a port given for it is + // the one it is told to listen on. + env = append(env, "-e", fmt.Sprintf("GITEA__server__HTTP_PORT=%d", ports.Packages)) + } args := append([]string{ "run", "-d", "--name", giteaBootstrap, // Host network, like the control plane: it reaches the foundation store on the machine's diff --git a/internal/bootstrap/ports.go b/internal/bootstrap/ports.go new file mode 100644 index 0000000..f7a77c3 --- /dev/null +++ b/internal/bootstrap/ports.go @@ -0,0 +1,460 @@ +package bootstrap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net" + "sort" + "strconv" + "strings" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/reachable" + "github.com/novox/mesh-host/internal/store" +) + +// FoundationPorts are the machine's ports the foundation binds (novox/hq ADR 0100). +// +// **The node's, not the catalogue's.** A machine in use may already hold one — a predecessor's +// registry on 5000, its broker's management port — and a port fixed in the bundle and the manifests +// surfaces as a container that fails to bind, and one changed at genesis would be changed back when +// the foundation is adopted as modules. So each is an input here, checked free, rewritten into the +// bundle, and handed to the controller as that node's setting for the module that binds it. +type FoundationPorts struct { + Store int `json:"store"` + Bus int `json:"bus"` + AMQP int `json:"amqp"` + Management int `json:"management"` + Registry int `json:"registry"` + Packages int `json:"packages"` + Hub int `json:"hub"` +} + +// DefaultPorts are the catalogue's numbers. +func DefaultPorts() FoundationPorts { + return FoundationPorts{Store: 5432, Bus: 5671, AMQP: 5672, Management: 15672, Registry: 5000, + Packages: 3000, Hub: 51820} +} + +// DefaultOverlayRange is the controller's default private-network range. +const DefaultOverlayRange = "10.42.0.0/16" + +// orDefaults fills every port left unsaid. +func (p FoundationPorts) orDefaults() FoundationPorts { + d := DefaultPorts() + for _, f := range []struct{ got, def *int }{ + {&p.Store, &d.Store}, {&p.Bus, &d.Bus}, {&p.AMQP, &d.AMQP}, {&p.Management, &d.Management}, + {&p.Registry, &d.Registry}, {&p.Packages, &d.Packages}, {&p.Hub, &d.Hub}, + } { + if *f.got == 0 { + *f.got = *f.def + } + } + return p +} + +// named is each port with what it is and its protocol, in a fixed order. +func (p FoundationPorts) named() []namedPort { + return []namedPort{ + {"the store", "tcp", p.Store}, {"the bus", "tcp", p.Bus}, {"the broker's AMQP", "tcp", p.AMQP}, + {"the broker's management", "tcp", p.Management}, {"the registry", "tcp", p.Registry}, + {"the package registry", "tcp", p.Packages}, {"the private network's hub", "udp", p.Hub}, + } +} + +type namedPort struct { + what, protocol string + port int +} + +// Check refuses a port out of range, or one port given for two things. +func (p FoundationPorts) Check() error { + seen := map[string]string{} + for _, n := range p.named() { + if n.port < 1 || n.port > 65535 { + return fmt.Errorf("%s's port is %d, and a port is 1-65535", n.what, n.port) + } + key := n.protocol + "/" + strconv.Itoa(n.port) + if other, twice := seen[key]; twice { + return fmt.Errorf("%s and %s were both given %s", other, n.what, key) + } + seen[key] = n.what + } + return nil +} + +// moduleSettings is what each foundation module is told about its ports on this node: the port it +// declares, to the machine's port it is given. Only what differs from the catalogue — a converged +// genesis on the defaults sets nothing, and so changes nothing it did before. +func (p FoundationPorts) moduleSettings() map[string]map[string]int { + d := DefaultPorts() + out := map[string]map[string]int{} + add := func(module string, declared, given int) { + if given == declared { + return + } + if out[module] == nil { + out[module] = map[string]int{} + } + out[module][strconv.Itoa(declared)] = given + } + add("postgres", d.Store, p.Store) + add("lavinmq", d.Bus, p.Bus) + add("lavinmq", d.AMQP, p.AMQP) + add("lavinmq", d.Management, p.Management) + add(RegistryModule, d.Registry, p.Registry) + return out +} + +// PortsSetting is the controller's settings key for a module's given ports. +const PortsSetting = "ports" + +// setFoundationSettings tells the controller the ports this node gave a foundation module — and, +// on an adopted node, that the registry is reached from anywhere, as a node pulls from it before it +// has a private-network address (novox/hq ADR 0100). Done after the module is registered and before +// the push that raises it, so the first declaration already names the node's ports. +func setFoundationSettings(ctx context.Context, o Options, control controlPlane, module string, + say func(string)) error { + values := map[string]any{} + if ports := o.Ports.orDefaults().moduleSettings()[module]; len(ports) > 0 { + values[PortsSetting] = ports + } + if o.Adopted && module == RegistryModule { + values["expose"] = map[string]string{strconv.Itoa(DefaultPorts().Registry): "anywhere"} + } + if len(values) == 0 { + return nil + } + raw, err := json.Marshal(values) + if err != nil { + return err + } + remote := "/" + module + "-settings.json" + if err := control.carrying(ctx, module+"-settings.json", raw, remote); err != nil { + return err + } + if _, err := control.tell(ctx, "settings", "set", module, remote, "--node", o.Node); err != nil { + return err + } + say(" settings " + module + " on " + o.Node + ": " + string(raw)) + return nil +} + +// PortsRewrite says what RewritePorts changed. +type PortsRewrite struct { + Places int +} + +// RewritePorts puts the node's foundation ports into the produced bundle, in place of the +// template's, byte for byte like every other rewrite — so the file keeps its comments and a person +// can read what was applied. A port left at its default is not touched, so a genesis on the +// defaults produces exactly the bundle it did before. +// +// Only the machine's side moves: the outer port of each mapping, the addresses the control plane +// dials on the machine's loopback, and the address nodes are told to dial. What a container listens +// on inside itself, and what an action reaches inside the store's own network, stay as they are. +func RewritePorts(r *Rewritten, p FoundationPorts, overlayRange string) (PortsRewrite, error) { + var out PortsRewrite + p = p.orDefaults() + d := DefaultPorts() + bundle := r.Bundle + var err error + + replace := func(from, to, what string) { + if err != nil || from == to { + return + } + bundle, err = replaceOnce(bundle, from, to, what) + out.Places++ + } + if p.Store != d.Store { + replace(`"ports": ["5432:5432"]`, fmt.Sprintf(`"ports": ["%d:5432"]`, p.Store), "the store's published port") + } + if p.Bus != d.Bus || p.AMQP != d.AMQP || p.Management != d.Management { + replace(`"ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"]`, + fmt.Sprintf(`"ports": ["%d:5671", "%d:5672", "127.0.0.1:%d:15672"]`, p.Bus, p.AMQP, p.Management), + "the broker's published ports") + } + if err != nil { + return out, err + } + + // The control plane runs on the machine's network and dials the store and the broker on its + // loopback, so its connection strings name the machine's ports. The schema step reaches the + // store inside the store's own network and keeps the container's port — so these are found by + // the control plane's environment, not by searching for the text. + control, cerr := controlPlaneIn(r.Declaration) + if cerr != nil { + return out, cerr + } + for _, key := range sortedKeys(control.Env) { + value := control.Env[key] + now := value + now = strings.ReplaceAll(now, "@127.0.0.1:5432/", fmt.Sprintf("@127.0.0.1:%d/", p.Store)) + now = strings.ReplaceAll(now, "@127.0.0.1:5672/", fmt.Sprintf("@127.0.0.1:%d/", p.AMQP)) + if strings.HasSuffix(now, "@127.0.0.1:15672") { + now = strings.TrimSuffix(now, "15672") + strconv.Itoa(p.Management) + } + if key == brokerAddressVar { + if host, port, splitErr := net.SplitHostPort(value); splitErr == nil && port == "5671" { + now = net.JoinHostPort(host, strconv.Itoa(p.Bus)) + } + } + if now != value { + replace(`"`+key+`": "`+value+`"`, `"`+key+`": "`+now+`"`, "the control plane's "+key) + } + } + if err != nil { + return out, err + } + + // The foundation's own filter, where the template carries one: it admits the bus and the + // registry from anywhere, on whatever port they are. + for _, f := range []struct{ def, now int }{{d.Bus, p.Bus}, {d.Registry, p.Registry}} { + if f.def == f.now { + continue + } + for _, form := range []string{"tcp dport %d accept", "ct original proto-dst %d accept"} { + from, to := fmt.Sprintf(form, f.def), fmt.Sprintf(form, f.now) + if n := bytes.Count(bundle, []byte(from)); n > 0 { + bundle = bytes.ReplaceAll(bundle, []byte(from), []byte(to)) + out.Places += n + } + } + } + + // The private network's range, when it is not the default, is the controller's to know. + if overlayRange != "" && overlayRange != DefaultOverlayRange { + replace(`"`+brokerAddressVar+`": `, + `"MESH_OVERLAY_CIDR": "`+overlayRange+`", + "`+brokerAddressVar+`": `, "where the control plane is told the private network's range") + if err != nil { + return out, err + } + } + + if out.Places == 0 { + return out, nil + } + parsed, perr := declaration.ParseFileTrusted(bundle) + if perr != nil { + return out, fmt.Errorf("the bundle stopped being a declaration after its ports were rewritten, which is this installer's fault: %w", perr) + } + r.Bundle, r.Declaration, r.Resources = bundle, parsed, len(parsed.Resources) + if c, cerr := controlPlaneIn(parsed); cerr == nil { + r.BrokerAddress = c.Env[brokerAddressVar] + } + return out, nil +} + +// PortsFree refuses a foundation port something else already holds, naming what holds it. What the +// mesh itself raised on an earlier run of genesis is not counted: ours says which holders are. +func PortsFree(ctx context.Context, run Runner, p FoundationPorts, ours func(reachable.Reach) bool) error { + out, err := run(ctx, "ss", "-Hltunp") + if err != nil { + return fmt.Errorf("cannot read which ports this machine holds, so the foundation's cannot be checked free: %w", err) + } + sockets := reachable.Sockets(out) + var published []reachable.Reach + if ps, err := run(ctx, "docker", "ps", "--format", "{{.Names}}\t{{.Ports}}"); err == nil { + published = reachable.Published(ps) + } + held := reachable.Merge(sockets, published) + + var problems []string + for _, n := range p.orDefaults().named() { + var by []string + for _, r := range held { + if r.Protocol != n.protocol || r.Port != n.port || ours(r) { + continue + } + holder := r.By + if holder == "" { + holder = "something ss does not name" + } + if r.Published { + holder = "the container " + r.By + } + if !contains(by, holder) { + by = append(by, holder) + } + } + if len(by) > 0 { + problems = append(problems, fmt.Sprintf("%s's port %s/%d is held by %s", + n.what, n.protocol, n.port, strings.Join(by, ", "))) + } + } + if len(problems) > 0 { + return fmt.Errorf("the foundation's ports must be free before anything is raised:\n - %s\n"+ + "Give it another with the matching flag (--store-port, --bus-port, --amqp-port, "+ + "--management-port, --registry-port, --packages-port, --hub-port); nothing was changed", + strings.Join(problems, "\n - ")) + } + return nil +} + +// OverlayClear refuses a private-network range that overlaps an address or a route the machine +// already has — a predecessor's tunnel still running — naming the interface. The mesh's own +// interface is not counted. +func OverlayClear(ctx context.Context, run Runner, overlayRange string) error { + if overlayRange == "" { + overlayRange = DefaultOverlayRange + } + _, mine, err := net.ParseCIDR(overlayRange) + if err != nil { + return fmt.Errorf("the private network's range %q is not a range: %w", overlayRange, err) + } + var clashes []string + if out, err := run(ctx, "ip", "-o", "addr", "show"); err == nil { + for _, line := range strings.Split(out, "\n") { + f := strings.Fields(line) + // 3: wg0 inet 10.42.0.1/24 scope global wg0 + if len(f) < 4 || (f[2] != "inet" && f[2] != "inet6") { + continue + } + iface := strings.TrimSuffix(f[1], ":") + if clash(mine, f[3]) && iface != meshInterface { + clashes = append(clashes, fmt.Sprintf("%s holds %s", iface, f[3])) + } + } + } else { + return fmt.Errorf("cannot read this machine's addresses to check the private network's range: %w", err) + } + if out, err := run(ctx, "ip", "-o", "route", "show"); err == nil { + for _, line := range strings.Split(out, "\n") { + f := strings.Fields(line) + // 10.42.0.0/16 dev wg0 proto kernel scope link src 10.42.0.1 + if len(f) < 3 || f[0] == "default" { + continue + } + iface := "" + for i := range f { + if f[i] == "dev" && i+1 < len(f) { + iface = f[i+1] + } + } + if iface != meshInterface && clash(mine, f[0]) { + clashes = append(clashes, fmt.Sprintf("%s routes %s", iface, f[0])) + } + } + } + if len(clashes) > 0 { + return fmt.Errorf("the private network's range %s overlaps what this machine already has: %s.\n"+ + "A tunnel a predecessor still runs would take the mesh's traffic. Give another range with "+ + "--overlay-range; nothing was changed", overlayRange, strings.Join(clashes, "; ")) + } + return nil +} + +// meshInterface is the private network's own interface, which a re-run finds holding its range. +const meshInterface = "mesh0" + +func clash(mine *net.IPNet, other string) bool { + if !strings.Contains(other, "/") { + if ip := net.ParseIP(other); ip != nil { + return mine.Contains(ip) + } + return false + } + ip, theirs, err := net.ParseCIDR(other) + if err != nil { + return false + } + return mine.Contains(theirs.IP) || theirs.Contains(mine.IP) || mine.Contains(ip) +} + +// NamesFree refuses a foundation or bundle container name that a container already has, when no +// host made that container and this node has no record of it — a predecessor's container under the +// mesh's name, which raising the foundation would replace. +func NamesFree(ctx context.Context, run Runner, names []string, known store.State) error { + var taken []string + sorted := append([]string{}, names...) + sort.Strings(sorted) + for _, name := range sorted { + out, err := run(ctx, "docker", "inspect", "--format", + "{{index .Config.Labels \"mesh-host.spec\"}}", name) + if err != nil { + continue // no such container + } + label := strings.TrimSpace(out) + if label != "" && label != "" { + continue + } + if known.Recorded(string(declaration.TypeContainer), name) { + continue + } + if name == giteaBootstrap && len(known.Resources) > 0 { + // Genesis raises the package registry itself, by hand and before the host records + // anything of it, so on a re-run it is found under its own name with no label and no + // record. A machine that carries what an earlier genesis raised made it. + continue + } + taken = append(taken, name) + } + if len(taken) > 0 { + return fmt.Errorf("this machine already runs a container under the name the foundation uses, "+ + "and nothing of the mesh's made it: %s.\nRaising the foundation would replace it. Rename or "+ + "stop it first; nothing was changed", strings.Join(taken, ", ")) + } + return nil +} + +// CheckTheMachine is every check genesis makes before raising anything that this machine does not +// already hold what the foundation needs: its ports, its private network's range, its containers' +// names (novox/hq ADR 0100). A re-run of genesis finds the foundation it raised and does not count +// it. +func CheckTheMachine(ctx context.Context, o Options, run Runner, bundle *declaration.Declaration, + say func(string)) error { + known, err := store.Load(o.State) + if err != nil { + return err + } + rerun := len(known.Resources) > 0 + names := foundationNames(bundle) + mine := map[string]bool{} + for _, n := range names { + mine[n] = true + } + p := o.Ports.orDefaults() + ours := func(r reachable.Reach) bool { + switch { + case mine[r.By]: + return true + case !rerun: + return false + case r.By == "gitea" && r.Port == p.Packages: + // The package registry runs on the machine's network, so ss names its process. + return true + case r.By == "" && r.Protocol == "udp" && r.Port == p.Hub: + // The private network's hub is a kernel interface and has no process. + return true + } + return false + } + if err := PortsFree(ctx, run, p, ours); err != nil { + return err + } + say(fmt.Sprintf(" ports free store %d, bus %d, amqp %d, management %d, registry %d, packages %d, hub %d/udp", + p.Store, p.Bus, p.AMQP, p.Management, p.Registry, p.Packages, p.Hub)) + if err := OverlayClear(ctx, run, o.OverlayRange); err != nil { + return err + } + if err := NamesFree(ctx, run, names, known); err != nil { + return err + } + return nil +} + +// foundationNames are the containers the foundation and genesis raise under fixed names. +func foundationNames(bundle *declaration.Declaration) []string { + names := []string{ControlPlaneModule, giteaBootstrap, "mesh-registry"} + for _, n := range containerNames(bundle) { + if !contains(names, n) { + names = append(names, n) + } + } + sort.Strings(names) + return names +} diff --git a/internal/bootstrap/ports_test.go b/internal/bootstrap/ports_test.go new file mode 100644 index 0000000..a94371a --- /dev/null +++ b/internal/bootstrap/ports_test.go @@ -0,0 +1,299 @@ +package bootstrap + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/reachable" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0100: the foundation's ports are the node's — inputs to genesis, checked free, +// rewritten into the bundle, and handed to the controller as the node's settings. + +func producedBundle(t *testing.T) Rewritten { + t.Helper() + template, err := os.ReadFile("../../examples/foundation-first-node.lock") + if err != nil { + t.Skip("no example bundle beside this checkout") + } + r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32)) + if err != nil { + t.Fatal(err) + } + if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil { + t.Fatal(err) + } + return r +} + +func containerNamed(d *declaration.Declaration, name string) *declaration.Container { + for _, r := range d.Resources { + if c, ok := r.(*declaration.Container); ok && c.Name == name { + return c + } + } + return nil +} + +func TestTheDefaultPortsLeaveTheBundleAsItWas(t *testing.T) { + r := producedBundle(t) + before := string(r.Bundle) + got, err := RewritePorts(&r, DefaultPorts(), DefaultOverlayRange) + if err != nil { + t.Fatal(err) + } + if got.Places != 0 || string(r.Bundle) != before { + t.Errorf("the default ports rewrote %d place(s)", got.Places) + } +} + +func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) { + r := producedBundle(t) + p := FoundationPorts{Store: 5433, Bus: 5771, AMQP: 5772, Management: 15673, Registry: 5100} + got, err := RewritePorts(&r, p, "10.77.0.0/16") + if err != nil { + t.Fatal(err) + } + if got.Places == 0 { + t.Fatal("nothing was rewritten") + } + storeC := containerNamed(r.Declaration, "mesh-store") + if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" { + t.Errorf("the store publishes %v", storeC.Ports) + } + broker := containerNamed(r.Declaration, "mesh-broker") + if strings.Join(broker.Ports, " ") != "5771:5671 5772:5672 127.0.0.1:15673:15672" { + t.Errorf("the broker publishes %v", broker.Ports) + } + control, err := controlPlaneIn(r.Declaration) + if err != nil { + t.Fatal(err) + } + for key, value := range control.Env { + if strings.HasPrefix(key, "MESH_STORE_") && !strings.Contains(value, "@127.0.0.1:5433/") { + t.Errorf("%s still dials %s", key, value) + } + } + if !strings.Contains(control.Env["MESH_BROKER_AMQP"], "@127.0.0.1:5772/") || + !strings.HasSuffix(control.Env["MESH_BROKER_MANAGEMENT"], "@127.0.0.1:15673") { + t.Errorf("the broker is dialled at %s and %s", control.Env["MESH_BROKER_AMQP"], control.Env["MESH_BROKER_MANAGEMENT"]) + } + if control.Env["MESH_BROKER_ADDRESS"] != "192.0.2.10:5771" || r.BrokerAddress != "192.0.2.10:5771" { + t.Errorf("nodes are told to dial %s (%s)", control.Env["MESH_BROKER_ADDRESS"], r.BrokerAddress) + } + if control.Env["MESH_OVERLAY_CIDR"] != "10.77.0.0/16" { + t.Errorf("the control plane is told the range %q", control.Env["MESH_OVERLAY_CIDR"]) + } + // The schema step reaches the store inside its own network, on the container's port. + text := string(r.Bundle) + if !strings.Contains(text, `MESH_STORE_INVENTORY=postgres://postgres:s@127.0.0.1:5432/inventory`) { + t.Error("the schema step's connection, inside the store's network, was moved off the container's port") + } + for _, want := range []string{"tcp dport 5771 accept", "ct original proto-dst 5771 accept", + "tcp dport 5100 accept", "ct original proto-dst 5100 accept"} { + if !strings.Contains(text, want) { + t.Errorf("the base filter does not say %q", want) + } + } + if strings.Contains(text, "dport 5671 accept") || strings.Contains(text, "dport 5000 accept") { + t.Error("the base filter still admits a default port") + } +} + +func TestATemplateThatDoesNotSayItsPortsAsExpectedIsRefused(t *testing.T) { + r := producedBundle(t) + r.Bundle = []byte(strings.Replace(string(r.Bundle), `"ports": ["5432:5432"]`, `"ports": [ "5432:5432" ]`, 1)) + if _, err := RewritePorts(&r, FoundationPorts{Store: 5433}, ""); err == nil { + t.Error("a store port the installer could not find was silently left") + } +} + +func TestTwoThingsOnOnePortAreRefused(t *testing.T) { + p := DefaultPorts() + p.Registry = p.Store + if err := p.Check(); err == nil { + t.Error("the registry and the store were both given one port") + } + p = DefaultPorts() + p.Hub = 5432 // udp, beside the store's tcp: two different ports + if err := p.Check(); err != nil { + t.Errorf("a udp port beside a tcp one of the same number was refused: %v", err) + } +} + +// machineRunner answers ss, docker ps, docker inspect and ip from fixtures. +type machineRunner struct { + ss, ps, addrs, routes string + unlabelled map[string]bool + labelled map[string]bool +} + +func (m machineRunner) run(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "ss": + return m.ss, nil + case name == "docker" && args[0] == "ps": + return m.ps, nil + case name == "docker" && args[0] == "inspect": + n := args[len(args)-1] + if m.labelled[n] { + return "abc\n", nil + } + if m.unlabelled[n] { + return "\n", nil + } + return "", errors.New("no such container") + case name == "ip" && args[1] == "addr": + return m.addrs, nil + case name == "ip" && args[1] == "route": + return m.routes, nil + } + return "", nil +} + +func noneOurs(reachable.Reach) bool { return false } + +func TestABusyPortIsRefusedNamingItsHolder(t *testing.T) { + m := machineRunner{ + ss: "tcp LISTEN 0 4096 0.0.0.0:5000 0.0.0.0:* users:((\"docker-proxy\",pid=1,fd=7))\n" + + "tcp LISTEN 0 4096 127.0.0.1:15672 0.0.0.0:* users:((\"beam.smp\",pid=2,fd=7))\n", + ps: "predecessor-registry\t0.0.0.0:5000->5000/tcp\n", + } + err := PortsFree(context.Background(), m.run, DefaultPorts(), noneOurs) + if err == nil { + t.Fatal("held ports were not refused") + } + for _, want := range []string{"predecessor-registry", "beam.smp", "tcp/5000", "tcp/15672", "--registry-port"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + p := DefaultPorts() + p.Registry, p.Management = 5100, 15673 + if err := PortsFree(context.Background(), m.run, p, noneOurs); err != nil { + t.Errorf("other ports given and still refused: %v", err) + } +} + +func TestTheFoundationsOwnContainersAreNotCountedOnARerun(t *testing.T) { + m := machineRunner{ + ss: "tcp LISTEN 0 4096 0.0.0.0:5432 0.0.0.0:* users:((\"docker-proxy\",pid=1,fd=7))\n", + ps: "mesh-store\t0.0.0.0:5432->5432/tcp\n", + } + ours := func(r reachable.Reach) bool { return r.By == "mesh-store" } + if err := PortsFree(context.Background(), m.run, DefaultPorts(), ours); err != nil { + t.Errorf("the foundation's own store was counted as holding its port: %v", err) + } +} + +func TestAnOverlappingTunnelIsRefusedNamingItsInterface(t *testing.T) { + m := machineRunner{ + addrs: "1: lo inet 127.0.0.1/8 scope host lo\n5: wg0 inet 10.42.3.1/24 scope global wg0\n7: mesh0 inet 10.42.0.1/16 scope global mesh0\n", + routes: "default via 192.0.2.1 dev eth0\n10.42.3.0/24 dev wg0 proto kernel scope link src 10.42.3.1\n", + } + err := OverlayClear(context.Background(), m.run, "") + if err == nil || !strings.Contains(err.Error(), "wg0") || strings.Contains(err.Error(), "mesh0") { + t.Fatalf("the overlap was not named by its interface alone: %v", err) + } + if err := OverlayClear(context.Background(), m.run, "10.77.0.0/16"); err != nil { + t.Errorf("a clear range was refused: %v", err) + } +} + +func TestAPredecessorsContainerUnderTheMeshsNameIsRefused(t *testing.T) { + m := machineRunner{unlabelled: map[string]bool{"mesh-registry": true}, labelled: map[string]bool{"mesh-store": true}} + err := NamesFree(context.Background(), m.run, []string{"mesh-store", "mesh-registry", "mesh-broker"}, store.State{}) + if err == nil || !strings.Contains(err.Error(), "mesh-registry") || strings.Contains(err.Error(), "mesh-store") { + t.Fatalf("names: %v", err) + } + known := store.State{Resources: []store.Applied{{ID: "x", Type: "container", Target: "mesh-registry"}}} + if err := NamesFree(context.Background(), m.run, []string{"mesh-registry"}, known); err != nil { + t.Errorf("a container this node has a record of was refused: %v", err) + } +} + +// controlRecorder is a control plane that answers everything and writes down what it was told, +// with the content of every settings file carried to it. +type controlRecorder struct { + told []string + settings map[string]string +} + +func (c *controlRecorder) run(_ context.Context, name string, args ...string) (string, error) { + if name == "docker" && args[0] == "cp" { + raw, _ := os.ReadFile(args[1]) + if strings.HasSuffix(args[2], "-settings.json") { + c.settings[filepath.Base(args[2])] = string(raw) + } + return "", nil + } + if name == "docker" && args[0] == "exec" { + c.told = append(c.told, strings.Join(args[3:], " ")) + } + return "", nil +} + +func (c *controlRecorder) index(prefix string) int { + for i, t := range c.told { + if strings.HasPrefix(t, prefix) { + return i + } + } + return -1 +} + +func TestTheNodesPortsAreSetBeforeTheModuleIsPushed(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + c := &controlRecorder{settings: map[string]string{}} + control := controlPlane{container: "temp-mesh-controller", run: c.run, timeout: time.Second} + o := Options{Node: "anchor", Ports: FoundationPorts{Registry: 5100}, Wait: time.Second} + if _, err := installModule(context.Background(), o, control, RegistryModule, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + set, push := c.index("settings set distribution"), c.index("push anchor") + if set < 0 || push < 0 || set > push { + t.Fatalf("settings were not set before the push: %v", c.told) + } + if add := c.index("module add"); add > set { + t.Errorf("settings were set before the module existed: %v", c.told) + } + if !strings.Contains(c.told[set], "--node anchor") { + t.Errorf("the settings are not the node's: %s", c.told[set]) + } + if got := c.settings["distribution-settings.json"]; got != `{"ports":{"5000":5100}}` { + t.Errorf("the registry was told %s", got) + } +} + +func TestAGenesisOnTheDefaultsSetsNoSettings(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + c := &controlRecorder{settings: map[string]string{}} + control := controlPlane{container: "temp-mesh-controller", run: c.run, timeout: time.Second} + o := Options{Node: "anchor", Wait: time.Second} + if _, err := installModule(context.Background(), o, control, RegistryModule, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + if c.index("settings") >= 0 { + t.Errorf("a converged genesis on the default ports set settings: %v", c.told) + } +} + +func TestARerunOfGenesisFindsItsOwnPackageRegistry(t *testing.T) { + // Raised by genesis itself with no label and no record, so a re-run finds it unlabelled. + m := machineRunner{unlabelled: map[string]bool{giteaBootstrap: true}} + rerun := store.State{Resources: []store.Applied{{ID: "store", Type: "container", Target: "mesh-store"}}} + if err := NamesFree(context.Background(), m.run, []string{giteaBootstrap, "mesh-store"}, rerun); err != nil { + t.Errorf("a re-run refused the package registry genesis raised: %v", err) + } + // On a machine genesis never ran on, a container under that name is a predecessor's. + if err := NamesFree(context.Background(), m.run, []string{giteaBootstrap}, store.State{}); err == nil { + t.Error("a container under the package registry's name on a fresh machine was not refused") + } +} diff --git a/internal/bootstrap/preflight.go b/internal/bootstrap/preflight.go index 5275aeb..7b8d145 100644 --- a/internal/bootstrap/preflight.go +++ b/internal/bootstrap/preflight.go @@ -105,6 +105,11 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte if err := waitForRuntime(ctx, d.Run, o.Timeout, o.Wait, say); err != nil { return nil, err } + // A converged genesis refuses a machine in use (novox/hq ADR 0100) — asked once the runtime + // answers, so what it runs can be counted, and before anything changes. + if err := RefuseAMachineInUse(ctx, o, d.Run, say); err != nil { + return nil, err + } // 4. Can this machine reach what the bundle's images come from? // diff --git a/internal/bootstrap/retire.go b/internal/bootstrap/retire.go index 2ed13f8..af41291 100644 --- a/internal/bootstrap/retire.go +++ b/internal/bootstrap/retire.go @@ -132,18 +132,29 @@ func RetireTheTemporaryControlPlane(ctx context.Context, o Options, sys system.S // where the comment explaining it lives. A comment that outlives the thing it describes is worse // than no comment: it is the file telling somebody the machine has a control plane it does not. func removeResource(bundle []byte, id string) ([]byte, error) { + previous, from, to, err := resourceAt(bundle, id) + if err != nil { + return nil, err + } + return cut(bundle, previous, from, to), nil +} + +// resourceAt finds one resource's object in a bundle's text by its id: where the one before it +// ended, and where it starts and ends — comments and strings skipped, so an id quoted in a comment +// or a command is never mistaken for the resource. +func resourceAt(bundle []byte, id string) (previous, from, to int, err error) { array := indexOutsideStrings(bundle, `"resources"`) if array < 0 { - return nil, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it") + return 0, 0, 0, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it") } open := indexOutsideStrings(bundle[array:], "[") if open < 0 { - return nil, fmt.Errorf("this bundle's resources are not a list") + return 0, 0, 0, fmt.Errorf("this bundle's resources are not a list") } open += array - depth, from := 0, -1 - previous := open + depth := 0 + from, previous = -1, open inString, escaped, inLine, inBlock := false, false, false, false for i := open + 1; i < len(bundle); i++ { c := bundle[i] @@ -181,16 +192,16 @@ func removeResource(bundle []byte, id string) ([]byte, error) { break } if isResource(bundle[from:i+1], id) { - return cut(bundle, previous, from, i+1), nil + return previous, from, i + 1, nil } previous = i + 1 from = -1 case c == ']' && depth == 0: - return nil, fmt.Errorf( + return 0, 0, 0, fmt.Errorf( "this bundle declares no %q, so there is nothing to take out of it", id) } } - return nil, fmt.Errorf("this bundle's resources list does not end") + return 0, 0, 0, fmt.Errorf("this bundle's resources list does not end") } // isResource reports whether one resource's text is the one wanted. diff --git a/internal/bootstrap/testdata/fresh-machine-listeners.txt b/internal/bootstrap/testdata/fresh-machine-listeners.txt new file mode 100644 index 0000000..defcf66 --- /dev/null +++ b/internal/bootstrap/testdata/fresh-machine-listeners.txt @@ -0,0 +1,12 @@ +udp UNCONN 0 0 0.0.0.0:5353 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=17)) +udp UNCONN 0 0 0.0.0.0:5355 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=13)) +udp UNCONN 0 0 127.0.0.54:53 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=24)) +udp UNCONN 0 0 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=22)) +udp UNCONN 0 0 [::]:5353 [::]:* users:(("systemd-resolve",pid=262,fd=18)) +udp UNCONN 0 0 [::]:5355 [::]:* users:(("systemd-resolve",pid=262,fd=15)) +udp UNCONN 0 0 [fe80::1266:6aff:fe24:628d]%enp5s0:546 [::]:* users:(("systemd-network",pid=272,fd=36)) +tcp LISTEN 0 4096 127.0.0.1:39473 0.0.0.0:* users:(("containerd",pid=394,fd=14)) +tcp LISTEN 0 4096 0.0.0.0:5355 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=14)) +tcp LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=23)) +tcp LISTEN 0 4096 127.0.0.54:53 0.0.0.0:* users:(("systemd-resolve",pid=262,fd=25)) +tcp LISTEN 0 4096 [::]:5355 [::]:* users:(("systemd-resolve",pid=262,fd=16)) diff --git a/internal/declaration/adoption_test.go b/internal/declaration/adoption_test.go new file mode 100644 index 0000000..1ca0d3b --- /dev/null +++ b/internal/declaration/adoption_test.go @@ -0,0 +1,107 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq ADR 0100: every declaration says whether the node is adopted and which of its +// modules are taken, and the host refuses one it cannot read that from unambiguously. + +const adoptedResources = `"resources":[ + {"id":"hello-web.page","type":"file","path":"/var/lib/hello-web/index.html","content":"a\n"}, + {"id":"hello-web.server","type":"container","name":"hello-web","image":"sha256:` + sixtyFour + `"}, + {"id":"hello-web.data","type":"directory","path":"/var/lib/hello-web"} + ]` + +const sixtyFour = "0000000000000000000000000000000000000000000000000000000000000000" + +func TestAnAdoptionIsReadWithTheDeclaration(t *testing.T) { + d, err := Parse([]byte(`{"adoption":{"taken":["postgres"],"untaken":{"hello-web":["hello-web.page","hello-web.server"]}}, + "declaration":1,` + adoptedResources + `}`)) + if err != nil { + t.Fatal(err) + } + if d.Adoption == nil { + t.Fatal("the adoption was dropped") + } + if len(d.Adoption.Taken) != 1 || d.Adoption.Taken[0] != "postgres" { + t.Errorf("taken read as %v", d.Adoption.Taken) + } + if module, ok := d.Adoption.UntakenModuleOf("hello-web.server"); !ok || module != "hello-web" { + t.Errorf("the container's untaken module read as %q, %v", module, ok) + } + if _, ok := d.Adoption.UntakenModuleOf("hello-web.data"); ok { + t.Error("a resource the adoption does not name was said to be untaken") + } +} + +func TestADeclarationWithNoAdoptionIsConverged(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,` + adoptedResources + `}`)) + if err != nil { + t.Fatal(err) + } + if d.Adoption != nil { + t.Errorf("a declaration saying nothing about adoption read as adopted: %+v", d.Adoption) + } + if _, ok := d.Adoption.UntakenModuleOf("hello-web.page"); ok { + t.Error("a converged node has an untaken module") + } +} + +func TestAnAdoptionNamingAnUnknownIDIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.missing"]}}, + "declaration":1,`+adoptedResources+`}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "hello-web.missing") { + t.Errorf("the unknown id was not named: %v", refusal.Problems) + } +} + +func TestAnAdoptionMayNameAResourceOfAnyKind(t *testing.T) { + // A directory, a service or an action can reach what was found as surely as a file can, so + // the controller lists every resource of an untaken module (novox/hq ADR 0103). + d, err := Parse([]byte(`{"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.data"]}}, + "declaration":1,` + adoptedResources + `}`)) + if err != nil { + t.Fatalf("a directory of an untaken module was refused: %v", err) + } + if module, ok := d.Adoption.UntakenModuleOf("hello-web.data"); !ok || module != "hello-web" { + t.Errorf("the directory is not its module's: %q %v", module, ok) + } +} + +func TestAnIDUnderTwoModulesIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"a":["hello-web.page"],"b":["hello-web.page"]}}, + "declaration":1,`+adoptedResources+`}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "both") { + t.Errorf("an id under two modules was accepted: %v", refusal.Problems) + } +} + +func TestAModuleBothTakenAndUntakenIsRefused(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":["hello-web"],"untaken":{"hello-web":["hello-web.page"]}}, + "declaration":1,`+adoptedResources+`}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "both taken and untaken") { + t.Errorf("a module both taken and untaken was accepted: %v", refusal.Problems) + } +} + +func TestTheMeshsOwnResourcesAreNeverUntaken(t *testing.T) { + refusal := refusalFor(t, `{"adoption":{"taken":[],"untaken":{"x":["adoption.guard"]}}, + "declaration":1,"resources":[ + {"id":"adoption.guard","type":"file","path":"/etc/mesh/guard.nft","content":"x"}]}`) + if !strings.Contains(strings.Join(refusal.Problems, "\n"), "belongs to no module") { + t.Errorf("an adoption. id was accepted as untaken: %v", refusal.Problems) + } +} + +func TestAnAdoptionWithAnUnknownFieldIsRefused(t *testing.T) { + refusalFor(t, `{"adoption":{"taken":[],"held":["x"]},"declaration":1,`+adoptedResources+`}`) +} + +func TestACarriedBundleCannotSayTheNodeIsAdopted(t *testing.T) { + _, err := ParseTrusted([]byte(`{"adoption":{"taken":[]},"declaration":1,` + adoptedResources + `}`)) + if err == nil || !strings.Contains(err.Error(), "only the mesh can say") { + t.Fatalf("a bundle claiming adoption was not refused: %v", err) + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index cd7cbd3..c0bc7af 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -78,6 +78,12 @@ const ( // cadence. Tools, hooks and event consumers are not separate modes: they are loaded by a tool // host, which is itself a process that stays up. TypeProcess Type = "process" + + // TypeOpening is a port the mesh needs reachable on an adopted node, converged through the + // firewall found there in that firewall's own terms (novox/hq ADR 0100). A state, not a + // command: the host adds the rule it marks as the mesh's when it is missing, and removes only + // what it marked — which is what lets it travel over the link. + TypeOpening Type = "opening" ) // Resource is one thing that should be true of the machine. @@ -148,6 +154,13 @@ type File struct { // (ADR 0030), and it does not overwrite that either. CreateOnce bool `json:"create-once,omitempty"` + // Into says the file is shared with software the mesh did not install, and the content is + // the mesh's part of it: written into what is there, never over it (novox/hq ADR 0102). Only + // "json" is spoken — the content is a JSON object whose keys the host sets in the file's + // object, keeping every other key as it found it and recording what each of its keys held + // before, so undeclaring the file gives those back. + Into string `json:"into,omitempty"` + // Sealed is content encrypted to this node's sealing key, for a file the mesh must deliver // without being able to read. // @@ -218,6 +231,24 @@ func (f *File) validate(where string, _ bool) []string { if f.Path == "" { problems = append(problems, where+": a file needs a path") } + switch f.Into { + case "": + case IntoJSON: + var object map[string]json.RawMessage + if err := json.Unmarshal([]byte(f.Content), &object); err != nil || object == nil { + problems = append(problems, where+ + ": a file written into JSON carries a JSON object of the keys it sets") + } + if f.Sealed != "" || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce { + problems = append(problems, where+ + ": a file written into says only its keys, in content — not sealed, bytes, "+ + "secrets or create-once") + } + default: + problems = append(problems, fmt.Sprintf( + "%s: into %q; a file is written into \"json\", or omits it to be written whole", + where, f.Into)) + } var said []string for name, value := range map[string]string{ "content": f.Content, "sealed": f.Sealed, "bytes": f.Bytes, @@ -580,6 +611,12 @@ type Service struct { // would be an action, and the link may not carry one (novox/hq ADR 0005) — so this is not a // way around that rule, it is the shape the rule leaves. RestartOn []string `json:"restart-on,omitempty"` + + // ReloadOn names resources whose change means this service must be reloaded — for a service + // that re-reads its configuration when told to, where a restart would stop what it runs: the + // container runtime, whose restart stops every container on the machine (novox/hq ADR 0102). + // A change that is also in RestartOn restarts it, which covers a reload. + ReloadOn []string `json:"reload-on,omitempty"` } func (s *Service) Identity() string { return s.ID } @@ -603,6 +640,77 @@ func (s *Service) validate(where string, _ bool) []string { return problems } +// IntoJSON is the one structured format a file is written into. +const IntoJSON = "json" + +// Opening is a port reachable on an adopted node, from where, and on which path. +// +// **From** is everywhere or mesh — the private network, by its interface. **Path** is incoming, +// for something listening on the machine, or forwarded, for a published container port: the found +// firewall sees a published port after the runtime has translated it, so a forwarded opening names +// the container's own port in To as well as the machine's in Port. +type Opening struct { + ID string `json:"id"` + Type Type `json:"type"` + Port int `json:"port"` + Protocol string `json:"protocol"` + From string `json:"from"` + Path string `json:"path"` + To int `json:"to,omitempty"` +} + +// Where an opening admits from, and the path it is on. +const ( + FromEverywhere = "everywhere" + FromMesh = "mesh" + PathIncoming = "incoming" + PathForwarded = "forwarded" +) + +func (o *Opening) Identity() string { return o.ID } +func (o *Opening) Kind() Type { return TypeOpening } + +func (o *Opening) Target() string { + if o.Path == PathForwarded { + return fmt.Sprintf("%s/%d forwarded to %d from %s", o.Protocol, o.Port, o.To, o.From) + } + return fmt.Sprintf("%s/%d %s from %s", o.Protocol, o.Port, o.Path, o.From) +} + +func (o *Opening) validate(where string, _ bool) []string { + var problems []string + if o.Port < 1 || o.Port > 65535 { + problems = append(problems, fmt.Sprintf("%s: an opening's port is 1-65535, not %d", where, o.Port)) + } + if o.Protocol != "tcp" && o.Protocol != "udp" { + problems = append(problems, fmt.Sprintf("%s: an opening is tcp or udp, not %q", where, o.Protocol)) + } + if o.From != FromEverywhere && o.From != FromMesh { + problems = append(problems, fmt.Sprintf( + "%s: an opening is from %q or %q, not %q", where, FromEverywhere, FromMesh, o.From)) + } + switch o.Path { + case PathIncoming: + if o.To != 0 { + problems = append(problems, where+ + ": an incoming opening names no container port; only a forwarded one does") + } + case PathForwarded: + if o.To < 1 || o.To > 65535 { + problems = append(problems, where+ + ": a forwarded opening names the container's port it reaches, as to, 1-65535") + } + default: + problems = append(problems, fmt.Sprintf( + "%s: an opening's path is %q or %q, not %q", where, PathIncoming, PathForwarded, o.Path)) + } + if !strings.HasPrefix(o.ID, AdoptionPrefix) { + problems = append(problems, fmt.Sprintf( + "%s: an opening is the mesh's own, so its id starts %q", where, AdoptionPrefix)) + } + return problems +} + // Package is a package that should be present. // // Present is the whole of what it asserts, never a version: version is the package manager's @@ -810,6 +918,8 @@ func newOf(t Type) Resource { return &Access{} case TypeProcess: return &Process{} + case TypeOpening: + return &Opening{} } return nil } @@ -818,7 +928,7 @@ func newOf(t Type) Resource { func Vocabulary() []Type { return []Type{ TypeAccess, TypeAction, TypeArchive, TypeContainer, TypeDirectory, TypeFile, - TypeNetwork, TypePackage, TypeProcess, TypeService, TypeUser, + TypeNetwork, TypeOpening, TypePackage, TypeProcess, TypeService, TypeUser, } } @@ -832,6 +942,99 @@ type Declaration struct { // Resources, in the order they are applied. The host does not sort them: ordering is a // decision, and deciding is not what the host does (novox/hq ADR 0005). Resources []Resource + + // Adoption says this node is adopted, and which of its modules have been taken. Nil is a + // converged node — which is every node the mesh raised before adoption existed, and so the + // only form an older controller ever sends (novox/hq ADR 0100). + Adoption *Adoption +} + +// Adoption is a node's mode, as the controller records it: the node is adopted, and these are +// the modules taken on it so far (novox/hq ADR 0100). +// +// **Authoritative, and only ever stated by the controller.** A host does not work out whether it +// is adopted; it is told, in every declaration, so a host restarted from the declaration it kept +// is in the same mode it was in before. +// +// Untaken names, per module assigned here and not yet taken, the ids of its resources. Any kind +// may be listed: a file, a directory, a service's unit or a container can already be on the +// machine, and an action run inside a held container reaches what was found (novox/hq ADR 0103); +// the host decides per kind what can be held. The host cannot split a resource id into its +// module, because module names may contain dots, so the controller says which ids belong to which +// module rather than leaving the host to guess. +type Adoption struct { + Taken []string `json:"taken"` + Untaken map[string][]string `json:"untaken,omitempty"` +} + +// AdoptionPrefix is the id prefix of what the mesh itself declares because a node is adopted — +// its openings and its guard. Nothing under it belongs to a module, so none of it is ever held. +const AdoptionPrefix = "adoption." + +// UntakenModuleOf says which untaken module declares a resource, if any. +func (a *Adoption) UntakenModuleOf(id string) (string, bool) { + if a == nil { + return "", false + } + for module, ids := range a.Untaken { + if slices.Contains(ids, id) { + return module, true + } + } + return "", false +} + +// checkAdoption holds what an adoption says against the resources beside it. Every problem is a +// refusal: a host that misread which module is untaken would replace a predecessor's service the +// operator never took. +func checkAdoption(a *Adoption, resources []Resource, allowActions bool) []string { + if a == nil { + return nil + } + if allowActions { + // The bundle is carried with the binary and raises a foundation before any mesh exists. + // Whether a node is adopted is the controller's record, and a bundle that claimed it would + // be the host deciding its own mode (novox/hq ADR 0100). + return []string{"a carried bundle says the node is adopted, and only the mesh can say " + + "that: a node's mode is the controller's record, sent in every declaration"} + } + kinds := map[string]Type{} + for _, r := range resources { + kinds[r.Identity()] = r.Kind() + } + var problems []string + for _, module := range a.Taken { + if _, both := a.Untaken[module]; both { + problems = append(problems, fmt.Sprintf( + "adoption: the module %q is said to be both taken and untaken", module)) + } + } + owner := map[string]string{} + modules := make([]string, 0, len(a.Untaken)) + for module := range a.Untaken { + modules = append(modules, module) + } + sort.Strings(modules) + for _, module := range modules { + for _, id := range a.Untaken[module] { + if strings.HasPrefix(id, AdoptionPrefix) { + problems = append(problems, fmt.Sprintf( + "adoption: %q is the mesh's own and belongs to no module, so it cannot be untaken", id)) + continue + } + if first, twice := owner[id]; twice { + problems = append(problems, fmt.Sprintf( + "adoption: %q is said to belong to both %q and %q", id, first, module)) + continue + } + owner[id] = module + if _, declared := kinds[id]; !declared { + problems = append(problems, fmt.Sprintf( + "adoption: %q of the untaken module %q is not in this declaration", id, module)) + } + } + } + return problems } // RefusalError refuses a whole declaration, naming every problem at once. @@ -872,6 +1075,7 @@ func ParseTrusted(raw []byte) (*Declaration, error) { return parse(raw, true) } type envelope struct { Version int `json:"declaration"` For string `json:"for,omitempty"` + Adoption *Adoption `json:"adoption,omitempty"` Resources []json.RawMessage `json:"resources"` } @@ -889,7 +1093,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { env.Version, Version)}} } - d := &Declaration{Version: env.Version, For: env.For} + d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption} var problems []string if len(env.Resources) == 0 { @@ -952,6 +1156,19 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { problems = append(problems, resource.validate(where, allowActions)...) d.Resources = append(d.Resources, resource) } + problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...) + if env.Adoption == nil { + for _, r := range d.Resources { + if r.Kind() == TypeOpening { + // On a converged node the mesh's own filter admits what is declared, and the + // found firewall is retired; an opening there would be a rule in a firewall the + // mesh has disabled (novox/hq ADR 0100). + problems = append(problems, fmt.Sprintf( + "resource %q: an opening is for an adopted node, and this declaration does not "+ + "say the node is adopted", r.Identity())) + } + } + } if len(problems) > 0 { return nil, &RefusalError{Problems: problems} diff --git a/internal/declaration/declaration_test.go b/internal/declaration/declaration_test.go index b2fd73e..d0823e0 100644 --- a/internal/declaration/declaration_test.go +++ b/internal/declaration/declaration_test.go @@ -266,7 +266,7 @@ func TestAFieldTheNewTypesDoNotUseIsRefused(t *testing.T) { } } -func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) { +func TestTheVocabularyIsTheTwelveShapesTheMeshNeeds(t *testing.T) { // Six of them the bootstrap uses (novox/hq 07-the-foundation.md), and removing one is a // failing test rather than a discovery during a first-node install. // @@ -282,7 +282,7 @@ func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) { } for _, want := range []Type{ TypeDirectory, TypeFile, TypeService, TypePackage, TypeContainer, TypeAction, - TypeUser, TypeArchive, TypeNetwork, TypeAccess, TypeProcess, + TypeUser, TypeArchive, TypeNetwork, TypeAccess, TypeProcess, TypeOpening, } { if !speaks[want] { t.Errorf("the host no longer speaks %q", want) @@ -309,8 +309,12 @@ func TestTheVocabularyIsTheElevenShapesTheMeshNeeds(t *testing.T) { // It is a full-host shape rather than a portable one: it needs a process supervisor to install // into. It does NOT need a container runtime, which is the point — only software that // genuinely needs isolation asks for a container. - if len(speaks) != 11 { - t.Errorf("the vocabulary is %d shapes rather than 11; every addition widens what a compromised "+ + // + // `opening` is the twelfth, and novox/hq ADR 0100 is its decision: on an adopted node the + // firewall found there stays in force, and what the mesh needs reachable is converged through + // it as a state the host marks as the mesh's — never a command, which the link may not carry. + if len(speaks) != 12 { + t.Errorf("the vocabulary is %d shapes rather than 12; every addition widens what a compromised "+ "control plane can express, so a change here is a decision: %s", len(speaks), vocabulary()) } diff --git a/internal/declaration/into_test.go b/internal/declaration/into_test.go new file mode 100644 index 0000000..531e015 --- /dev/null +++ b/internal/declaration/into_test.go @@ -0,0 +1,35 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq ADR 0102: a file written into carries only a JSON object of its keys, in a +// format the host speaks, and a service may name what it is reloaded on. + +func TestAFileWrittenIntoIsRefusedUnlessItIsAnObjectOfKeys(t *testing.T) { + for name, c := range map[string]struct{ resource, refusal string }{ + "another format": {`{"id":"f","type":"file","path":"/etc/x","into":"toml","content":"a = 1"}`, `into "toml"`}, + "not an object": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"[1,2]"}`, "JSON object"}, + "with create-once": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","create-once":true}`, "create-once"}, + } { + _, err := Parse([]byte(`{"declaration":1,"resources":[` + c.resource + `]}`)) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err) + } + } + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/etc/x","into":"json","content":"{\"k\":1}"}, + {"id":"s","type":"service","unit":"docker.service","state":"running","reload-on":["f"]} + ]}`)) + if err != nil { + t.Fatal(err) + } + if f := d.Resources[0].(*File); f.Into != IntoJSON { + t.Errorf("into was read as %q", f.Into) + } + if s := d.Resources[1].(*Service); len(s.ReloadOn) != 1 || s.ReloadOn[0] != "f" { + t.Errorf("reload-on was read as %v", s.ReloadOn) + } +} diff --git a/internal/firewall/firewall.go b/internal/firewall/firewall.go new file mode 100644 index 0000000..37b9052 --- /dev/null +++ b/internal/firewall/firewall.go @@ -0,0 +1,929 @@ +// Package firewall speaks the firewall found on an adopted node, in that firewall's own terms +// (novox/hq ADR 0100). +// +// **The found firewall stays in force.** On an adopted node the mesh loads nothing that drops by +// default or holds an accept; what it needs reachable it converges as openings through what it +// found, marks each rule as its own, and removes only what it marked. It never resets or flushes: +// the rules the machine already had are the operator's, and they are what keeps it serving. +package firewall + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/system" +) + +// Runner executes a command. +type Runner = system.Runner + +// Kind is what firewall a machine has, as far as the mesh is concerned. +type Kind string + +const ( + // UFW is an active ufw — the one kind found on the machines measured, and the one spoken. + UFW Kind = "ufw" + // None is a machine where nothing refuses anything, which needs no openings. + None Kind = "none" + // Unsupported is a firewall no host speaks yet. A machine with one is refused adoption: the + // mesh could neither open what it needs nor know what it would be closing. + Unsupported Kind = "unsupported" +) + +// deletion is the arguments that delete a rule as `ufw show added` printed it. A route rule is +// deleted with `route delete …`: ufw refuses `delete route …` as invalid syntax. And ufw answers +// success when asked to delete a rule it does not hold, so every deletion is read back. +func deletion(rule string) []string { + w := words(rule) + if len(w) > 0 && w[0] == "route" { + return append([]string{"route", "delete"}, w[1:]...) + } + return append([]string{"delete"}, w...) +} + +// MeshInterface is the private network's interface, the way an opening from the mesh is known. +// It must be the controller's overlay interface name. +const MeshInterface = "mesh0" + +// Detect says which firewall this machine has. For Unsupported the string names it. +func Detect(ctx context.Context, run Runner) (Kind, string, error) { + if out, err := run(ctx, "firewall-cmd", "--state"); err == nil && strings.TrimSpace(out) == "running" { + return Unsupported, "firewalld", nil + } + ufwActive := false + if out, err := run(ctx, "ufw", "status"); err == nil { + ufwActive = statusActive(out) + } + + noNft := false + out, err := run(ctx, "nft", "list", "ruleset") + switch { + case err == nil: + if refusing := Refusing(out, ufwActive); len(refusing) > 0 { + return Unsupported, "nftables rules that refuse traffic, in " + strings.Join(refusing, ", "), nil + } + case missing(err): + // **No nft on this machine does not mean no rules.** iptables-nft writes tables nft would + // have shown, and a machine whose only tool is iptables answers about them through that. + // Read as "nothing filters here", a machine with an iptables firewall would be adopted + // with no openings and nothing would reach the mesh (novox/hq ADR 0100). + noNft = true + default: + return "", "", fmt.Errorf("cannot read this machine's packet filter to know what it has: %w", err) + } + + if !ufwActive { + // iptables with the legacy backend is invisible to nft; and where nft is not installed, + // the iptables command is the only way to see anything at all. + tools := []string{"iptables-legacy", "ip6tables-legacy"} + if noNft { + tools = append(tools, "iptables", "ip6tables") + } + for _, legacy := range tools { + out, err := run(ctx, legacy, "-S") + if err != nil { + continue + } + if refusing := RefusingLegacy(out); len(refusing) > 0 { + return Unsupported, legacy + " rules that refuse traffic, in " + strings.Join(refusing, ", "), nil + } + } + } + + if ufwActive { + return UFW, "ufw", nil + } + return None, "", nil +} + +func missing(err error) bool { + return errors.Is(err, exec.ErrNotFound) +} + +func statusActive(out string) bool { + for _, line := range strings.Split(out, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "Status:") { + return strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "Status:")) == "active" + } + } + return false +} + +// Refusing names the tables of an `nft list ruleset` holding something that refuses traffic — a +// drop or reject, or a base chain whose policy drops — and that is neither the mesh's own nor the +// container runtime's. With ufw active, the tables iptables-nft manages are ufw's and the runtime's +// and are not counted. +// +// **A ban is not a firewall.** fail2ban refuses the sources it banned and passes everything else; +// captured on a lab machine with both of its backends (testdata/fail2ban-nftables.nft, +// testdata/fail2ban-iptables.nft). The mesh opens nothing through it and it closes nothing the +// mesh needs, so a refusal that names the sources it refuses, in a table or a chain that accepts +// nothing and is entered only from chains whose policy accepts, is not counted. +func Refusing(ruleset string, ufwActive bool) []string { + type rule struct{ table, chain, line string } + type chainOf struct { + base, dropping, accepts bool + policyLine string + jumpedFrom []string + } + chains := map[string]*chainOf{} // by "table\x00chain" + tableAccepts := map[string]bool{} + var tables []string + var refusals []rule + managed := map[string]bool{} + var table, chain string + get := func(t, c string) *chainOf { + k := t + "\x00" + c + if chains[k] == nil { + chains[k] = &chainOf{} + } + return chains[k] + } + for _, raw := range strings.Split(ruleset, "\n") { + line := strings.TrimSpace(raw) + switch { + case strings.HasPrefix(line, "# Warning: table ") && strings.Contains(line, "managed by iptables-nft"): + name := strings.TrimPrefix(line, "# Warning: table ") + name, _, _ = strings.Cut(name, " is managed") + managed[name] = true + continue + case strings.HasPrefix(line, "table "): + table = strings.TrimSuffix(strings.TrimSpace(strings.TrimPrefix(line, "table ")), "{") + table = strings.TrimSpace(table) + tables = append(tables, table) + chain = "" + continue + case strings.HasPrefix(line, "chain "): + chain = strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(line, "chain "), "{")) + get(table, chain) + continue + case strings.HasPrefix(line, "set ") || strings.HasPrefix(line, "map ") || + strings.HasPrefix(line, "flowtable "): + chain = "" + continue + case line == "" || line == "}" || strings.HasPrefix(line, "#") || chain == "": + continue + } + c := get(table, chain) + if strings.HasPrefix(line, "type ") { + c.base = true + c.policyLine = line + c.dropping = strings.Contains(line, "policy drop") + continue + } + for _, verb := range []string{"jump ", "goto "} { + if i := strings.Index(line, verb); i >= 0 { + target := strings.Fields(line[i+len(verb):]) + if len(target) > 0 { + get(table, target[0]).jumpedFrom = append(get(table, target[0]).jumpedFrom, chain) + } + } + } + if accepts(line) { + c.accepts = true + tableAccepts[table] = true + } + if verdictRefuses(line) { + refusals = append(refusals, rule{table, chain, line}) + } + } + + skipped := func(table string) bool { + if table == "inet mesh" || table == "inet mesh_guard" { + return true + } + return (managed[table] || iptablesTable(table)) && ufwActive + } + // onlyBans is whether a refusal only refuses the sources it names: in a table that accepts + // nothing and whose base chains all accept by default, or in a chain that accepts nothing and + // is entered only from base chains that accept by default. + onlyBans := func(r rule) bool { + if !bansSources(r.line) { + return false + } + allAccepting := true + for k, c := range chains { + if strings.HasPrefix(k, r.table+"\x00") && c.base && !strings.Contains(c.policyLine, "policy accept") { + allAccepting = false + } + } + if !tableAccepts[r.table] && allAccepting { + return true + } + c := get(r.table, r.chain) + if c.base || c.accepts || len(c.jumpedFrom) == 0 { + return false + } + for _, from := range c.jumpedFrom { + caller := get(r.table, from) + if !caller.base || !strings.Contains(caller.policyLine, "policy accept") { + return false + } + } + return true + } + + counted := map[string]bool{} + for k, c := range chains { + t, name, _ := strings.Cut(k, "\x00") + if skipped(t) || !c.dropping { + continue + } + if (managed[t] || iptablesTable(t)) && runtimes(t, name, c.policyLine) { + continue + } + counted[t] = true + } + for _, r := range refusals { + if skipped(r.table) || counted[r.table] { + continue + } + if (managed[r.table] || iptablesTable(r.table)) && runtimes(r.table, r.chain, r.line) { + continue + } + if onlyBans(r) { + continue + } + counted[r.table] = true + } + var refusing []string + for _, t := range tables { + if counted[t] { + counted[t] = false + refusing = append(refusing, "table "+t) + } + } + return refusing +} + +// iptablesTable is whether a table is one iptables-nft writes. Named rather than read from the +// warning nft prints above it, because nft does not print that for every such table: a captured +// ruleset carried it on ip filter and not on ip raw, where the runtime keeps its drops. +func iptablesTable(table string) bool { + family, name, _ := strings.Cut(table, " ") + if family != "ip" && family != "ip6" { + return false + } + switch name { + case "filter", "nat", "raw", "mangle", "security": + return true + } + return false +} + +// runtimes is whether a refusal in an iptables-nft table is the container runtime's own: in its +// DOCKER chains, its forward policy, or its guard against reaching a container's address directly +// from outside its bridge, in the raw table. +func runtimes(table, chain, line string) bool { + _, name, _ := strings.Cut(table, " ") + switch { + case strings.HasPrefix(chain, "DOCKER"): + return true + case name == "filter" && chain == "FORWARD" && strings.HasPrefix(line, "type "): + return true + case name == "raw" && chain == "PREROUTING": + return strings.Contains(line, "daddr") && strings.Contains(line, "iifname !=") + } + return false +} + +// iptables-nft prints a REJECT target it cannot translate as `xt target "REJECT"`, measured in +// testdata/fail2ban-iptables.nft; a refusal written that way is a refusal too. +var verdict = regexp.MustCompile(`(^|\s)(drop|reject)(\s|$)|xt target "(DROP|REJECT)"`) + +func verdictRefuses(line string) bool { + return verdict.MatchString(line) +} + +var acceptVerdict = regexp.MustCompile(`(^|\s)accept(\s|;|$)|xt target "ACCEPT"`) + +func accepts(line string) bool { + return acceptVerdict.MatchString(line) +} + +// bansSources is whether a refusal names the sources it refuses — a set or an address — rather +// than refusing everyone but some. +func bansSources(line string) bool { + f := strings.Fields(line) + for i, w := range f { + if (w == "saddr" || w == "-s") && i+1 < len(f) && f[i+1] != "!=" && !strings.HasPrefix(f[i+1], "!") { + return i == 0 || f[i-1] != "!" + } + } + return false +} + +// RefusingLegacy names the chains of an `iptables-legacy -S` that refuse traffic outside the +// container runtime's own. A ban — a refusal of the sources it names, in a chain that accepts +// nothing and is entered only from built-in chains whose policy accepts — is not counted, as in +// Refusing (testdata/fail2ban-iptables-S.txt). +func RefusingLegacy(rules string) []string { + policy := map[string]string{} + accepting := map[string]bool{} + jumpedFrom := map[string][]string{} + for _, line := range strings.Split(rules, "\n") { + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + switch fields[0] { + case "-P": + policy[fields[1]] = fields[2] + case "-A": + for i, f := range fields { + if (f == "-j" || f == "-g") && i+1 < len(fields) { + switch fields[i+1] { + case "ACCEPT": + accepting[fields[1]] = true + case "DROP", "REJECT", "RETURN", "LOG": + default: + jumpedFrom[fields[i+1]] = append(jumpedFrom[fields[i+1]], fields[1]) + } + } + } + } + } + ban := func(chain, line string) bool { + if !bansSources(line) || accepting[chain] || len(jumpedFrom[chain]) == 0 { + return false + } + for _, from := range jumpedFrom[chain] { + if policy[from] != "ACCEPT" { + return false + } + } + return true + } + var refusing []string + seen := map[string]bool{} + for _, line := range strings.Split(rules, "\n") { + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + chain := fields[1] + refuses := false + switch fields[0] { + case "-P": + refuses = fields[2] == "DROP" && chain != "FORWARD" + case "-A": + for i, f := range fields { + if f == "-j" && i+1 < len(fields) && (fields[i+1] == "DROP" || fields[i+1] == "REJECT") { + refuses = !strings.HasPrefix(chain, "DOCKER") && !ban(chain, line) + } + } + } + if refuses && !seen[chain] { + seen[chain] = true + refusing = append(refusing, "chain "+chain) + } + } + return refusing +} + +// --- ufw --------------------------------------------------------------------------------------- + +// Mark is the comment every rule the mesh adds carries: whose it is, which opening, and a digest +// of the rule itself, so a rule the opening no longer describes is recognised as stale without the +// host having to know how ufw prints a rule back. +func Mark(o *declaration.Opening) string { + sum := sha256.Sum256([]byte(strings.Join(Rule(o), " "))) + return marker(o.ID) + " " + hex.EncodeToString(sum[:])[:8] +} + +func marker(id string) string { return "mesh-host " + id } + +// markedFor is whether a comment is the mesh's, for this opening. +func markedFor(comment, id string) bool { + return comment == marker(id) || strings.HasPrefix(comment, marker(id)+" ") +} + +// Rule is the ufw rule an opening becomes, without its comment. +// +// incoming from everywhere allow proto tcp to any port P +// incoming from the mesh allow in on mesh0 proto tcp to any port P +// forwarded route allow [in on mesh0] proto tcp to any port +// +// A forwarded opening names the container's port because ufw's route rules are matched after the +// runtime's destination translation. +func Rule(o *declaration.Opening) []string { + var rule []string + port := o.Port + if o.Path == declaration.PathForwarded { + rule = append(rule, "route") + port = o.To + } + rule = append(rule, "allow") + if o.From == declaration.FromMesh { + rule = append(rule, "in", "on", MeshInterface) + } + return append(rule, "proto", o.Protocol, "to", "any", "port", strconv.Itoa(port)) +} + +var commentOf = regexp.MustCompile(`comment '([^']*)'`) + +// added is every rule `ufw show added` lists, each without its leading "ufw". +func added(ctx context.Context, run Runner) ([]string, error) { + out, err := run(ctx, "ufw", "show", "added") + if err != nil { + return nil, fmt.Errorf("reading ufw's rules: %w", err) + } + var rules []string + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "ufw ") { + rules = append(rules, strings.TrimPrefix(line, "ufw ")) + } + } + return rules, nil +} + +func comment(rule string) string { + m := commentOf.FindStringSubmatch(rule) + if m == nil { + return "" + } + return m[1] +} + +// words splits a rule as ufw printed it into arguments, keeping a quoted comment whole. +func words(rule string) []string { + var out []string + var cur strings.Builder + quoted, any := false, false + for _, r := range rule { + switch { + case r == '\'': + quoted = !quoted + any = true + case r == ' ' && !quoted: + if any { + out = append(out, cur.String()) + cur.Reset() + any = false + } + default: + cur.WriteRune(r) + any = true + } + } + if any { + out = append(out, cur.String()) + } + return out +} + +// A ufw rule, as `ufw show added` prints it or as it is given, reduced to what ufw compares. +// +// **ufw treats two rules that differ only in their comment as one rule.** Measured on a lab +// machine (testdata/ufw-comment-only.txt): adding `route allow proto tcp to any port 8080 comment +// 'mesh-host …'` beside an operator's `route allow 8080/tcp` answers "Rule updated", and the +// operator's rule now carries the mesh's mark — so removing the opening later would delete the +// operator's rule. The same holds for an incoming rule and for one with a comment of its own. +type ufwRule struct { + route bool + action, in, out string + // dir is which way the rule matches: "" (ufw's default, incoming and forwarded), "in" or + // "out". A rule on the outgoing path admits nothing that arrives. + dir string + log string + from, fromPort, to string + port, proto, app string + comment string +} + +// parseRule reads a rule in either of ufw's forms — the short `allow 22/tcp` and the long `allow +// in on mesh0 to any port 5432 proto tcp` — into the fields ufw compares. Not ok for anything it +// does not recognise, which is then never taken to answer an opening. +func parseRule(rule string) (ufwRule, bool) { + r := ufwRule{from: "any", to: "any", comment: comment(rule)} + // A log type may stand after the action or after the direction; either way it is a property + // of the rule, not of where it matches. The word after `comment` is the comment, whatever it + // says. + var w []string + all := words(rule) + for i := 0; i < len(all); i++ { + switch { + case all[i] == "comment" && i+1 < len(all): + w = append(w, all[i], all[i+1]) + i++ + case all[i] == "log" || all[i] == "log-all": + r.log = all[i] + default: + w = append(w, all[i]) + } + } + i := 0 + if i < len(w) && w[i] == "route" { + r.route = true + i++ + } + if i >= len(w) { + return r, false + } + switch w[i] { + case "allow", "deny", "reject", "limit": + r.action = w[i] + default: + return r, false + } + i++ + for i < len(w) && (w[i] == "in" || w[i] == "out") { + dir := w[i] + i++ + iface := "" + if i+1 < len(w) && w[i] == "on" { + iface = w[i+1] + i += 2 + } + r.dir = dir + if dir == "in" { + r.in = iface + } else { + r.out = iface + } + } + if i < len(w) && w[i] != "from" && w[i] != "to" && w[i] != "proto" && w[i] != "comment" && + w[i] != "app" && w[i] != "log" && w[i] != "log-all" { + // The short form: a port with its protocol, a bare port, or an application's name. + port, proto, hasProto := strings.Cut(w[i], "/") + if isPorts(port) { + r.port = port + if hasProto { + r.proto = proto + } + } else { + r.app = w[i] + } + i++ + } + for ; i < len(w); i++ { + next := func() string { + if i+1 < len(w) { + i++ + return w[i] + } + return "" + } + switch w[i] { + case "from": + r.from = next() + if i+1 < len(w) && w[i+1] == "port" { + i++ + r.fromPort = next() + } + case "to": + r.to = next() + if i+1 < len(w) && w[i+1] == "port" { + i++ + r.port = next() + } + case "port": + r.port = next() + case "proto": + r.proto = next() + case "app": + r.app = next() + case "comment": + next() + case "log", "log-all": + default: + return r, false + } + } + if r.proto == "any" { + r.proto = "" + } + // Incoming is ufw's default direction, and it prints `allow in 9005/tcp` back as + // `allow 9005/tcp` and merges the two — captured in testdata/ufw-direction.txt. An outgoing + // rule is its own rule and stays one. + if r.dir == "in" && r.in == "" { + r.dir = "" + } + return r, true +} + +func isPorts(s string) bool { + if s == "" { + return false + } + for _, c := range s { + if (c < '0' || c > '9') && c != ':' && c != ',' { + return false + } + } + return true +} + +// sameAs is whether ufw would take two rules for one: everything equal but the comment, the +// action and the log type. Adding one beside the other updates it in place — its comment, and its +// action or log type — rather than adding a second. +func (r ufwRule) sameAs(o ufwRule) bool { + r.comment, o.comment = "", "" + r.action, o.action = "", "" + r.log, o.log = "", "" + return r == o +} + +// admits is whether a rule already lets through what an opening says: the same path, allowed from +// any source to any address of this machine, on the opening's port and protocol — or on any +// protocol — and on any interface, or the private network's for an opening from it. +func (r ufwRule) admits(o *declaration.Opening) bool { + want, ok := parseRule(strings.Join(Rule(o), " ")) + if !ok || r.route != want.route || r.action != "allow" || r.app != "" || + r.from != "any" || r.fromPort != "" || r.to != "any" { + return false + } + // A rule on the outgoing path lets this machine reach others; it admits nothing that arrives, + // so it never answers an opening. + if r.dir == "out" || r.out != "" { + return false + } + if r.proto != "" && r.proto != want.proto { + return false + } + if r.in != "" && r.in != want.in { + return false + } + return portsInclude(r.port, want.port) +} + +// portsInclude is whether a ufw port list — 80, 80,443, or 8000:8100 — names a port. +func portsInclude(list, port string) bool { + p, err := strconv.Atoi(port) + if err != nil { + return false + } + for _, part := range strings.Split(list, ",") { + lo, hi, isRange := strings.Cut(part, ":") + a, err := strconv.Atoi(lo) + if err != nil { + continue + } + b := a + if isRange { + if b, err = strconv.Atoi(hi); err != nil { + continue + } + } + if a <= p && p <= b { + return true + } + } + return false +} + +// Converged is what converging an opening did. SatisfiedBy names the rule already there that +// answers the opening, when one does; the mesh then adds nothing, and so will remove nothing. +type Converged struct { + Action string + SatisfiedBy string +} + +// Converge makes one opening true in ufw: its marked rule present, and any rule marked for it that +// no longer describes it deleted. Nothing unmarked is touched. The outcome is created, updated or +// unchanged, read back from ufw rather than assumed. +// +// **An opening a rule already answers is not added** (novox/hq ADR 0103). If ufw holds a rule not +// marked for this opening that already admits what it says — the operator's, or one the mesh +// added for another opening — the opening is satisfied by it: adding the mesh's would take that +// rule over if it differs only in its comment, and removing the opening would then delete it. +func Converge(ctx context.Context, run Runner, o *declaration.Opening) (Converged, error) { + rules, err := added(ctx, run) + if err != nil { + return Converged{}, err + } + mark := Mark(o) + present := false + var stale []string + satisfiedBy := "" + want, _ := parseRule(strings.Join(Rule(o), " ")) + for _, rule := range rules { + c := comment(rule) + switch { + case c == mark: + present = true + case markedFor(c, o.ID): + stale = append(stale, rule) + default: + parsed, ok := parseRule(rule) + if !ok { + continue + } + // ufw would take the mesh's rule for this one and rewrite its action or log type: + // an operator's refusal, or a limit, would silently become an allow — and removing the + // opening would then delete it. A plain allow answers the opening; anything else is a + // conflict the operator decides (novox/hq ADR 0103). + if parsed.sameAs(want) && (parsed.action != "allow" || parsed.log != "") { + return Converged{}, fmt.Errorf("ufw holds %q, which ufw takes for the same rule as the "+ + "mesh's opening for %s, differing in what it does; adding the opening would change "+ + "it, so nothing was added. Change or remove that rule, or have the mesh stop "+ + "declaring the opening", rule, o.Target()) + } + if satisfiedBy == "" && parsed.admits(o) { + satisfiedBy = rule + } + } + } + if present && len(stale) == 0 { + return Converged{Action: "unchanged"}, nil + } + for _, rule := range stale { + if _, err := run(ctx, "ufw", deletion(rule)...); err != nil { + return Converged{}, fmt.Errorf("deleting the mesh's stale ufw rule %q: %w", rule, err) + } + } + if !present && satisfiedBy != "" { + after, err := added(ctx, run) + if err != nil { + return Converged{}, err + } + for _, rule := range after { + if markedFor(comment(rule), o.ID) { + return Converged{}, fmt.Errorf("ufw still lists a stale rule marked for %s after deleting it", o.ID) + } + } + action := "unchanged" + if len(stale) > 0 { + action = "updated" + } + return Converged{Action: action, SatisfiedBy: satisfiedBy}, nil + } + if !present { + args := append(Rule(o), "comment", mark) + if _, err := run(ctx, "ufw", args...); err != nil { + return Converged{}, fmt.Errorf("adding the ufw rule for %s: %w", o.Target(), err) + } + } + after, err := added(ctx, run) + if err != nil { + return Converged{}, err + } + found, leftover := false, 0 + for _, rule := range after { + c := comment(rule) + if c == mark { + found = true + } else if markedFor(c, o.ID) { + leftover++ + } + } + if !found { + return Converged{}, fmt.Errorf("ufw was asked for %s and does not list it afterwards", o.Target()) + } + if leftover > 0 { + return Converged{}, fmt.Errorf("ufw still lists %d stale rule(s) marked for %s after deleting them", leftover, o.ID) + } + if len(stale) > 0 { + return Converged{Action: "updated"}, nil + } + return Converged{Action: "created"}, nil +} + +// Remove deletes the rules marked for one opening, and nothing else. +func Remove(ctx context.Context, run Runner, id string) (int, error) { + rules, err := added(ctx, run) + if err != nil { + return 0, err + } + removed := 0 + for _, rule := range rules { + if !markedFor(comment(rule), id) { + continue + } + if _, err := run(ctx, "ufw", deletion(rule)...); err != nil { + return removed, fmt.Errorf("deleting the mesh's ufw rule %q: %w", rule, err) + } + removed++ + } + after, err := added(ctx, run) + if err != nil { + return removed, err + } + for _, rule := range after { + if markedFor(comment(rule), id) { + return removed, fmt.Errorf("ufw still lists a rule marked for %s after deleting it", id) + } + } + return removed, nil +} + +// Enable turns ufw back on, as found, and reads back that it is. +func Enable(ctx context.Context, run Runner) error { + if _, err := run(ctx, "ufw", "--force", "enable"); err != nil { + return fmt.Errorf("enabling ufw again: %w", err) + } + return expectActive(ctx, run, true) +} + +// Disable retires ufw without flushing it: its configuration stays on disk, and the container +// runtime's rules are not its to remove. +// +// **Nor is the forward policy ufw's to open.** Measured on a lab machine running the container +// runtime with a published port (testdata/ufw-disable-iptables-before.txt and -after.txt): +// `ufw disable` sets every built-in chain's policy to accept, the forward chain's among them. The +// runtime had set that one to drop when it turned forwarding on, and it does not set it again while +// forwarding stays on — not even on a restart. Left so, a retired ufw turns the machine into a +// router for anyone who can reach it. So each family's forward policy is read before, and one that +// was drop is put back and read back. before is ForwardPolicies as read before the first attempt. +func Disable(ctx context.Context, run Runner, before map[string]string) error { + if _, err := run(ctx, "ufw", "disable"); err != nil { + return fmt.Errorf("disabling ufw: %w", err) + } + if err := expectActive(ctx, run, false); err != nil { + return err + } + for _, tool := range []string{"iptables", "ip6tables"} { + if before[tool] != "DROP" { + continue + } + if now, ok := forwardPolicy(ctx, run, tool); ok && now == "DROP" { + continue + } + if _, err := run(ctx, tool, "-P", "FORWARD", "DROP"); err != nil { + return fmt.Errorf("ufw is disabled, and %s's forward policy, which was drop, could not be put back: %w", + tool, err) + } + if now, ok := forwardPolicy(ctx, run, tool); !ok || now != "DROP" { + return fmt.Errorf("ufw is disabled, and %s's forward policy was put back to drop and reads %q", + tool, now) + } + } + return nil +} + +// MeshTable is the derived filter's table, the thing that must be in force before the firewall +// found on a machine is retired. +const MeshTable = "inet mesh" + +// MeshTableLoaded asks the machine whether the mesh's own filter is loaded. Read from the machine +// rather than assumed from the declaration: a table declared and not loaded is exactly the case +// where disabling the found firewall would leave the machine with nothing. +func MeshTableLoaded(ctx context.Context, run Runner) (bool, error) { + out, err := run(ctx, "nft", "list", "tables") + if err != nil { + if missing(err) { + return false, nil + } + return false, fmt.Errorf("cannot read which tables this machine has loaded: %w", err) + } + for _, line := range strings.Split(out, "\n") { + rest, ok := strings.CutPrefix(strings.TrimSpace(line), "table "+MeshTable) + if ok && (rest == "" || strings.HasPrefix(rest, " ") || strings.HasPrefix(rest, "{")) { + return true, nil + } + } + return false, nil +} + +// ForwardPolicies reads each family's forward policy, by the tool that sets it. Read before ufw is +// disabled and kept by the caller, so a retirement that fails half-way is retried with what the +// machine had — not with what the half-done disable left. +func ForwardPolicies(ctx context.Context, run Runner) map[string]string { + out := map[string]string{} + for _, tool := range []string{"iptables", "ip6tables"} { + if policy, ok := forwardPolicy(ctx, run, tool); ok { + out[tool] = policy + } + } + return out +} + +// forwardPolicy reads the forward chain's policy the way iptables prints it: "-P FORWARD DROP". +// Not ok when the tool is absent or says nothing readable. +func forwardPolicy(ctx context.Context, run Runner, tool string) (string, bool) { + out, err := run(ctx, tool, "-S", "FORWARD") + if err != nil { + return "", false + } + for _, line := range strings.Split(out, "\n") { + f := strings.Fields(line) + if len(f) == 3 && f[0] == "-P" && f[1] == "FORWARD" { + return f[2], true + } + } + return "", false +} + +func expectActive(ctx context.Context, run Runner, want bool) error { + out, err := run(ctx, "ufw", "status") + if err != nil { + return fmt.Errorf("reading ufw's status back: %w", err) + } + if statusActive(out) != want { + state := "inactive" + if want { + state = "active" + } + return fmt.Errorf("ufw was asked to be %s and says: %s", state, strings.TrimSpace(out)) + } + return nil +} diff --git a/internal/firewall/firewall_test.go b/internal/firewall/firewall_test.go new file mode 100644 index 0000000..acee5d2 --- /dev/null +++ b/internal/firewall/firewall_test.go @@ -0,0 +1,789 @@ +package firewall + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force, the mesh opens +// what it needs through it in its own terms, and removes only what it marked. + +func dockerOnly(t *testing.T) string { + t.Helper() + // Captured from a real machine running the container runtime and nothing else that filters: + // its nat, filter and raw tables as iptables-nft writes them. + raw, err := os.ReadFile("testdata/docker-only.nft") + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +const aDroppingTable = ` +table inet filter { + chain input { + type filter hook input priority filter; policy drop; + ct state established,related accept + tcp dport 22 accept + } +} +` + +const ufwChains = ` +# Warning: table ip filter is managed by iptables-nft, do not touch! +table ip filter { + chain INPUT { + type filter hook input priority filter; policy drop; + counter packets 0 bytes 0 jump ufw-before-input + } + chain ufw-user-input { + tcp dport 22 counter packets 0 bytes 0 accept + } + chain ufw-reject-input { + counter packets 0 bytes 0 reject + } +} +` + +const theMeshsOwn = ` +table inet mesh { + chain input { + type filter hook input priority filter; policy drop; + iif lo accept + } +} +table inet mesh_guard { + chain prerouting { + type filter hook prerouting priority raw; policy accept; + iifname != "lo" tcp dport { 5432, 15672 } drop + } +} +` + +func TestTheContainerRuntimesOwnRulesAreNotAFirewall(t *testing.T) { + if got := Refusing(dockerOnly(t), false); len(got) != 0 { + t.Errorf("the runtime's own rules read as a firewall: %v", got) + } +} + +func TestTheMeshsOwnTablesAreNotAFirewall(t *testing.T) { + if got := Refusing(dockerOnly(t)+theMeshsOwn, false); len(got) != 0 { + t.Errorf("the mesh's own tables read as a found firewall: %v", got) + } +} + +func TestATableThatDropsIsAFirewall(t *testing.T) { + got := Refusing(dockerOnly(t)+aDroppingTable, false) + if len(got) != 1 || got[0] != "table inet filter" { + t.Errorf("a dropping table was not named: %v", got) + } +} + +func TestUfwsOwnChainsAreUfwsWhenItIsActive(t *testing.T) { + if got := Refusing(dockerOnly(t)+ufwChains, true); len(got) != 0 { + t.Errorf("ufw's own chains read as a second firewall: %v", got) + } + if got := Refusing(dockerOnly(t)+ufwChains, false); len(got) == 0 { + t.Error("iptables rules that refuse, with ufw not active, were not counted") + } +} + +func TestLegacyIptablesThatRefusesIsAFirewall(t *testing.T) { + docker := "-P INPUT ACCEPT\n-P FORWARD DROP\n-P OUTPUT ACCEPT\n-N DOCKER\n-A DOCKER -i docker0 -j DROP\n" + if got := RefusingLegacy(docker); len(got) != 0 { + t.Errorf("the runtime's legacy rules read as a firewall: %v", got) + } + if got := RefusingLegacy(docker + "-A INPUT -p tcp --dport 25 -j REJECT\n"); len(got) != 1 { + t.Errorf("a legacy reject was not counted: %v", got) + } +} + +// fakeUFW is ufw as far as the host can see it: a status, and user rules it prints back in its +// own canonical form — deliberately not the order the host wrote them in. +type fakeUFW struct { + active bool + installed bool + rules []string + ruleset string + firewalld bool + asked []string + + // iptablesActive and iptablesInactive are what `iptables -S` prints with ufw active and + // after it is disabled; empty is a machine without iptables. forward is a policy set since. + iptablesActive, iptablesInactive string + forward string + // noNft is a machine with no nft binary; iptablesRules is what `iptables -S` prints there. + noNft bool + iptablesRules string +} + +// iptables answers `iptables -S FORWARD` from the captured output for ufw's state, and records +// a forward policy set with -P. +func (f *fakeUFW) iptables(name string, args []string) (string, error) { + if f.iptablesRules != "" && len(args) == 1 && args[0] == "-S" { + if name == "ip6tables" { + return "", nil + } + return f.iptablesRules, nil + } + if f.iptablesActive == "" { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + if name == "ip6tables" { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + if len(args) == 3 && args[0] == "-P" && args[1] == "FORWARD" { + f.forward = args[2] + return "", nil + } + captured := f.iptablesInactive + if f.active { + captured = f.iptablesActive + } + var out []string + for _, line := range strings.Split(captured, "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[1] == "FORWARD" { + if fields[0] == "-P" && f.forward != "" && !f.active { + line = "-P FORWARD " + f.forward + } + out = append(out, line) + } + } + return strings.Join(out, "\n") + "\n", nil +} + +// canonical is a rule the way ufw prints it back, as captured (testdata/ufw-comment-only.txt): the +// short form `allow 5671/tcp` for a rule on no interface, the long form `allow in on mesh0 to any +// port 5432 proto tcp` for one on an interface; the comment last. +func canonical(args []string) (rule, commentText string) { + var route, in, port, proto string + for i := 0; i < len(args); i++ { + switch args[i] { + case "route": + route = "route " + case "in": + in = args[i+2] + i += 2 + case "port": + port = args[i+1] + i++ + case "proto": + proto = args[i+1] + i++ + case "comment": + commentText = args[i+1] + i++ + } + } + if in != "" { + return route + "allow in on " + in + " to any port " + port + " proto " + proto, commentText + } + return route + "allow " + port + "/" + proto, commentText +} + +func (f *fakeUFW) run(_ context.Context, name string, args ...string) (string, error) { + f.asked = append(f.asked, name+" "+strings.Join(args, " ")) + switch name { + case "firewall-cmd": + if f.firewalld { + return "running\n", nil + } + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + case "nft": + if f.noNft { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + return f.ruleset, nil + case "iptables-legacy", "ip6tables-legacy": + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + case "iptables", "ip6tables": + return f.iptables(name, args) + case "ufw": + default: + return "", fmt.Errorf("unexpected %s", name) + } + if !f.installed { + return "", &exec.Error{Name: name, Err: exec.ErrNotFound} + } + switch { + case args[0] == "status": + if f.active { + return "Status: active\n\nTo Action From\n", nil + } + return "Status: inactive\n", nil + case args[0] == "show": + out := "Added user rules (see 'ufw status' for running firewall):\n" + for _, r := range f.rules { + out += "ufw " + r + "\n" + } + return out, nil + case args[0] == "--force" && args[1] == "enable": + f.active = true + return "Firewall is active and enabled on system startup\n", nil + case args[0] == "disable": + f.active = false + f.forward = "" + return "Firewall stopped and disabled on system startup\n", nil + case args[0] == "delete" && len(args) > 1 && args[1] == "route": + // As the real ufw answers it (captured in testdata/ufw-delete.txt): a route rule is + // deleted with `route delete`, never `delete route`. + return "", errors.New("ERROR: Invalid syntax") + case args[0] == "delete", args[0] == "route" && len(args) > 1 && args[1] == "delete": + rest := args[1:] + if args[0] == "route" { + rest = append([]string{"route"}, args[2:]...) + } + for i, r := range f.rules { + if strings.Join(words(r), "\x00") == strings.Join(rest, "\x00") { + f.rules = append(f.rules[:i], f.rules[i+1:]...) + return "Rule deleted\n", nil + } + } + return "", errors.New("Could not delete non-existent rule") + default: + rule, note := canonical(args) + line := rule + if note != "" { + line += " comment '" + note + "'" + } + // As the real ufw does (testdata/ufw-comment-only.txt): a rule differing from one it holds + // only in its comment is the same rule, and its comment is replaced. + for i, r := range f.rules { + if bare, _, _ := strings.Cut(r, " comment '"); bare == rule { + f.rules[i] = line + return "Rule updated\nRule updated (v6)\n", nil + } + } + f.rules = append(f.rules, line) + return "Rule added\nRule added (v6)\n", nil + } +} + +func (f *fakeUFW) added() int { + n := 0 + for _, a := range f.asked { + if strings.HasPrefix(a, "ufw allow") || strings.HasPrefix(a, "ufw route") { + n++ + } + } + return n +} + +func opening(id string, port int, from, path string, to int) *declaration.Opening { + return &declaration.Opening{ID: id, Type: declaration.TypeOpening, Port: port, Protocol: "tcp", + From: from, Path: path, To: to} +} + +func TestAnOpeningBecomesTheUfwRuleForItsPathAndOrigin(t *testing.T) { + for _, c := range []struct { + o *declaration.Opening + want string + }{ + {opening("adoption.a", 5671, "everywhere", "incoming", 0), "allow proto tcp to any port 5671"}, + {opening("adoption.b", 5432, "mesh", "incoming", 0), "allow in on mesh0 proto tcp to any port 5432"}, + {opening("adoption.c", 20001, "everywhere", "forwarded", 8080), "route allow proto tcp to any port 8080"}, + {opening("adoption.d", 20001, "mesh", "forwarded", 8080), "route allow in on mesh0 proto tcp to any port 8080"}, + } { + if got := strings.Join(Rule(c.o), " "); got != c.want { + t.Errorf("%s: %q, want %q", c.o.ID, got, c.want) + } + } +} + +func TestAnOpeningIsAddedOnceAndMarkedAsTheMeshs(t *testing.T) { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp"}} + o := opening("adoption.opening-tcp-5671-incoming", 5671, "everywhere", "incoming", 0) + + action, err := Converge(context.Background(), f.run, o) + if err != nil || action.Action != "created" { + t.Fatalf("first converge: %q %v", action, err) + } + if !strings.Contains(f.rules[1], "comment 'mesh-host adoption.opening-tcp-5671-incoming ") { + t.Errorf("the rule is not marked as the mesh's: %v", f.rules) + } + action, err = Converge(context.Background(), f.run, o) + if err != nil || action.Action != "unchanged" { + t.Fatalf("second converge: %q %v", action, err) + } + if f.added() != 1 { + t.Errorf("re-converging added again: %v", f.asked) + } +} + +func TestAnOpeningLostToAReloadIsAddedAgain(t *testing.T) { + f := &fakeUFW{installed: true, active: true} + o := opening("adoption.x", 5671, "everywhere", "incoming", 0) + if _, err := Converge(context.Background(), f.run, o); err != nil { + t.Fatal(err) + } + f.rules = nil // what a reload that lost the rule leaves + action, err := Converge(context.Background(), f.run, o) + if err != nil || action.Action != "created" || len(f.rules) != 1 { + t.Fatalf("a lost opening was not put back: %q %v %v", action, err, f.rules) + } +} + +func TestAChangedOpeningReplacesOnlyItsOwnRule(t *testing.T) { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 8080/tcp comment 'someone else'"}} + if _, err := Converge(context.Background(), f.run, opening("adoption.x", 5671, "everywhere", "incoming", 0)); err != nil { + t.Fatal(err) + } + action, err := Converge(context.Background(), f.run, opening("adoption.x", 5671, "mesh", "incoming", 0)) + if err != nil || action.Action != "updated" { + t.Fatalf("%q %v", action, err) + } + if len(f.rules) != 3 || f.rules[0] != "allow 22/tcp" || f.rules[1] != "allow 8080/tcp comment 'someone else'" || + !strings.Contains(f.rules[2], "in on mesh0") { + t.Errorf("rules afterwards: %v", f.rules) + } +} + +func TestRemovingAnOpeningRemovesOnlyWhatWasMarkedForIt(t *testing.T) { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 8080/tcp"}} + for _, o := range []*declaration.Opening{ + opening("adoption.a", 5671, "everywhere", "incoming", 0), + opening("adoption.ab", 5000, "everywhere", "incoming", 0), + } { + if _, err := Converge(context.Background(), f.run, o); err != nil { + t.Fatal(err) + } + } + n, err := Remove(context.Background(), f.run, "adoption.a") + if err != nil || n != 1 { + t.Fatalf("removed %d: %v", n, err) + } + if len(f.rules) != 3 || f.rules[0] != "allow 22/tcp" || f.rules[1] != "allow 8080/tcp" || + !strings.Contains(f.rules[2], "adoption.ab") { + t.Errorf("more than the marked rule went: %v", f.rules) + } +} + +func TestEnableAndDisableReadBack(t *testing.T) { + f := &fakeUFW{installed: true, active: true} + if err := Disable(context.Background(), f.run, nil); err != nil || f.active { + t.Fatalf("disable: %v", err) + } + if err := Enable(context.Background(), f.run); err != nil || !f.active { + t.Fatalf("enable: %v", err) + } + for _, a := range f.asked { + if strings.Contains(a, "reset") || strings.Contains(a, "flush") { + t.Errorf("the found firewall was reset: %s", a) + } + } +} + +func TestDetectingTheFoundFirewall(t *testing.T) { + for _, c := range []struct { + name string + f *fakeUFW + want Kind + }{ + {"nothing but the runtime", &fakeUFW{ruleset: dockerOnly(t)}, None}, + {"ufw active", &fakeUFW{installed: true, active: true, ruleset: dockerOnly(t) + ufwChains}, UFW}, + {"ufw installed and inactive", &fakeUFW{installed: true, ruleset: dockerOnly(t)}, None}, + {"firewalld", &fakeUFW{firewalld: true, ruleset: dockerOnly(t)}, Unsupported}, + {"an nftables table of its own", &fakeUFW{ruleset: dockerOnly(t) + aDroppingTable}, Unsupported}, + {"ufw beside an nftables table", &fakeUFW{installed: true, active: true, ruleset: dockerOnly(t) + ufwChains + aDroppingTable}, Unsupported}, + } { + got, name, err := Detect(context.Background(), c.f.run) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + if got != c.want { + t.Errorf("%s: detected %s (%s), want %s", c.name, got, name, c.want) + } + if got == Unsupported && name == "" { + t.Errorf("%s: an unsupported firewall was not named", c.name) + } + } +} + +// The fixtures below were captured from a real ufw 0.36.2 on a lab machine, not written by hand: +// ufw prints a rule back in its own shorter form, so the mark in the comment is the only thing the +// host relies on. + +func TestTheMarksAreReadFromWhatUfwReallyPrints(t *testing.T) { + raw, err := os.ReadFile("testdata/ufw-show-added.txt") + if err != nil { + t.Fatal(err) + } + run := func(ctx context.Context, name string, args ...string) (string, error) { return string(raw), nil } + rules, err := added(context.Background(), run) + if err != nil { + t.Fatal(err) + } + if len(rules) != 7 { + t.Fatalf("read %d rules, want 7: %q", len(rules), rules) + } + marked := 0 + for _, r := range rules { + if strings.HasPrefix(comment(r), "mesh-host ") { + marked++ + } + } + if marked != 5 { + t.Errorf("read %d marked rules, want 5", marked) + } + if !markedFor(comment(rules[5]), "adoption.opening-tcp-8443-forwarded") { + t.Errorf("the forwarded rule from the mesh lost its mark: %q", rules[5]) + } +} + +func TestEveryRealRuleIsDeletedInTheFormUfwAccepts(t *testing.T) { + raw, err := os.ReadFile("testdata/ufw-show-added.txt") + if err != nil { + t.Fatal(err) + } + run := func(ctx context.Context, name string, args ...string) (string, error) { return string(raw), nil } + rules, _ := added(context.Background(), run) + // Each of these was run on the lab machine and answered "Rule deleted" (testdata/ufw-delete.txt). + want := map[string]string{ + "allow 5671/tcp comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d'": "delete allow 5671/tcp comment|mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d", + "allow in on mesh0 to any port 5432 proto tcp comment 'mesh-host adoption.opening-tcp-5432-incoming deadbeef'": "delete allow in on mesh0 to any port 5432 proto tcp comment|mesh-host adoption.opening-tcp-5432-incoming deadbeef", + "route allow 80/tcp comment 'mesh-host adoption.opening-tcp-8081-forwarded 0badf00d'": "route delete allow 80/tcp comment|mesh-host adoption.opening-tcp-8081-forwarded 0badf00d", + "route allow in on mesh0 to any port 443 proto tcp comment 'mesh-host adoption.opening-tcp-8443-forwarded cafe0001'": "route delete allow in on mesh0 to any port 443 proto tcp comment|mesh-host adoption.opening-tcp-8443-forwarded cafe0001", + } + seen := 0 + for _, r := range rules { + w, ok := want[r] + if !ok { + continue + } + seen++ + d := deletion(r) + got := strings.Join(d[:len(d)-1], " ") + "|" + d[len(d)-1] + if got != w { + t.Errorf("deleting %q\n got %s\n want %s", r, got, w) + } + } + if seen != len(want) { + t.Errorf("matched %d of %d captured rules", seen, len(want)) + } +} + +func TestARealUfwRulesetIsUfw(t *testing.T) { + raw, err := os.ReadFile("testdata/ufw-active.nft") + if err != nil { + t.Fatal(err) + } + status, err := os.ReadFile("testdata/ufw-status-active.txt") + if err != nil { + t.Fatal(err) + } + if !statusActive(string(status)) { + t.Fatal("the captured status does not read as active") + } + if refusing := Refusing(string(raw), true); len(refusing) > 0 { + t.Errorf("a machine with ufw active and nothing else read as refusing in %v", refusing) + } + if refusing := Refusing(string(raw), false); len(refusing) == 0 { + t.Error("ufw's drop chains, with ufw not known to be active, read as refusing nothing") + } +} + +func TestRetiringUfwKeepsTheMachineFromRoutingForOthers(t *testing.T) { + // Captured on a lab machine running the container runtime with a published port: ufw active, + // then `ufw disable`. Disabling set the forward policy the runtime had set to drop to accept. + before, err := os.ReadFile("testdata/ufw-disable-iptables-before.txt") + if err != nil { + t.Fatal(err) + } + after, err := os.ReadFile("testdata/ufw-disable-iptables-after.txt") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(before), "-P FORWARD DROP") || !strings.Contains(string(after), "-P FORWARD ACCEPT") { + t.Fatal("the captures no longer show ufw disable opening the forward policy") + } + f := &fakeUFW{installed: true, active: true, iptablesActive: string(before), iptablesInactive: string(after)} + if err := Disable(context.Background(), f.run, ForwardPolicies(context.Background(), f.run)); err != nil { + t.Fatal(err) + } + if f.active { + t.Fatal("ufw is still active") + } + if f.forward != "DROP" { + t.Errorf("the forward policy was left open after ufw was retired: %v", f.asked) + } + for _, a := range f.asked { + if strings.Contains(a, "-F") || strings.Contains(a, "flush") || strings.Contains(a, "reset") { + t.Errorf("retiring ufw flushed something: %s", a) + } + } +} + +func TestRetiringUfwOnAMachineWithoutIptablesStillRetiresIt(t *testing.T) { + f := &fakeUFW{installed: true, active: true} + if err := Disable(context.Background(), f.run, nil); err != nil || f.active { + t.Fatalf("disable: %v, active %v", err, f.active) + } +} + +// Captured on a lab machine with fail2ban banning one documentation address in its sshd jail, +// once with its nftables backend and once with its iptables backend (iptables-nft), ufw inactive. + +func captured(t *testing.T, name string) string { + t.Helper() + raw, err := os.ReadFile("testdata/" + name) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func TestFail2bansBansAreNotAFirewall(t *testing.T) { + for _, name := range []string{"fail2ban-nftables.nft", "fail2ban-iptables.nft"} { + ruleset := captured(t, name) + if !strings.Contains(ruleset, "192.0.2.55") { + t.Fatalf("%s holds no ban", name) + } + if got := Refusing(ruleset, false); len(got) != 0 { + t.Errorf("%s: fail2ban's bans read as a firewall: %v", name, got) + } + kind, what, err := Detect(context.Background(), (&fakeUFW{ruleset: ruleset}).run) + if err != nil || kind != None { + t.Errorf("%s: a machine with only fail2ban detected as %s (%s) %v", name, kind, what, err) + } + } + if got := RefusingLegacy(captured(t, "fail2ban-iptables-S.txt")); len(got) != 0 { + t.Errorf("fail2ban's iptables bans read as a firewall: %v", got) + } +} + +func TestARefusalOfEveryoneButSomeIsStillAFirewall(t *testing.T) { + // A ban names the sources it refuses. A table that refuses every source but some, or every + // port but some, closes what the mesh would open, whatever its policy says. + for name, table := range map[string]string{ + "all but a range": "table inet own {\n\tchain input {\n\t\ttype filter hook input priority filter; policy accept;\n\t\tip saddr != 10.0.0.0/8 drop\n\t}\n}\n", + "all but ssh": "table inet own {\n\tchain input {\n\t\ttype filter hook input priority filter; policy accept;\n\t\ttcp dport != 22 drop\n\t}\n}\n", + "iptables reject": "# Warning: table ip filter is managed by iptables-nft, do not touch!\ntable ip filter {\n\tchain INPUT {\n\t\ttype filter hook input priority filter; policy accept;\n\t\tcounter packets 0 bytes 0 xt target \"REJECT\"\n\t}\n}\n", + "ban beside a dropping policy": "table inet own {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t\tip saddr 192.0.2.9 drop\n\t}\n}\n", + } { + if got := Refusing(dockerOnly(t)+table, false); len(got) == 0 { + t.Errorf("%s: not counted as a firewall", name) + } + } + legacy := "-P INPUT ACCEPT\n-N own\n-A INPUT -j own\n-A own ! -s 10.0.0.0/8 -j DROP\n" + if got := RefusingLegacy(legacy); len(got) == 0 { + t.Error("a legacy refusal of all but a range was not counted") + } +} + +// Defends novox/hq ADR 0103: an opening a found rule already answers is not added, because ufw +// takes two rules differing only in their comment for one (testdata/ufw-comment-only.txt). + +func TestUfwTakesTheMeshsRuleAndTheOperatorsForOne(t *testing.T) { + // The capture: each mesh rule answered "Rule updated" beside the operator's equivalent. + raw := captured(t, "ufw-comment-only.txt") + if strings.Count(raw, "Rule updated\n") != 3 { + t.Fatalf("the capture no longer shows ufw updating an equivalent rule:\n%s", raw) + } + for _, c := range []struct { + operators string + o *declaration.Opening + }{ + {"route allow 8080/tcp", opening("adoption.opening-tcp-8080-forwarded", 20001, "everywhere", "forwarded", 8080)}, + {"allow 5671/tcp", opening("adoption.opening-tcp-5671-incoming", 5671, "everywhere", "incoming", 0)}, + {"allow in on mesh0 to any port 5432 proto tcp comment 'operator note'", opening("adoption.opening-tcp-5432-incoming", 5432, "mesh", "incoming", 0)}, + } { + theirs, ok := parseRule(c.operators) + mine, ok2 := parseRule(strings.Join(Rule(c.o), " ") + " comment '" + Mark(c.o) + "'") + if !ok || !ok2 || !theirs.sameAs(mine) { + t.Errorf("%q and the mesh's %v are one rule to ufw, and read as two", c.operators, Rule(c.o)) + } + if !theirs.admits(c.o) { + t.Errorf("%q does not read as answering %s", c.operators, c.o.Target()) + } + } +} + +func TestEveryCapturedRuleFormIsRead(t *testing.T) { + want := map[string]string{ + "allow 22/tcp": "tcp 22 in= from=any", "allow 9200": " 9200 in= from=any", + "allow from 192.0.2.0/24 to any port 9300 proto tcp": "tcp 9300 in= from=192.0.2.0/24", + "allow in on eth0 to any port 9301 proto tcp": "tcp 9301 in=eth0 from=any", + "allow 9500:9510/tcp": "tcp 9500:9510 in= from=any", + "allow 80,443/tcp": "tcp 80,443 in= from=any", + "allow in on mesh0 to any port 5432 proto tcp": "tcp 5432 in=mesh0 from=any", + "route allow 8080/tcp": "tcp 8080 in= from=any", + "allow 9900/tcp": "tcp 9900 in= from=any", + "allow out 5671/tcp": "tcp 5671 in= from=any", + "deny out 5672/tcp": "tcp 5672 in= from=any", + "allow out on eth0 to any port 5673 proto tcp": "tcp 5673 in= from=any", + "allow log 9001/tcp": "tcp 9001 in= from=any", + "route allow log 8084/tcp": "tcp 8084 in= from=any", + "allow in on mesh0 log-all to any port 9002 proto tcp": "tcp 9002 in=mesh0 from=any", + } + rules, err := added(context.Background(), func(context.Context, string, ...string) (string, error) { + return captured(t, "ufw-forms.txt"), nil + }) + if err != nil || len(rules) != 21 { + t.Fatalf("read %d rules: %v", len(rules), err) + } + for _, rule := range rules { + r, ok := parseRule(rule) + if !ok { + t.Errorf("a rule ufw printed was not read: %q", rule) + continue + } + if w, listed := want[rule]; listed { + if got := r.proto + " " + r.port + " in=" + r.in + " from=" + r.from; got != w { + t.Errorf("%q read as %q, want %q", rule, got, w) + } + } + } +} + +func TestAnOpeningAFoundRuleAnswersIsNotAddedAndItsRemovalLeavesTheRule(t *testing.T) { + for _, c := range []struct { + name, operators string + o *declaration.Opening + }{ + {"forwarded, the same rule", "route allow 8080/tcp", opening("adoption.fwd", 20001, "everywhere", "forwarded", 8080)}, + {"incoming, the same rule", "allow 5671/tcp", opening("adoption.bus", 5671, "everywhere", "incoming", 0)}, + {"with a comment of its own", "allow in on mesh0 to any port 5432 proto tcp comment 'operator note'", opening("adoption.store", 5432, "mesh", "incoming", 0)}, + {"broader: from anywhere", "allow 5432/tcp", opening("adoption.store", 5432, "mesh", "incoming", 0)}, + {"broader: any protocol, a range", "allow 5000:5100", opening("adoption.registry", 5000, "everywhere", "incoming", 0)}, + } { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 22/tcp", c.operators}} + done, err := Converge(context.Background(), f.run, c.o) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + if done.SatisfiedBy != c.operators || done.Action != "unchanged" || f.added() != 0 { + t.Errorf("%s: %+v, asked %v", c.name, done, f.asked) + } + if n, err := Remove(context.Background(), f.run, c.o.ID); err != nil || n != 0 { + t.Errorf("%s: removing the opening removed %d: %v", c.name, n, err) + } + if len(f.rules) != 2 || f.rules[1] != c.operators { + t.Errorf("%s: the operator's rule did not survive: %v", c.name, f.rules) + } + } +} + +func TestARuleThatDoesNotAnswerTheOpeningLeavesItToBeAdded(t *testing.T) { + for _, operators := range []string{ + "allow from 192.0.2.0/24 to any port 5671 proto tcp", // narrower: from one range + "allow in on eth0 to any port 5671 proto tcp", // narrower: one interface + "allow 5671/udp", // another protocol + "route allow 5671/tcp", // another path + "allow to 192.0.2.1 port 5671 proto tcp", // one address + } { + f := &fakeUFW{installed: true, active: true, rules: []string{operators}} + done, err := Converge(context.Background(), f.run, opening("adoption.bus", 5671, "everywhere", "incoming", 0)) + if err != nil || done.Action != "created" || done.SatisfiedBy != "" { + t.Errorf("%q: %+v %v", operators, done, err) + } + } +} + +func TestAnOpeningWhoseFoundRuleIsGoneIsAddedAgain(t *testing.T) { + f := &fakeUFW{installed: true, active: true, rules: []string{"allow 5671/tcp"}} + o := opening("adoption.bus", 5671, "everywhere", "incoming", 0) + if done, err := Converge(context.Background(), f.run, o); err != nil || done.SatisfiedBy == "" { + t.Fatalf("%+v %v", done, err) + } + f.rules = nil // the operator deleted theirs + if done, err := Converge(context.Background(), f.run, o); err != nil || done.Action != "created" { + t.Fatalf("the opening was not added once nothing answered it: %+v %v", done, err) + } +} + +func TestARuleUfwWouldMergeThatDoesOtherThanAllowRefusesTheOpening(t *testing.T) { + // ufw takes two rules differing only in action or log type for one, and adding the mesh's + // would turn the operator's refusal into an allow (novox/hq ADR 0103). + for _, operators := range []string{ + "deny 5671/tcp", + "reject 5671/tcp", + "limit 5671/tcp", + "allow log 5671/tcp", + "allow log-all proto tcp to any port 5671", + "deny in log to any port 5671 proto tcp comment 'operator note'", + } { + f := &fakeUFW{installed: true, active: true, rules: []string{operators}} + _, err := Converge(context.Background(), f.run, opening("adoption.bus", 5671, "everywhere", "incoming", 0)) + if err == nil || !strings.Contains(err.Error(), operators) { + t.Errorf("%q: the conflict was not refused naming the rule: %v", operators, err) + } + if f.added() != 0 || len(f.rules) != 1 || f.rules[0] != operators { + t.Errorf("%q: something was added or changed: %v %v", operators, f.asked, f.rules) + } + } +} + +func TestALogTypeIsReadInEitherPlace(t *testing.T) { + for rule, want := range map[string]string{ + "allow log 22/tcp": "allow log 22 tcp in=", + "allow in log-all on mesh0 to any port 5432 proto tcp": "allow log-all 5432 tcp in=mesh0", + "route deny log in on mesh0 to any port 80 proto tcp": "deny log 80 tcp in=mesh0", + "allow 22/tcp comment 'log'": "allow 22 tcp in=", + } { + r, ok := parseRule(rule) + if got := r.action + " " + r.log + " " + r.port + " " + r.proto + " in=" + r.in; !ok || got != want { + t.Errorf("%q read as %q (%v), want %q", rule, got, ok, want) + } + } +} + +func TestAnOutgoingRuleNeverAnswersAnOpening(t *testing.T) { + // `ufw allow out 5671/tcp` lets this machine reach others; nothing arrives through it, and + // ufw keeps it as a rule of its own — captured in testdata/ufw-direction.txt. + raw := captured(t, "ufw-direction.txt") + if !strings.Contains(raw, "ufw allow out 9007/tcp\nufw allow 9007/tcp") { + t.Fatalf("the capture no longer shows an outgoing rule standing beside an incoming one:\n%s", raw) + } + for _, operators := range []string{"allow out 5671/tcp", "allow out on eth0 to any port 5671 proto tcp", + "deny out 5671/tcp"} { + f := &fakeUFW{installed: true, active: true, rules: []string{operators}} + done, err := Converge(context.Background(), f.run, opening("adoption.bus", 5671, "everywhere", "incoming", 0)) + if err != nil { + t.Errorf("%q: an outgoing rule was taken for a conflict: %v", operators, err) + continue + } + if done.Action != "created" || done.SatisfiedBy != "" { + t.Errorf("%q: an outgoing rule answered an incoming opening: %+v", operators, done) + } + } +} + +func TestIncomingIsUfwsDefaultDirection(t *testing.T) { + // Captured: `deny in 9006/tcp` and `allow 9006/tcp` are one rule to ufw, so the mesh must read + // them as one too, or it would take an operator's refusal over. + raw := captured(t, "ufw-direction.txt") + if !strings.Contains(raw, "ufw allow 9005/tcp") || strings.Contains(raw, "ufw deny 9006/tcp") { + t.Fatalf("the capture no longer shows `in` as the default direction:\n%s", raw) + } + f := &fakeUFW{installed: true, active: true, rules: []string{"deny in to any port 5671 proto tcp"}} + if _, err := Converge(context.Background(), f.run, opening("adoption.bus", 5671, "everywhere", "incoming", 0)); err == nil { + t.Error("an incoming refusal ufw would merge was not refused") + } +} + +func TestAMachineWithIptablesRulesAndNoNftIsNotReadAsUnfiltered(t *testing.T) { + // nft is not installed, and iptables-nft holds a firewall of somebody's. Read as "nothing + // filters here" the mesh would adopt it, open nothing, and be unreachable (novox/hq ADR 0100). + rules := "-P INPUT DROP\n-P FORWARD DROP\n-P OUTPUT ACCEPT\n-A INPUT -p tcp -m tcp --dport 22 -j ACCEPT\n" + f := &fakeUFW{noNft: true, iptablesRules: rules} + kind, what, err := Detect(context.Background(), f.run) + if err != nil { + t.Fatal(err) + } + if kind != Unsupported { + t.Errorf("a machine filtered by iptables with no nft read as %s (%s)", kind, what) + } + // And a machine with nothing but the runtime's own rules and no nft is still unfiltered. + docker := "-P INPUT ACCEPT\n-P FORWARD DROP\n-P OUTPUT ACCEPT\n-N DOCKER\n-A DOCKER -i docker0 -j DROP\n" + if kind, _, err := Detect(context.Background(), (&fakeUFW{noNft: true, iptablesRules: docker}).run); err != nil || kind != None { + t.Errorf("a machine with only the runtime's rules read as %s: %v", kind, err) + } +} diff --git a/internal/firewall/testdata/docker-only.nft b/internal/firewall/testdata/docker-only.nft new file mode 100644 index 0000000..7df77ba --- /dev/null +++ b/internal/firewall/testdata/docker-only.nft @@ -0,0 +1,297 @@ +# Warning: table ip nat is managed by iptables-nft, do not touch! +table ip nat { + chain DOCKER { + iifname != "br-c70303d221ee" tcp dport 5680 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-c70303d221ee" tcp dport 15673 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 59000 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 59001 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 55672 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 55673 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-3636e05760a9" tcp dport 55432 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "docker0" tcp dport 57732 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "docker0" tcp dport 57733 counter packets 0 bytes 0 xt target "DNAT" + iifname != "docker0" tcp dport 5314 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-613eb68ef5fb" tcp dport 4848 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-b3240c822bce" tcp dport 5679 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-b3240c822bce" tcp dport 15672 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-4b504efd6080" tcp dport 9000 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-4b504efd6080" tcp dport 9001 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-ef6df03f71a0" tcp dport 5432 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-ef6df03f71a0" tcp dport 8081 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-ec480f77ac34" tcp dport 6379 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-af4c9c2aa60a" tcp dport 8001 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-669fda75f1ac" tcp dport 28080 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "br-af4c9c2aa60a" tcp dport 6789 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-af4c9c2aa60a" tcp dport 8770 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-af4c9c2aa60a" tcp dport 1212 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-af4c9c2aa60a" tcp dport 80 counter packets 0 bytes 0 xt target "DNAT" + iifname != "br-af4c9c2aa60a" tcp dport 443 counter packets 0 bytes 0 xt target "DNAT" + ip daddr 127.0.0.1 iifname != "docker0" tcp dport 55541 counter packets 0 bytes 0 xt target "DNAT" + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 437947 bytes 71410534 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 5761 bytes 423317 jump DOCKER + } + + chain POSTROUTING { + type nat hook postrouting priority srcnat; policy accept; + ip saddr 172.22.0.0/16 oifname != "br-65f6dc782562" counter packets 124 bytes 16328 xt target "MASQUERADE" + ip saddr 172.28.0.0/16 oifname != "br-669fda75f1ac" counter packets 127 bytes 16928 xt target "MASQUERADE" + ip saddr 172.31.0.0/16 oifname != "br-ec480f77ac34" counter packets 127 bytes 16928 xt target "MASQUERADE" + ip saddr 192.168.48.0/20 oifname != "br-ef6df03f71a0" counter packets 127 bytes 16928 xt target "MASQUERADE" + ip saddr 172.25.0.0/16 oifname != "br-4b504efd6080" counter packets 129 bytes 17052 xt target "MASQUERADE" + ip saddr 192.168.32.0/20 oifname != "br-613eb68ef5fb" counter packets 1124 bytes 76748 xt target "MASQUERADE" + ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 1833 bytes 150990 xt target "MASQUERADE" + ip saddr 172.18.0.0/16 oifname != "br-07a5e2f2c42f" counter packets 504 bytes 65112 xt target "MASQUERADE" + ip saddr 172.27.0.0/16 oifname != "br-160f55da427c" counter packets 508 bytes 65784 xt target "MASQUERADE" + ip saddr 192.168.16.0/20 oifname != "br-dba077b9b543" counter packets 503 bytes 64784 xt target "MASQUERADE" + ip saddr 192.168.64.0/20 oifname != "br-d44fef8fd602" counter packets 1282 bytes 111308 xt target "MASQUERADE" + ip saddr 172.30.0.0/16 oifname != "br-af4c9c2aa60a" counter packets 2503 bytes 190324 xt target "MASQUERADE" + ip saddr 172.21.0.0/16 oifname != "br-9d6c95e8d80c" counter packets 1289 bytes 112644 xt target "MASQUERADE" + ip saddr 172.23.0.0/16 oifname != "br-679db9b21e00" counter packets 506 bytes 65384 xt target "MASQUERADE" + ip saddr 172.24.0.0/16 oifname != "br-40094534a5ee" counter packets 508 bytes 65784 xt target "MASQUERADE" + ip saddr 172.26.0.0/16 oifname != "br-3dcb6ef83ea1" counter packets 508 bytes 65784 xt target "MASQUERADE" + ip saddr 172.20.0.0/16 oifname != "br-3a760a74f4f6" counter packets 1153 bytes 104344 xt target "MASQUERADE" + ip saddr 192.168.80.0/20 oifname != "br-3636e05760a9" counter packets 546 bytes 70540 xt target "MASQUERADE" + ip saddr 172.29.0.0/16 oifname != "br-b3240c822bce" counter packets 551 bytes 71492 xt target "MASQUERADE" + ip saddr 172.19.0.0/16 oifname != "br-c70303d221ee" counter packets 1136 bytes 106592 xt target "MASQUERADE" + } +} +# Warning: table ip filter is managed by iptables-nft, do not touch! +table ip filter { + chain DOCKER { + ip daddr 172.17.0.5 iifname != "docker0" oifname "docker0" tcp dport 5432 counter packets 0 bytes 0 accept + ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 443 counter packets 0 bytes 0 accept + ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 80 counter packets 0 bytes 0 accept + ip daddr 172.30.0.10 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 3000 counter packets 0 bytes 0 accept + ip daddr 172.30.0.5 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 8000 counter packets 0 bytes 0 accept + ip daddr 172.30.0.3 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 6789 counter packets 0 bytes 0 accept + ip daddr 172.28.0.3 iifname != "br-669fda75f1ac" oifname "br-669fda75f1ac" tcp dport 8080 counter packets 0 bytes 0 accept + ip daddr 172.30.0.4 iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" tcp dport 5540 counter packets 0 bytes 0 accept + ip daddr 172.31.0.2 iifname != "br-ec480f77ac34" oifname "br-ec480f77ac34" tcp dport 6379 counter packets 0 bytes 0 accept + ip daddr 192.168.48.3 iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" tcp dport 8081 counter packets 0 bytes 0 accept + ip daddr 192.168.48.2 iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" tcp dport 5432 counter packets 0 bytes 0 accept + ip daddr 172.25.0.2 iifname != "br-4b504efd6080" oifname "br-4b504efd6080" tcp dport 9001 counter packets 0 bytes 0 accept + ip daddr 172.25.0.2 iifname != "br-4b504efd6080" oifname "br-4b504efd6080" tcp dport 9000 counter packets 0 bytes 0 accept + ip daddr 172.29.0.2 iifname != "br-b3240c822bce" oifname "br-b3240c822bce" tcp dport 15672 counter packets 0 bytes 0 accept + ip daddr 172.29.0.2 iifname != "br-b3240c822bce" oifname "br-b3240c822bce" tcp dport 5672 counter packets 0 bytes 0 accept + ip daddr 192.168.32.2 iifname != "br-613eb68ef5fb" oifname "br-613eb68ef5fb" tcp dport 1433 counter packets 0 bytes 0 accept + ip daddr 172.17.0.3 iifname != "docker0" oifname "docker0" tcp dport 5000 counter packets 0 bytes 0 accept + ip daddr 172.17.0.2 iifname != "docker0" oifname "docker0" tcp dport 15672 counter packets 0 bytes 0 accept + ip daddr 172.17.0.2 iifname != "docker0" oifname "docker0" tcp dport 5672 counter packets 0 bytes 0 accept + ip daddr 192.168.80.4 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 5432 counter packets 0 bytes 0 accept + ip daddr 192.168.80.3 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 15672 counter packets 0 bytes 0 accept + ip daddr 192.168.80.3 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 5672 counter packets 0 bytes 0 accept + ip daddr 192.168.80.2 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 9001 counter packets 0 bytes 0 accept + ip daddr 192.168.80.2 iifname != "br-3636e05760a9" oifname "br-3636e05760a9" tcp dport 9000 counter packets 0 bytes 0 accept + ip daddr 172.19.0.2 iifname != "br-c70303d221ee" oifname "br-c70303d221ee" tcp dport 15672 counter packets 0 bytes 0 accept + ip daddr 172.19.0.2 iifname != "br-c70303d221ee" oifname "br-c70303d221ee" tcp dport 5672 counter packets 0 bytes 0 accept + iifname != "br-c70303d221ee" oifname "br-c70303d221ee" counter packets 0 bytes 0 drop + iifname != "br-b3240c822bce" oifname "br-b3240c822bce" counter packets 0 bytes 0 drop + iifname != "br-3636e05760a9" oifname "br-3636e05760a9" counter packets 0 bytes 0 drop + iifname != "br-3a760a74f4f6" oifname "br-3a760a74f4f6" counter packets 0 bytes 0 drop + iifname != "br-3dcb6ef83ea1" oifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 drop + iifname != "br-40094534a5ee" oifname "br-40094534a5ee" counter packets 0 bytes 0 drop + iifname != "br-679db9b21e00" oifname "br-679db9b21e00" counter packets 0 bytes 0 drop + iifname != "br-9d6c95e8d80c" oifname "br-9d6c95e8d80c" counter packets 0 bytes 0 drop + iifname != "br-af4c9c2aa60a" oifname "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + iifname != "br-d44fef8fd602" oifname "br-d44fef8fd602" counter packets 0 bytes 0 drop + iifname != "br-dba077b9b543" oifname "br-dba077b9b543" counter packets 0 bytes 0 drop + iifname != "br-160f55da427c" oifname "br-160f55da427c" counter packets 0 bytes 0 drop + iifname != "br-07a5e2f2c42f" oifname "br-07a5e2f2c42f" counter packets 0 bytes 0 drop + iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop + iifname != "br-613eb68ef5fb" oifname "br-613eb68ef5fb" counter packets 0 bytes 0 drop + iifname != "br-4b504efd6080" oifname "br-4b504efd6080" counter packets 0 bytes 0 drop + iifname != "br-ef6df03f71a0" oifname "br-ef6df03f71a0" counter packets 0 bytes 0 drop + iifname != "br-ec480f77ac34" oifname "br-ec480f77ac34" counter packets 0 bytes 0 drop + iifname != "br-669fda75f1ac" oifname "br-669fda75f1ac" counter packets 0 bytes 0 drop + iifname != "br-65f6dc782562" oifname "br-65f6dc782562" counter packets 0 bytes 0 drop + } + + chain DOCKER-FORWARD { + counter packets 6530319 bytes 11196484299 jump DOCKER-CT + counter packets 3312487 bytes 5001091084 jump DOCKER-INTERNAL + counter packets 3312487 bytes 5001091084 jump DOCKER-BRIDGE + iifname "br-c70303d221ee" counter packets 0 bytes 0 accept + iifname "br-b3240c822bce" counter packets 0 bytes 0 accept + iifname "br-3636e05760a9" counter packets 43 bytes 9355 accept + iifname "br-3a760a74f4f6" counter packets 0 bytes 0 accept + iifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 accept + iifname "br-40094534a5ee" counter packets 0 bytes 0 accept + iifname "br-679db9b21e00" counter packets 0 bytes 0 accept + iifname "br-9d6c95e8d80c" counter packets 0 bytes 0 accept + iifname "br-af4c9c2aa60a" counter packets 2761311 bytes 4959915711 accept + iifname "br-d44fef8fd602" counter packets 0 bytes 0 accept + iifname "br-dba077b9b543" counter packets 0 bytes 0 accept + iifname "br-160f55da427c" counter packets 0 bytes 0 accept + iifname "br-07a5e2f2c42f" counter packets 0 bytes 0 accept + iifname "docker0" counter packets 460394 bytes 25754554 accept + iifname "br-613eb68ef5fb" counter packets 10805 bytes 1792862 accept + iifname "br-4b504efd6080" counter packets 33 bytes 2892 accept + iifname "br-ef6df03f71a0" counter packets 0 bytes 0 accept + iifname "br-ec480f77ac34" counter packets 0 bytes 0 accept + iifname "br-669fda75f1ac" counter packets 0 bytes 0 accept + iifname "br-65f6dc782562" counter packets 0 bytes 0 accept + } + + chain DOCKER-BRIDGE { + oifname "br-c70303d221ee" counter packets 0 bytes 0 jump DOCKER + oifname "br-b3240c822bce" counter packets 0 bytes 0 jump DOCKER + oifname "br-3636e05760a9" counter packets 0 bytes 0 jump DOCKER + oifname "br-3a760a74f4f6" counter packets 0 bytes 0 jump DOCKER + oifname "br-3dcb6ef83ea1" counter packets 0 bytes 0 jump DOCKER + oifname "br-40094534a5ee" counter packets 0 bytes 0 jump DOCKER + oifname "br-679db9b21e00" counter packets 0 bytes 0 jump DOCKER + oifname "br-9d6c95e8d80c" counter packets 0 bytes 0 jump DOCKER + oifname "br-af4c9c2aa60a" counter packets 118 bytes 8400 jump DOCKER + oifname "br-d44fef8fd602" counter packets 0 bytes 0 jump DOCKER + oifname "br-dba077b9b543" counter packets 0 bytes 0 jump DOCKER + oifname "br-160f55da427c" counter packets 0 bytes 0 jump DOCKER + oifname "br-07a5e2f2c42f" counter packets 0 bytes 0 jump DOCKER + oifname "docker0" counter packets 0 bytes 0 jump DOCKER + oifname "br-613eb68ef5fb" counter packets 0 bytes 0 jump DOCKER + oifname "br-4b504efd6080" counter packets 0 bytes 0 jump DOCKER + oifname "br-ef6df03f71a0" counter packets 0 bytes 0 jump DOCKER + oifname "br-ec480f77ac34" counter packets 0 bytes 0 jump DOCKER + oifname "br-669fda75f1ac" counter packets 0 bytes 0 jump DOCKER + oifname "br-65f6dc782562" counter packets 0 bytes 0 jump DOCKER + } + + chain DOCKER-CT { + oifname "br-c70303d221ee" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-b3240c822bce" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-3636e05760a9" xt match "conntrack" counter packets 35 bytes 23113 accept + oifname "br-3a760a74f4f6" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-3dcb6ef83ea1" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-40094534a5ee" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-679db9b21e00" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-9d6c95e8d80c" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-af4c9c2aa60a" xt match "conntrack" counter packets 2488066 bytes 1053784742 accept + oifname "br-d44fef8fd602" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-dba077b9b543" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-160f55da427c" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-07a5e2f2c42f" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "docker0" xt match "conntrack" counter packets 666252 bytes 5079577802 accept + oifname "br-613eb68ef5fb" xt match "conntrack" counter packets 7926 bytes 7636543 accept + oifname "br-4b504efd6080" xt match "conntrack" counter packets 29 bytes 10870 accept + oifname "br-ef6df03f71a0" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-ec480f77ac34" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-669fda75f1ac" xt match "conntrack" counter packets 0 bytes 0 accept + oifname "br-65f6dc782562" xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy drop; + counter packets 33747166 bytes 176750349038 jump DOCKER-USER + counter packets 6530319 bytes 11196484299 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + oifname "mlab*" counter packets 15657582 bytes 162801189460 accept + iifname "mlab*" counter packets 10958315 bytes 687322055 accept + oifname "incusbr0" counter packets 388768 bytes 2053271282 accept + iifname "incusbr0" counter packets 212182 bytes 12081942 accept + } +} +# Warning: table ip6 nat is managed by iptables-nft, do not touch! +table ip6 nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 532 bytes 113834 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } +} +table ip6 filter { + chain DOCKER { + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + } + + chain DOCKER-BRIDGE { + } + + chain DOCKER-CT { + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy accept; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + } +} +table ip raw { + chain PREROUTING { + type filter hook prerouting priority raw; policy accept; + ip daddr 172.19.0.2 iifname != "br-c70303d221ee" counter packets 0 bytes 0 drop + ip daddr 192.168.80.2 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop + ip daddr 192.168.80.3 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 15673 counter packets 0 bytes 0 drop + ip daddr 192.168.80.4 iifname != "br-3636e05760a9" counter packets 0 bytes 0 drop + ip daddr 172.17.0.2 iifname != "docker0" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 57732 counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 57733 counter packets 0 bytes 0 drop + ip daddr 172.17.0.3 iifname != "docker0" counter packets 0 bytes 0 drop + ip daddr 172.17.0.4 iifname != "docker0" counter packets 0 bytes 0 drop + ip daddr 192.168.32.2 iifname != "br-613eb68ef5fb" counter packets 0 bytes 0 drop + ip daddr 172.30.0.7 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.29.0.2 iifname != "br-b3240c822bce" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 15672 counter packets 0 bytes 0 drop + ip daddr 172.25.0.2 iifname != "br-4b504efd6080" counter packets 0 bytes 0 drop + ip daddr 172.30.0.9 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 192.168.48.2 iifname != "br-ef6df03f71a0" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 5432 counter packets 0 bytes 0 drop + ip daddr 192.168.48.3 iifname != "br-ef6df03f71a0" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 8081 counter packets 0 bytes 0 drop + ip daddr 172.30.0.8 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.31.0.2 iifname != "br-ec480f77ac34" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 6379 counter packets 0 bytes 0 drop + ip daddr 172.30.0.4 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 8001 counter packets 0 bytes 0 drop + ip daddr 172.31.0.3 iifname != "br-ec480f77ac34" counter packets 0 bytes 0 drop + ip daddr 172.28.0.2 iifname != "br-669fda75f1ac" counter packets 0 bytes 0 drop + ip daddr 172.30.0.6 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.28.0.3 iifname != "br-669fda75f1ac" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 28080 counter packets 0 bytes 0 drop + ip daddr 172.30.0.3 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 6789 counter packets 0 bytes 0 drop + ip daddr 172.30.0.5 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.22.0.2 iifname != "br-65f6dc782562" counter packets 0 bytes 0 drop + ip daddr 172.30.0.10 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.22.0.3 iifname != "br-65f6dc782562" counter packets 0 bytes 0 drop + ip daddr 172.30.0.2 iifname != "br-af4c9c2aa60a" counter packets 0 bytes 0 drop + ip daddr 172.17.0.5 iifname != "docker0" counter packets 0 bytes 0 drop + ip daddr 127.0.0.1 iifname != "lo" tcp dport 55541 counter packets 0 bytes 0 drop + } +} +table ip mangle { + chain FORWARD { + type filter hook forward priority mangle; policy accept; + tcp flags & (syn | rst) == syn counter packets 13760 bytes 825476 xt target "TCPMSS" + } +} diff --git a/internal/firewall/testdata/fail2ban-iptables-S.txt b/internal/firewall/testdata/fail2ban-iptables-S.txt new file mode 100644 index 0000000..7b43c32 --- /dev/null +++ b/internal/firewall/testdata/fail2ban-iptables-S.txt @@ -0,0 +1,22 @@ +-P INPUT ACCEPT +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-BRIDGE +-N DOCKER-CT +-N DOCKER-FORWARD +-N DOCKER-INTERNAL +-N DOCKER-USER +-N f2b-sshd +-A INPUT -p tcp -m multiport --dports 22 -j f2b-sshd +-A FORWARD -j DOCKER-USER +-A FORWARD -j DOCKER-FORWARD +-A DOCKER ! -i docker0 -o docker0 -j DROP +-A DOCKER-BRIDGE -o docker0 -j DOCKER +-A DOCKER-CT -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A DOCKER-FORWARD -j DOCKER-CT +-A DOCKER-FORWARD -j DOCKER-INTERNAL +-A DOCKER-FORWARD -j DOCKER-BRIDGE +-A DOCKER-FORWARD -i docker0 -j ACCEPT +-A f2b-sshd -s 192.0.2.55/32 -j REJECT --reject-with icmp-port-unreachable +-A f2b-sshd -j RETURN diff --git a/internal/firewall/testdata/fail2ban-iptables.nft b/internal/firewall/testdata/fail2ban-iptables.nft new file mode 100644 index 0000000..bbe9f5a --- /dev/null +++ b/internal/firewall/testdata/fail2ban-iptables.nft @@ -0,0 +1,103 @@ +table ip nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain POSTROUTING { + type nat hook postrouting priority srcnat; policy accept; + ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 0 bytes 0 xt target "MASQUERADE" + } +} +table ip filter { + chain DOCKER { + iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + iifname "docker0" counter packets 0 bytes 0 accept + } + + chain DOCKER-BRIDGE { + oifname "docker0" counter packets 0 bytes 0 jump DOCKER + } + + chain DOCKER-CT { + oifname "docker0" xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy drop; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + } + + chain f2b-sshd { + ip saddr 192.0.2.55 counter packets 0 bytes 0 xt target "REJECT" + counter packets 0 bytes 0 return + } + + chain INPUT { + type filter hook input priority filter; policy accept; + ip protocol tcp xt match "multiport" counter packets 0 bytes 0 jump f2b-sshd + } +} +table ip6 nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } +} +table ip6 filter { + chain DOCKER { + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + } + + chain DOCKER-BRIDGE { + } + + chain DOCKER-CT { + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy accept; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + } +} diff --git a/internal/firewall/testdata/fail2ban-nftables.nft b/internal/firewall/testdata/fail2ban-nftables.nft new file mode 100644 index 0000000..cc38447 --- /dev/null +++ b/internal/firewall/testdata/fail2ban-nftables.nft @@ -0,0 +1,105 @@ +table ip nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain POSTROUTING { + type nat hook postrouting priority srcnat; policy accept; + ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 0 bytes 0 xt target "MASQUERADE" + } +} +table ip filter { + chain DOCKER { + iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + iifname "docker0" counter packets 0 bytes 0 accept + } + + chain DOCKER-BRIDGE { + oifname "docker0" counter packets 0 bytes 0 jump DOCKER + } + + chain DOCKER-CT { + oifname "docker0" xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy drop; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + } +} +table ip6 nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } +} +table ip6 filter { + chain DOCKER { + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + } + + chain DOCKER-BRIDGE { + } + + chain DOCKER-CT { + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy accept; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + } + + chain DOCKER-USER { + } +} +table inet f2b-table { + set addr-set-sshd { + type ipv4_addr + flags interval + elements = { 192.0.2.55 } + } + + chain f2b-chain { + type filter hook input priority filter - 1; policy accept; + tcp dport 22 ip saddr @addr-set-sshd reject with icmp port-unreachable + } +} diff --git a/internal/firewall/testdata/ufw-active.nft b/internal/firewall/testdata/ufw-active.nft new file mode 100644 index 0000000..93b24a4 --- /dev/null +++ b/internal/firewall/testdata/ufw-active.nft @@ -0,0 +1,478 @@ +table ip nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 2 bytes 1160 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip daddr != 127.0.0.0/8 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain POSTROUTING { + type nat hook postrouting priority srcnat; policy accept; + ip saddr 172.17.0.0/16 oifname != "docker0" counter packets 0 bytes 0 xt target "MASQUERADE" + } +} +table ip filter { + chain DOCKER { + iifname != "docker0" oifname "docker0" counter packets 0 bytes 0 drop + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + iifname "docker0" counter packets 0 bytes 0 accept + } + + chain DOCKER-BRIDGE { + oifname "docker0" counter packets 0 bytes 0 jump DOCKER + } + + chain DOCKER-CT { + oifname "docker0" xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy drop; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + counter packets 0 bytes 0 jump ufw-before-logging-forward + counter packets 0 bytes 0 jump ufw-before-forward + counter packets 0 bytes 0 jump ufw-after-forward + counter packets 0 bytes 0 jump ufw-after-logging-forward + counter packets 0 bytes 0 jump ufw-reject-forward + counter packets 0 bytes 0 jump ufw-track-forward + } + + chain DOCKER-USER { + } + + chain ufw-before-logging-input { + } + + chain ufw-before-logging-output { + } + + chain ufw-before-logging-forward { + } + + chain ufw-before-input { + iifname "lo" counter packets 0 bytes 0 accept + xt match "conntrack" counter packets 0 bytes 0 accept + xt match "conntrack" counter packets 0 bytes 0 jump ufw-logging-deny + xt match "conntrack" counter packets 0 bytes 0 drop + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + udp sport 67 udp dport 68 counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw-not-local + ip daddr 224.0.0.251 udp dport 5353 counter packets 0 bytes 0 accept + ip daddr 239.255.255.250 udp dport 1900 counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw-user-input + } + + chain ufw-before-output { + oifname "lo" counter packets 0 bytes 0 accept + xt match "conntrack" counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw-user-output + } + + chain ufw-before-forward { + xt match "conntrack" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + ip protocol icmp xt match "icmp" counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw-user-forward + } + + chain ufw-after-input { + udp dport 137 counter packets 0 bytes 0 jump ufw-skip-to-policy-input + udp dport 138 counter packets 0 bytes 0 jump ufw-skip-to-policy-input + tcp dport 139 counter packets 0 bytes 0 jump ufw-skip-to-policy-input + tcp dport 445 counter packets 0 bytes 0 jump ufw-skip-to-policy-input + udp dport 67 counter packets 0 bytes 0 jump ufw-skip-to-policy-input + udp dport 68 counter packets 0 bytes 0 jump ufw-skip-to-policy-input + xt match "addrtype" counter packets 0 bytes 0 jump ufw-skip-to-policy-input + } + + chain ufw-after-output { + } + + chain ufw-after-forward { + } + + chain ufw-after-logging-input { + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw-after-logging-output { + } + + chain ufw-after-logging-forward { + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw-reject-input { + } + + chain ufw-reject-output { + } + + chain ufw-reject-forward { + } + + chain ufw-track-input { + } + + chain ufw-track-output { + ip protocol tcp xt match "conntrack" counter packets 0 bytes 0 accept + ip protocol udp xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain ufw-track-forward { + } + + chain INPUT { + type filter hook input priority filter; policy drop; + counter packets 1 bytes 76 jump ufw-before-logging-input + counter packets 1 bytes 76 jump ufw-before-input + counter packets 0 bytes 0 jump ufw-after-input + counter packets 0 bytes 0 jump ufw-after-logging-input + counter packets 0 bytes 0 jump ufw-reject-input + counter packets 0 bytes 0 jump ufw-track-input + } + + chain OUTPUT { + type filter hook output priority filter; policy accept; + counter packets 1 bytes 76 jump ufw-before-logging-output + counter packets 1 bytes 76 jump ufw-before-output + counter packets 1 bytes 76 jump ufw-after-output + counter packets 1 bytes 76 jump ufw-after-logging-output + counter packets 1 bytes 76 jump ufw-reject-output + counter packets 1 bytes 76 jump ufw-track-output + } + + chain ufw-logging-deny { + xt match "conntrack" limit rate 3/minute burst 10 packets counter packets 0 bytes 0 return + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw-logging-allow { + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw-skip-to-policy-input { + counter packets 0 bytes 0 drop + } + + chain ufw-skip-to-policy-output { + counter packets 0 bytes 0 accept + } + + chain ufw-skip-to-policy-forward { + counter packets 0 bytes 0 drop + } + + chain ufw-not-local { + xt match "addrtype" counter packets 0 bytes 0 return + xt match "addrtype" counter packets 0 bytes 0 return + xt match "addrtype" counter packets 0 bytes 0 return + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 jump ufw-logging-deny + counter packets 0 bytes 0 drop + } + + chain ufw-user-input { + tcp dport 22 counter packets 0 bytes 0 accept + tcp dport 8080 counter packets 0 bytes 0 accept + udp dport 51820 counter packets 0 bytes 0 accept + } + + chain ufw-user-output { + } + + chain ufw-user-forward { + tcp dport 80 counter packets 0 bytes 0 accept + iifname "mesh0" tcp dport 443 counter packets 0 bytes 0 accept + } + + chain ufw-user-logging-input { + } + + chain ufw-user-logging-output { + } + + chain ufw-user-logging-forward { + } + + chain ufw-user-limit { + limit rate 3/minute burst 5 packets counter packets 0 bytes 0 xt target "LOG" + counter packets 0 bytes 0 xt target "REJECT" + } + + chain ufw-user-limit-accept { + counter packets 0 bytes 0 accept + } +} +table ip6 nat { + chain DOCKER { + } + + chain PREROUTING { + type nat hook prerouting priority dstnat; policy accept; + xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } + + chain OUTPUT { + type nat hook output priority dstnat; policy accept; + ip6 daddr != ::1 xt match "addrtype" counter packets 0 bytes 0 jump DOCKER + } +} +table ip6 filter { + chain DOCKER { + } + + chain DOCKER-FORWARD { + counter packets 0 bytes 0 jump DOCKER-CT + counter packets 0 bytes 0 jump DOCKER-INTERNAL + counter packets 0 bytes 0 jump DOCKER-BRIDGE + } + + chain DOCKER-BRIDGE { + } + + chain DOCKER-CT { + } + + chain DOCKER-INTERNAL { + } + + chain FORWARD { + type filter hook forward priority filter; policy drop; + counter packets 0 bytes 0 jump DOCKER-USER + counter packets 0 bytes 0 jump DOCKER-FORWARD + counter packets 0 bytes 0 jump ufw6-before-logging-forward + counter packets 0 bytes 0 jump ufw6-before-forward + counter packets 0 bytes 0 jump ufw6-after-forward + counter packets 0 bytes 0 jump ufw6-after-logging-forward + counter packets 0 bytes 0 jump ufw6-reject-forward + counter packets 0 bytes 0 jump ufw6-track-forward + } + + chain DOCKER-USER { + } + + chain ufw6-before-logging-input { + } + + chain ufw6-before-logging-output { + } + + chain ufw6-before-logging-forward { + } + + chain ufw6-before-input { + iifname "lo" counter packets 0 bytes 0 accept + xt match "rt" counter packets 0 bytes 0 drop + xt match "conntrack" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + xt match "conntrack" counter packets 0 bytes 0 jump ufw6-logging-deny + xt match "conntrack" counter packets 0 bytes 0 drop + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 ip6 daddr fe80::/10 udp sport 547 udp dport 546 counter packets 0 bytes 0 accept + ip6 daddr ff02::fb udp dport 5353 counter packets 0 bytes 0 accept + ip6 daddr ff02::f udp dport 1900 counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw6-user-input + } + + chain ufw6-before-output { + oifname "lo" counter packets 0 bytes 0 accept + xt match "rt" counter packets 0 bytes 0 drop + xt match "conntrack" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + ip6 saddr fe80::/10 meta l4proto ipv6-icmp xt match "icmp6" xt match "hl" counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw6-user-output + } + + chain ufw6-before-forward { + xt match "rt" counter packets 0 bytes 0 drop + xt match "conntrack" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 accept + counter packets 0 bytes 0 jump ufw6-user-forward + } + + chain ufw6-after-input { + udp dport 137 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input + udp dport 138 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input + tcp dport 139 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input + tcp dport 445 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input + udp dport 546 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input + udp dport 547 counter packets 0 bytes 0 jump ufw6-skip-to-policy-input + } + + chain ufw6-after-output { + } + + chain ufw6-after-forward { + } + + chain ufw6-after-logging-input { + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw6-after-logging-output { + } + + chain ufw6-after-logging-forward { + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw6-reject-input { + } + + chain ufw6-reject-output { + } + + chain ufw6-reject-forward { + } + + chain ufw6-track-input { + } + + chain ufw6-track-output { + meta l4proto tcp xt match "conntrack" counter packets 0 bytes 0 accept + meta l4proto udp xt match "conntrack" counter packets 0 bytes 0 accept + } + + chain ufw6-track-forward { + } + + chain INPUT { + type filter hook input priority filter; policy drop; + counter packets 1 bytes 128 jump ufw6-before-logging-input + counter packets 1 bytes 128 jump ufw6-before-input + counter packets 0 bytes 0 jump ufw6-after-input + counter packets 0 bytes 0 jump ufw6-after-logging-input + counter packets 0 bytes 0 jump ufw6-reject-input + counter packets 0 bytes 0 jump ufw6-track-input + } + + chain OUTPUT { + type filter hook output priority filter; policy accept; + counter packets 4 bytes 304 jump ufw6-before-logging-output + counter packets 4 bytes 304 jump ufw6-before-output + counter packets 0 bytes 0 jump ufw6-after-output + counter packets 0 bytes 0 jump ufw6-after-logging-output + counter packets 0 bytes 0 jump ufw6-reject-output + counter packets 0 bytes 0 jump ufw6-track-output + } + + chain ufw6-logging-deny { + xt match "conntrack" limit rate 3/minute burst 10 packets counter packets 0 bytes 0 return + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw6-logging-allow { + limit rate 3/minute burst 10 packets counter packets 0 bytes 0 xt target "LOG" + } + + chain ufw6-skip-to-policy-input { + counter packets 0 bytes 0 drop + } + + chain ufw6-skip-to-policy-output { + counter packets 0 bytes 0 accept + } + + chain ufw6-skip-to-policy-forward { + counter packets 0 bytes 0 drop + } + + chain ufw6-user-input { + tcp dport 22 counter packets 0 bytes 0 accept + tcp dport 8080 counter packets 0 bytes 0 accept + udp dport 51820 counter packets 0 bytes 0 accept + } + + chain ufw6-user-output { + } + + chain ufw6-user-forward { + tcp dport 80 counter packets 0 bytes 0 accept + iifname "mesh0" tcp dport 443 counter packets 0 bytes 0 accept + } + + chain ufw6-user-logging-input { + } + + chain ufw6-user-logging-output { + } + + chain ufw6-user-logging-forward { + } + + chain ufw6-user-limit { + limit rate 3/minute burst 5 packets counter packets 0 bytes 0 xt target "LOG" + counter packets 0 bytes 0 xt target "REJECT" + } + + chain ufw6-user-limit-accept { + counter packets 0 bytes 0 accept + } +} diff --git a/internal/firewall/testdata/ufw-comment-only.txt b/internal/firewall/testdata/ufw-comment-only.txt new file mode 100644 index 0000000..13f758c --- /dev/null +++ b/internal/firewall/testdata/ufw-comment-only.txt @@ -0,0 +1,37 @@ +$ ufw allow 22/tcp +Rules updated +Rules updated (v6) +$ ufw --force enable +Firewall is active and enabled on system startup +$ ufw route allow 8080/tcp +Rule added +Rule added (v6) +$ ufw route allow proto tcp to any port 8080 comment 'mesh-host adoption.opening-tcp-8080-forwarded 1a2b3c4d' +Rule updated +Rule updated (v6) +$ ufw allow 5671/tcp +Rule added +Rule added (v6) +$ ufw allow proto tcp to any port 5671 comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d' +Rule updated +Rule updated (v6) +$ ufw allow in on mesh0 to any port 5432 proto tcp comment 'operator note' +Rule added +Rule added (v6) +$ ufw allow in on mesh0 proto tcp to any port 5432 comment 'mesh-host adoption.opening-tcp-5432-incoming 1a2b3c4d' +Rule updated +Rule updated (v6) +$ ufw show added +Added user rules (see 'ufw status' for running firewall): +ufw allow 22/tcp +ufw route allow 8080/tcp comment 'mesh-host adoption.opening-tcp-8080-forwarded 1a2b3c4d' +ufw allow 5671/tcp comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d' +ufw allow in on mesh0 to any port 5432 proto tcp comment 'mesh-host adoption.opening-tcp-5432-incoming 1a2b3c4d' +$ ufw route delete allow 8080/tcp comment 'mesh-host adoption.opening-tcp-8080-forwarded 1a2b3c4d' +Rule deleted +Rule deleted (v6) +$ ufw show added +Added user rules (see 'ufw status' for running firewall): +ufw allow 22/tcp +ufw allow 5671/tcp comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d' +ufw allow in on mesh0 to any port 5432 proto tcp comment 'mesh-host adoption.opening-tcp-5432-incoming 1a2b3c4d' diff --git a/internal/firewall/testdata/ufw-delete.txt b/internal/firewall/testdata/ufw-delete.txt new file mode 100644 index 0000000..b887dad --- /dev/null +++ b/internal/firewall/testdata/ufw-delete.txt @@ -0,0 +1,40 @@ +Rule deleted +Rule deleted (v6) +rc=0 +Rule deleted +Rule deleted (v6) +rc=0 +ERROR: Invalid syntax +rc=1 +===ADDED2 +Added user rules (see 'ufw status' for running firewall): +ufw allow 22/tcp +ufw allow 8080/tcp +ufw route allow 80/tcp comment 'mesh-host adoption.opening-tcp-8081-forwarded 0badf00d' +ufw route allow in on mesh0 to any port 443 proto tcp comment 'mesh-host adoption.opening-tcp-8443-forwarded cafe0001' +ufw allow 51820/udp comment 'mesh-host adoption.opening-udp-51820-incoming 11112222' +Firewall reloaded +reload rc=0 +===AFTERRELOAD +3 +Firewall stopped and disabled on system startup +===DISABLED +Status: inactive +/etc/ufw/user.rules +3 +Firewall is active and enabled on system startup +Status: active +Rule deleted +Rule deleted (v6) +rc=0 +Rule deleted +Rule deleted (v6) +rc=0 +Could not delete non-existent rule +Could not delete non-existent rule (v6) +wrongcomment rc=0 +Added user rules (see 'ufw status' for running firewall): +ufw allow 22/tcp +ufw allow 8080/tcp +ufw allow 51820/udp comment 'mesh-host adoption.opening-udp-51820-incoming 11112222' +Status: active diff --git a/internal/firewall/testdata/ufw-direction.txt b/internal/firewall/testdata/ufw-direction.txt new file mode 100644 index 0000000..0fa5371 --- /dev/null +++ b/internal/firewall/testdata/ufw-direction.txt @@ -0,0 +1,21 @@ +$ ufw allow in 9005/tcp +Rules updated +Rules updated (v6) +$ ufw deny in 9006/tcp +Rules updated +Rules updated (v6) +$ ufw allow 9006/tcp +Rules updated +Rules updated (v6) +$ ufw allow out 9007/tcp +Rules updated +Rules updated (v6) +$ ufw allow 9007/tcp +Rules updated +Rules updated (v6) +$ ufw show added +Added user rules (see 'ufw status' for running firewall): +ufw allow 9005/tcp +ufw allow 9006/tcp +ufw allow out 9007/tcp +ufw allow 9007/tcp diff --git a/internal/firewall/testdata/ufw-disable-iptables-after.txt b/internal/firewall/testdata/ufw-disable-iptables-after.txt new file mode 100644 index 0000000..22dbc77 --- /dev/null +++ b/internal/firewall/testdata/ufw-disable-iptables-after.txt @@ -0,0 +1,55 @@ +-P INPUT ACCEPT +-P FORWARD ACCEPT +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-BRIDGE +-N DOCKER-CT +-N DOCKER-FORWARD +-N DOCKER-INTERNAL +-N DOCKER-USER +-N ufw-after-forward +-N ufw-after-input +-N ufw-after-logging-forward +-N ufw-after-logging-input +-N ufw-after-logging-output +-N ufw-after-output +-N ufw-before-forward +-N ufw-before-input +-N ufw-before-logging-forward +-N ufw-before-logging-input +-N ufw-before-logging-output +-N ufw-before-output +-N ufw-reject-forward +-N ufw-reject-input +-N ufw-reject-output +-N ufw-track-forward +-N ufw-track-input +-N ufw-track-output +-A INPUT -j ufw-before-logging-input +-A INPUT -j ufw-before-input +-A INPUT -j ufw-after-input +-A INPUT -j ufw-after-logging-input +-A INPUT -j ufw-reject-input +-A INPUT -j ufw-track-input +-A FORWARD -j DOCKER-USER +-A FORWARD -j DOCKER-FORWARD +-A FORWARD -j ufw-before-logging-forward +-A FORWARD -j ufw-before-forward +-A FORWARD -j ufw-after-forward +-A FORWARD -j ufw-after-logging-forward +-A FORWARD -j ufw-reject-forward +-A FORWARD -j ufw-track-forward +-A OUTPUT -j ufw-before-logging-output +-A OUTPUT -j ufw-before-output +-A OUTPUT -j ufw-after-output +-A OUTPUT -j ufw-after-logging-output +-A OUTPUT -j ufw-reject-output +-A OUTPUT -j ufw-track-output +-A DOCKER -d 172.17.0.2/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT +-A DOCKER ! -i docker0 -o docker0 -j DROP +-A DOCKER-BRIDGE -o docker0 -j DOCKER +-A DOCKER-CT -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A DOCKER-FORWARD -j DOCKER-CT +-A DOCKER-FORWARD -j DOCKER-INTERNAL +-A DOCKER-FORWARD -j DOCKER-BRIDGE +-A DOCKER-FORWARD -i docker0 -j ACCEPT diff --git a/internal/firewall/testdata/ufw-disable-iptables-before.txt b/internal/firewall/testdata/ufw-disable-iptables-before.txt new file mode 100644 index 0000000..4e18c44 --- /dev/null +++ b/internal/firewall/testdata/ufw-disable-iptables-before.txt @@ -0,0 +1,119 @@ +-P INPUT DROP +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-BRIDGE +-N DOCKER-CT +-N DOCKER-FORWARD +-N DOCKER-INTERNAL +-N DOCKER-USER +-N ufw-after-forward +-N ufw-after-input +-N ufw-after-logging-forward +-N ufw-after-logging-input +-N ufw-after-logging-output +-N ufw-after-output +-N ufw-before-forward +-N ufw-before-input +-N ufw-before-logging-forward +-N ufw-before-logging-input +-N ufw-before-logging-output +-N ufw-before-output +-N ufw-logging-allow +-N ufw-logging-deny +-N ufw-not-local +-N ufw-reject-forward +-N ufw-reject-input +-N ufw-reject-output +-N ufw-skip-to-policy-forward +-N ufw-skip-to-policy-input +-N ufw-skip-to-policy-output +-N ufw-track-forward +-N ufw-track-input +-N ufw-track-output +-N ufw-user-forward +-N ufw-user-input +-N ufw-user-limit +-N ufw-user-limit-accept +-N ufw-user-logging-forward +-N ufw-user-logging-input +-N ufw-user-logging-output +-N ufw-user-output +-A INPUT -j ufw-before-logging-input +-A INPUT -j ufw-before-input +-A INPUT -j ufw-after-input +-A INPUT -j ufw-after-logging-input +-A INPUT -j ufw-reject-input +-A INPUT -j ufw-track-input +-A FORWARD -j DOCKER-USER +-A FORWARD -j DOCKER-FORWARD +-A FORWARD -j ufw-before-logging-forward +-A FORWARD -j ufw-before-forward +-A FORWARD -j ufw-after-forward +-A FORWARD -j ufw-after-logging-forward +-A FORWARD -j ufw-reject-forward +-A FORWARD -j ufw-track-forward +-A OUTPUT -j ufw-before-logging-output +-A OUTPUT -j ufw-before-output +-A OUTPUT -j ufw-after-output +-A OUTPUT -j ufw-after-logging-output +-A OUTPUT -j ufw-reject-output +-A OUTPUT -j ufw-track-output +-A DOCKER -d 172.17.0.2/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT +-A DOCKER ! -i docker0 -o docker0 -j DROP +-A DOCKER-BRIDGE -o docker0 -j DOCKER +-A DOCKER-CT -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A DOCKER-FORWARD -j DOCKER-CT +-A DOCKER-FORWARD -j DOCKER-INTERNAL +-A DOCKER-FORWARD -j DOCKER-BRIDGE +-A DOCKER-FORWARD -i docker0 -j ACCEPT +-A ufw-after-input -p udp -m udp --dport 137 -j ufw-skip-to-policy-input +-A ufw-after-input -p udp -m udp --dport 138 -j ufw-skip-to-policy-input +-A ufw-after-input -p tcp -m tcp --dport 139 -j ufw-skip-to-policy-input +-A ufw-after-input -p tcp -m tcp --dport 445 -j ufw-skip-to-policy-input +-A ufw-after-input -p udp -m udp --dport 67 -j ufw-skip-to-policy-input +-A ufw-after-input -p udp -m udp --dport 68 -j ufw-skip-to-policy-input +-A ufw-after-input -m addrtype --dst-type BROADCAST -j ufw-skip-to-policy-input +-A ufw-after-logging-forward -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW BLOCK] " +-A ufw-after-logging-input -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW BLOCK] " +-A ufw-before-forward -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A ufw-before-forward -p icmp -m icmp --icmp-type 3 -j ACCEPT +-A ufw-before-forward -p icmp -m icmp --icmp-type 11 -j ACCEPT +-A ufw-before-forward -p icmp -m icmp --icmp-type 12 -j ACCEPT +-A ufw-before-forward -p icmp -m icmp --icmp-type 8 -j ACCEPT +-A ufw-before-forward -j ufw-user-forward +-A ufw-before-input -i lo -j ACCEPT +-A ufw-before-input -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A ufw-before-input -m conntrack --ctstate INVALID -j ufw-logging-deny +-A ufw-before-input -m conntrack --ctstate INVALID -j DROP +-A ufw-before-input -p icmp -m icmp --icmp-type 3 -j ACCEPT +-A ufw-before-input -p icmp -m icmp --icmp-type 11 -j ACCEPT +-A ufw-before-input -p icmp -m icmp --icmp-type 12 -j ACCEPT +-A ufw-before-input -p icmp -m icmp --icmp-type 8 -j ACCEPT +-A ufw-before-input -p udp -m udp --sport 67 --dport 68 -j ACCEPT +-A ufw-before-input -j ufw-not-local +-A ufw-before-input -d 224.0.0.251/32 -p udp -m udp --dport 5353 -j ACCEPT +-A ufw-before-input -d 239.255.255.250/32 -p udp -m udp --dport 1900 -j ACCEPT +-A ufw-before-input -j ufw-user-input +-A ufw-before-output -o lo -j ACCEPT +-A ufw-before-output -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A ufw-before-output -j ufw-user-output +-A ufw-logging-allow -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW ALLOW] " +-A ufw-logging-deny -m conntrack --ctstate INVALID -m limit --limit 3/min --limit-burst 10 -j RETURN +-A ufw-logging-deny -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW BLOCK] " +-A ufw-not-local -m addrtype --dst-type LOCAL -j RETURN +-A ufw-not-local -m addrtype --dst-type MULTICAST -j RETURN +-A ufw-not-local -m addrtype --dst-type BROADCAST -j RETURN +-A ufw-not-local -m limit --limit 3/min --limit-burst 10 -j ufw-logging-deny +-A ufw-not-local -j DROP +-A ufw-skip-to-policy-forward -j DROP +-A ufw-skip-to-policy-input -j DROP +-A ufw-skip-to-policy-output -j ACCEPT +-A ufw-track-output -p tcp -m conntrack --ctstate NEW -j ACCEPT +-A ufw-track-output -p udp -m conntrack --ctstate NEW -j ACCEPT +-A ufw-user-input -p tcp -m tcp --dport 22 -j ACCEPT +-A ufw-user-input -p tcp -m tcp --dport 5671 -j ACCEPT +-A ufw-user-input -i mesh0 -p tcp -m tcp --dport 5432 -j ACCEPT +-A ufw-user-limit -m limit --limit 3/min -j LOG --log-prefix "[UFW LIMIT BLOCK] " +-A ufw-user-limit -j REJECT --reject-with icmp-port-unreachable +-A ufw-user-limit-accept -j ACCEPT diff --git a/internal/firewall/testdata/ufw-forms.txt b/internal/firewall/testdata/ufw-forms.txt new file mode 100644 index 0000000..7c89bca --- /dev/null +++ b/internal/firewall/testdata/ufw-forms.txt @@ -0,0 +1,22 @@ +Added user rules (see 'ufw status' for running firewall): +ufw allow 22/tcp +ufw allow 9200 +ufw allow from 192.0.2.0/24 to any port 9300 proto tcp +ufw allow in on eth0 to any port 9301 proto tcp +ufw deny 9400/tcp +ufw limit 2222/tcp +ufw allow 9500:9510/tcp +ufw route allow in on mesh0 out on docker0 to any port 8082 proto tcp +ufw allow to 192.0.2.1 port 9600 proto tcp +ufw allow 9700/udp +ufw route allow in on mesh0 to any port 8083 proto tcp +ufw allow 9900/tcp +ufw allow 80,443/tcp +ufw allow in on mesh0 to any port 5432 proto tcp +ufw route allow 8080/tcp +ufw allow out 5671/tcp +ufw deny out 5672/tcp +ufw allow out on eth0 to any port 5673 proto tcp +ufw allow log 9001/tcp +ufw route allow log 8084/tcp +ufw allow in on mesh0 log-all to any port 9002 proto tcp diff --git a/internal/firewall/testdata/ufw-show-added.txt b/internal/firewall/testdata/ufw-show-added.txt new file mode 100644 index 0000000..85d9c6b --- /dev/null +++ b/internal/firewall/testdata/ufw-show-added.txt @@ -0,0 +1,8 @@ +Added user rules (see 'ufw status' for running firewall): +ufw allow 22/tcp +ufw allow 8080/tcp +ufw allow 5671/tcp comment 'mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d' +ufw allow in on mesh0 to any port 5432 proto tcp comment 'mesh-host adoption.opening-tcp-5432-incoming deadbeef' +ufw route allow 80/tcp comment 'mesh-host adoption.opening-tcp-8081-forwarded 0badf00d' +ufw route allow in on mesh0 to any port 443 proto tcp comment 'mesh-host adoption.opening-tcp-8443-forwarded cafe0001' +ufw allow 51820/udp comment 'mesh-host adoption.opening-udp-51820-incoming 11112222' diff --git a/internal/firewall/testdata/ufw-status-active.txt b/internal/firewall/testdata/ufw-status-active.txt new file mode 100644 index 0000000..9f32038 --- /dev/null +++ b/internal/firewall/testdata/ufw-status-active.txt @@ -0,0 +1,21 @@ +Status: active + +To Action From +-- ------ ---- +22/tcp ALLOW Anywhere +8080/tcp ALLOW Anywhere +5671/tcp ALLOW Anywhere # mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d +5432/tcp on mesh0 ALLOW Anywhere # mesh-host adoption.opening-tcp-5432-incoming deadbeef +51820/udp ALLOW Anywhere # mesh-host adoption.opening-udp-51820-incoming 11112222 +22/tcp (v6) ALLOW Anywhere (v6) +8080/tcp (v6) ALLOW Anywhere (v6) +5671/tcp (v6) ALLOW Anywhere (v6) # mesh-host adoption.opening-tcp-5671-incoming 1a2b3c4d +5432/tcp (v6) on mesh0 ALLOW Anywhere (v6) # mesh-host adoption.opening-tcp-5432-incoming deadbeef +51820/udp (v6) ALLOW Anywhere (v6) # mesh-host adoption.opening-udp-51820-incoming 11112222 + +80/tcp ALLOW FWD Anywhere # mesh-host adoption.opening-tcp-8081-forwarded 0badf00d +443/tcp ALLOW FWD Anywhere on mesh0 # mesh-host adoption.opening-tcp-8443-forwarded cafe0001 +80/tcp (v6) ALLOW FWD Anywhere (v6) # mesh-host adoption.opening-tcp-8081-forwarded 0badf00d +443/tcp (v6) ALLOW FWD Anywhere (v6) on mesh0 # mesh-host adoption.opening-tcp-8443-forwarded cafe0001 + +===STATUSV diff --git a/internal/identity/identity_test.go b/internal/identity/identity_test.go index 2d0c4f9..fcc0b2a 100644 --- a/internal/identity/identity_test.go +++ b/internal/identity/identity_test.go @@ -220,6 +220,19 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { if len(fields) != 5 { t.Errorf("the token has %d fields, expected 5: %v", len(fields), fields) } + + // novox/hq ADR 0100: an adopted node's token says so, and a converged one's is unchanged. + raw, err = json.Marshal(Token{Version: 1, Secret: "s", Adopted: true}) + if err != nil { + t.Fatal(err) + } + fields = map[string]any{} + if err := json.Unmarshal(raw, &fields); err != nil { + t.Fatal(err) + } + if fields["adopted"] != true { + t.Errorf("an adopted token does not say \"adopted\": %v", fields) + } } func TestACompleteTokenParses(t *testing.T) { diff --git a/internal/identity/token.go b/internal/identity/token.go index 40fab81..4fbaf46 100644 --- a/internal/identity/token.go +++ b/internal/identity/token.go @@ -30,6 +30,11 @@ type Token struct { Fingerprint string `json:"fingerprint,omitempty"` Signer []byte `json:"signer,omitempty"` Secret string `json:"secret"` + + // Adopted says this node joins adopted (novox/hq ADR 0100). The host checks it speaks the + // firewall found here before enrolling, because an adopted node keeps that firewall in force. + // Absent for a converged node. + Adopted bool `json:"adopted,omitempty"` } // ParseToken reads a token a person pasted. diff --git a/internal/link/messages.go b/internal/link/messages.go index 6a48e41..953fe6c 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -1,5 +1,7 @@ package link +import "time" + // The wire formats shared with the control plane, which defines them separately because this // binary requires nothing present and does not import it. A test on each side asserts the field // names, so a rename breaks both at once rather than on a real machine months later. @@ -85,4 +87,44 @@ type Report struct { // than the send, and the machine reads as caught up with words it has not read yet. Clocks // cannot answer "which"; the digest is the answer itself. Declared string `json:"declared,omitempty"` + + // Held is what this adopted node found and is keeping as it was until its module is taken + // (novox/hq ADR 0100). Without it an adopted node reads as converged. + Held []Held `json:"held,omitempty"` + + // Firewall is the firewall found on this machine — "ufw" or "none" — and empty on a node that + // was never asked, which is every converged one. + Firewall string `json:"firewall,omitempty"` + + // Reachable is what can be reached on this machine now: every listening socket and every + // published container port. Only an adopted node reports it; it is what converging the node + // previews, so nothing closes without being named first. + Reachable []Reach `json:"reachable,omitempty"` +} + +// Held is one file or container found on an adopted node and kept as it was. +type Held struct { + ID string `json:"id"` + Module string `json:"module"` + Kind string `json:"kind"` + Target string `json:"target"` + Since time.Time `json:"since"` + // Changed is what something other than the mesh did to it since — rewritten, stopped, + // replaced or gone — and empty while it is as found. + Changed string `json:"changed,omitempty"` + // Kept is where a file's original was kept. + Kept string `json:"kept,omitempty"` +} + +// Reach is one thing reachable on the machine: a listening socket, or a published container port. +type Reach struct { + Protocol string `json:"protocol"` + Address string `json:"address"` + Port int `json:"port"` + // By is what holds it — a process, or a container's name. + By string `json:"by,omitempty"` + // Published is a container port the runtime publishes, reached on the forwarded path; its + // container's own port is ContainerPort. + Published bool `json:"published,omitempty"` + ContainerPort int `json:"container-port,omitempty"` } diff --git a/internal/link/messages_test.go b/internal/link/messages_test.go index 7983221..76a9a7e 100644 --- a/internal/link/messages_test.go +++ b/internal/link/messages_test.go @@ -120,6 +120,14 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { {Signed{Declaration: []byte("{}"), Signature: []byte("x")}, []string{"declaration", "signature"}}, {Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"}, []string{"node", "applied", "failed", "refused"}}, + // novox/hq ADR 0100: what an adopted node holds, the firewall it was found with, and what + // is reachable on it. + {Report{Node: "n", Held: []Held{{ID: "i"}}, Firewall: "ufw", Reachable: []Reach{{Port: 1}}}, + []string{"node", "held", "firewall", "reachable"}}, + {Held{ID: "i", Module: "m", Kind: "file", Target: "/t", Changed: "rewritten", Kept: "/k"}, + []string{"id", "module", "kind", "target", "since", "changed", "kept"}}, + {Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "c", Published: true, ContainerPort: 80}, + []string{"protocol", "address", "port", "by", "published", "container-port"}}, } { raw, err := json.Marshal(c.value) if err != nil { diff --git a/internal/link/run.go b/internal/link/run.go index 247d015..1379369 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -70,15 +70,29 @@ type Announce func(string) type Roused <-chan struct{} func Hold(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration) error { - return HoldRoused(ctx, m, apply, say, timeout, nil) + return HoldRoused(ctx, m, apply, say, timeout, nil, nil) } -// HoldRoused is Hold, told when the machine has reason to think its link is stale. +// Outbox carries reports the node has to say without having been sent anything — what a +// reconcile found changed on an adopted node (novox/hq ADR 0100). Published while the link is up; +// a report made while it is down waits in the channel for the next one. Nil is allowed. +type Outbox <-chan Unasked + +// Unasked is one such report, with the way to say whether it reached the mesh. Done is called +// with true only when the broker took it — a node that marked a change said because it queued it +// would never say it again, and the mesh would go on believing nothing changed. +type Unasked struct { + Report Report + Done func(published bool) +} + +// HoldRoused is Hold, told when the machine has reason to think its link is stale, and handed +// reports to publish between deliveries. func HoldRoused(ctx context.Context, m Membership, apply Applier, say Announce, - timeout time.Duration, roused Roused) error { + timeout time.Duration, roused Roused, outbox Outbox) error { return holdWith(ctx, func(ctx context.Context) error { - return Run(ctx, m, apply, say, timeout) + return Run(ctx, m, apply, say, timeout, outbox) }, say, roused) } @@ -171,7 +185,8 @@ func holdWith(ctx context.Context, run attempt, say Announce, roused Roused) err // // Outbound only, and nothing listens on this machine. Returns when the link ends, for any reason; // Hold is what decides whether to open it again. -func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration) error { +func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration, + outbox Outbox) error { if say == nil { say = func(string) {} } @@ -254,6 +269,13 @@ func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout return nil case <-beat.C: publishAlive(ctx, channel, m, say, timeout) + case unasked := <-outbox: + // Said without having been asked: a reconcile found what an adopted node holds, or + // its firewall, changed since it last said. + published := publishReport(ctx, channel, m, unasked.Report, say, timeout) + if unasked.Done != nil { + unasked.Done(published) + } case reason := <-closed: return fmt.Errorf("the link closed: %v", reason) case delivery, ok := <-deliveries: @@ -354,13 +376,14 @@ func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) R return apply(ctx, signed.Declaration, signed.Signature) } +// publishReport tells the mesh what this node did, and says whether the broker took it. func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report, - say Announce, timeout time.Duration) { + say Announce, timeout time.Duration) bool { report.Node = m.Node body, err := json.Marshal(report) if err != nil { say("cannot encode this node's own report: " + err.Error()) - return + return false } publish, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -371,7 +394,9 @@ func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, rep if err := channel.PublishWithContext(publish, Exchange, KeyReport, true, false, amqp.Publishing{ContentType: "application/json", Body: body}); err != nil { say(fmt.Sprintf("applied, and could not tell the mesh: %v", err)) + return false } + return true } // publishAlive says this node is here, and nothing else. diff --git a/internal/reachable/collect.go b/internal/reachable/collect.go new file mode 100644 index 0000000..537e2e8 --- /dev/null +++ b/internal/reachable/collect.go @@ -0,0 +1,181 @@ +// Package reachable reads what can be reached on this machine now: every listening socket, and +// every container port the runtime publishes (novox/hq ADR 0100). +// +// It is what converging an adopted node previews — each port, whether a module declares it or it +// will close — and what a converged genesis counts before refusing a machine in use. It reads; it +// never decides what is the mesh's. +package reachable + +import ( + "context" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/system" +) + +// Runner executes a command. +type Runner = system.Runner + +// Reach is one thing reachable on this machine, in the words the report carries. +type Reach = link.Reach + +// Collect reads the machine's listening sockets and the runtime's published ports. A published +// port is reported once, as published, rather than again as the runtime's proxy listening for it. +func Collect(ctx context.Context, run Runner) ([]Reach, error) { + out, err := run(ctx, "ss", "-Hltunp") + if err != nil { + return nil, fmt.Errorf("reading this machine's listening sockets: %w", err) + } + sockets := Sockets(out) + + var published []Reach + if ps, err := run(ctx, "docker", "ps", "--format", "{{.Names}}\t{{.Ports}}"); err == nil { + published = Published(ps) + } + return Merge(sockets, published), nil +} + +var process = regexp.MustCompile(`users:\(\("([^"]+)"`) + +// Sockets parses `ss -Hltunp`: each line a netid, a state, two queues, the local address and +// port, the peer, and the process when ss may name it. +func Sockets(out string) []Reach { + var reached []Reach + for _, line := range strings.Split(out, "\n") { + fields := strings.Fields(line) + if len(fields) < 5 { + continue + } + protocol := fields[0] + if protocol != "tcp" && protocol != "udp" { + continue + } + address, port, ok := splitLocal(fields[4]) + if !ok { + continue + } + r := Reach{Protocol: protocol, Address: address, Port: port} + if m := process.FindStringSubmatch(line); m != nil { + r.By = m[1] + } + reached = append(reached, r) + } + return reached +} + +// splitLocal reads "127.0.0.1:53", "[::]:22", "*:22" and "[fe80::1]%veth0:123". +func splitLocal(local string) (string, int, bool) { + i := strings.LastIndex(local, ":") + if i < 0 { + return "", 0, false + } + port, err := strconv.Atoi(local[i+1:]) + if err != nil { + return "", 0, false + } + address := local[:i] + if at := strings.Index(address, "%"); at >= 0 { + address = address[:at] + } + address = strings.TrimSuffix(strings.TrimPrefix(address, "["), "]") + if address == "*" { + address = "0.0.0.0" + } + return address, port, true +} + +// Published parses `docker ps --format '{{.Names}}\t{{.Ports}}'`. Only what is published on the +// machine counts; a port a container exposes and nothing publishes is not reachable from outside it. +func Published(out string) []Reach { + var reached []Reach + for _, line := range strings.Split(out, "\n") { + name, ports, ok := strings.Cut(strings.TrimSpace(line), "\t") + if !ok { + continue + } + for _, mapping := range strings.Split(ports, ",") { + reached = append(reached, mappingOf(name, strings.TrimSpace(mapping))...) + } + } + return reached +} + +// mappingOf reads "0.0.0.0:9000-9001->9000-9001/tcp" into one reach per port. +func mappingOf(name, mapping string) []Reach { + outer, inner, ok := strings.Cut(mapping, "->") + if !ok { + return nil + } + inner, protocol, ok := strings.Cut(inner, "/") + if !ok { + return nil + } + i := strings.LastIndex(outer, ":") + if i < 0 { + return nil + } + address := strings.TrimSuffix(strings.TrimPrefix(outer[:i], "["), "]") + from, to, ok := portRange(outer[i+1:]) + if !ok { + return nil + } + cfrom, _, ok := portRange(inner) + if !ok { + return nil + } + var reached []Reach + for p := from; p <= to; p++ { + reached = append(reached, Reach{Protocol: protocol, Address: address, Port: p, By: name, + Published: true, ContainerPort: cfrom + (p - from)}) + } + return reached +} + +func portRange(s string) (int, int, bool) { + a, b, isRange := strings.Cut(s, "-") + from, err := strconv.Atoi(a) + if err != nil { + return 0, 0, false + } + if !isRange { + return from, from, true + } + to, err := strconv.Atoi(b) + if err != nil || to < from { + return 0, 0, false + } + return from, to, true +} + +// Merge puts the published ports beside the sockets, dropping the runtime proxy's own socket for a +// port that is reported as published already, and sorts the whole by port. +func Merge(sockets, published []Reach) []Reach { + key := func(r Reach) string { return r.Protocol + " " + r.Address + " " + strconv.Itoa(r.Port) } + isPublished := map[string]bool{} + for _, p := range published { + isPublished[key(p)] = true + } + var out []Reach + for _, s := range sockets { + if s.By == "docker-proxy" && isPublished[key(s)] { + continue + } + out = append(out, s) + } + out = append(out, published...) + sort.SliceStable(out, func(i, j int) bool { + if out[i].Port != out[j].Port { + return out[i].Port < out[j].Port + } + if out[i].Protocol != out[j].Protocol { + return out[i].Protocol < out[j].Protocol + } + return out[i].Address < out[j].Address + }) + return out +} diff --git a/internal/reachable/collect_test.go b/internal/reachable/collect_test.go new file mode 100644 index 0000000..1045631 --- /dev/null +++ b/internal/reachable/collect_test.go @@ -0,0 +1,100 @@ +package reachable + +import ( + "context" + "os" + "strings" + "testing" +) + +// Defends novox/hq ADR 0100: converging previews every listening socket and every published +// container port. Fixtures are captured from a real machine. + +func fixture(t *testing.T, name string) string { + t.Helper() + raw, err := os.ReadFile("testdata/" + name) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func find(rs []Reach, protocol, address string, port int) (Reach, bool) { + for _, r := range rs { + if r.Protocol == protocol && r.Address == address && r.Port == port { + return r, true + } + } + return Reach{}, false +} + +func TestSocketsAreReadWithWhatHoldsThem(t *testing.T) { + got := Sockets(fixture(t, "ss.txt")) + if r, ok := find(got, "tcp", "0.0.0.0", 22); !ok || r.By != "sshd" { + t.Errorf("ssh not read: %+v", r) + } + if r, ok := find(got, "tcp", "::", 445); !ok || r.By != "smbd" { + t.Errorf("an IPv6 wildcard listener not read: %+v", r) + } + if _, ok := find(got, "udp", "fe80::849e:ccff:fea8:24c7", 123); !ok { + t.Error("a link-local address with a scope was not read") + } + if r, ok := find(got, "udp", "127.0.0.1", 53); !ok || r.By != "dnsmasq" { + t.Errorf("a loopback udp socket not read: %+v", r) + } +} + +func TestPublishedPortsNameTheirContainerAndItsPort(t *testing.T) { + got := Published(fixture(t, "docker-ps.txt")) + if r, ok := find(got, "tcp", "0.0.0.0", 8770); !ok || r.By != "whisper" || r.ContainerPort != 8000 || !r.Published { + t.Errorf("a published port: %+v", r) + } + if r, ok := find(got, "tcp", "0.0.0.0", 9001); !ok || r.ContainerPort != 9001 { + t.Errorf("a published range was not expanded: %+v", r) + } + if r, ok := find(got, "tcp", "127.0.0.1", 15673); !ok || r.ContainerPort != 15672 { + t.Errorf("a loopback-published port: %+v", r) + } + for _, r := range got { + if r.By == "umami_db" { + t.Errorf("an exposed and unpublished port was reported reachable: %+v", r) + } + } +} + +func TestAPublishedPortIsReportedOnceAsPublished(t *testing.T) { + merged := Merge(Sockets(fixture(t, "ss.txt")), Published(fixture(t, "docker-ps.txt"))) + n := 0 + for _, r := range merged { + if r.Protocol == "tcp" && r.Address == "0.0.0.0" && r.Port == 8770 { + n++ + if !r.Published { + t.Errorf("the runtime's proxy was reported instead of the published port: %+v", r) + } + } + } + if n != 1 { + t.Errorf("port 8770 reported %d times", n) + } + if _, ok := find(merged, "tcp", "0.0.0.0", 22); !ok { + t.Error("a socket was lost in the merge") + } +} + +func TestCollectAsksSsAndTheRuntime(t *testing.T) { + var asked []string + run := func(_ context.Context, name string, args ...string) (string, error) { + asked = append(asked, name+" "+strings.Join(args, " ")) + if name == "ss" { + return fixture(t, "ss.txt"), nil + } + return fixture(t, "docker-ps.txt"), nil + } + got, err := Collect(context.Background(), run) + if err != nil || len(got) == 0 { + t.Fatalf("%v %v", got, err) + } + if len(asked) != 2 { + t.Errorf("asked %v", asked) + } +} diff --git a/internal/reachable/testdata/docker-ps.txt b/internal/reachable/testdata/docker-ps.txt new file mode 100644 index 0000000..41d56c1 --- /dev/null +++ b/internal/reachable/testdata/docker-ps.txt @@ -0,0 +1,7 @@ +mesh-controller-check-adoption 127.0.0.1:55541->5432/tcp +umami_db 5432/tcp +whisper 0.0.0.0:8770->8000/tcp, [::]:8770->8000/tcp +keycloak 8443/tcp, 127.0.0.1:28080->8080/tcp +minio-lb 0.0.0.0:9000-9001->9000-9001/tcp, [::]:9000-9001->9000-9001/tcp +wonderful_mahavira +anton-lavinmq 0.0.0.0:5680->5672/tcp, [::]:5680->5672/tcp, 127.0.0.1:15673->15672/tcp diff --git a/internal/reachable/testdata/ss.txt b/internal/reachable/testdata/ss.txt new file mode 100644 index 0000000..4d1c61e --- /dev/null +++ b/internal/reachable/testdata/ss.txt @@ -0,0 +1,23 @@ +udp UNCONN 0 0 0.0.0.0:55558 0.0.0.0:* users:(("firefox",pid=2283907,fd=288)) +udp UNCONN 0 0 0.0.0.0:59541 0.0.0.0:* users:(("firefox",pid=2283907,fd=241)) +udp UNCONN 0 0 127.0.0.1:53 0.0.0.0:* users:(("dnsmasq",pid=1189392,fd=6)) +udp UNCONN 0 0 0.0.0.0:33525 0.0.0.0:* users:(("firefox",pid=2283907,fd=304)) +udp UNCONN 0 0 0.0.0.0:41749 0.0.0.0:* users:(("firefox",pid=2283907,fd=351)) +tcp LISTEN 0 4096 127.0.0.1:55541 0.0.0.0:* users:(("docker-proxy",pid=4108732,fd=7)) +tcp LISTEN 0 4096 0.0.0.0:9001 0.0.0.0:* users:(("docker-proxy",pid=1849130,fd=7)) +tcp LISTEN 0 4096 0.0.0.0:8770 0.0.0.0:* users:(("docker-proxy",pid=1920035,fd=7)) +tcp LISTEN 0 50 0.0.0.0:445 0.0.0.0:* users:(("smbd",pid=1248,fd=29)) +tcp LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=1188536,fd=6)) +tcp LISTEN 0 50 0.0.0.0:139 0.0.0.0:* users:(("smbd",pid=1248,fd=30)) +tcp LISTEN 0 4096 127.0.0.1:5432 0.0.0.0:* users:(("docker-proxy",pid=1854543,fd=7)) +tcp LISTEN 0 32 127.0.0.1:53 0.0.0.0:* users:(("dnsmasq",pid=1189392,fd=7)) +tcp LISTEN 0 4096 127.0.0.1:15673 0.0.0.0:* users:(("docker-proxy",pid=3170,fd=7)) +tcp LISTEN 0 4096 [::]:9001 [::]:* users:(("docker-proxy",pid=1849138,fd=7)) +tcp LISTEN 0 4096 [::]:8770 [::]:* users:(("docker-proxy",pid=1920043,fd=7)) +tcp LISTEN 0 50 [::]:445 [::]:* users:(("smbd",pid=1248,fd=27)) +tcp LISTEN 0 128 [::]:22 [::]:* users:(("sshd",pid=1188536,fd=7)) +tcp LISTEN 0 50 [::]:139 [::]:* users:(("smbd",pid=1248,fd=28)) +udp UNCONN 0 0 [fd42:f8c5:dae:d74c::1]:53 [::]:* +udp UNCONN 0 0 [fe80::849e:ccff:fea8:24c7]%veth6b2b7ba:123 [::]:* +udp UNCONN 0 0 [fe80::e45a:90ff:feca:148f]%vethb5e5a61:123 [::]:* +udp UNCONN 0 0 [fe80::c4ed:ccff:feb1:afd2]%veth005a182:123 [::]:* diff --git a/internal/store/store.go b/internal/store/store.go index d9c097f..d2a2da7 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -69,6 +69,23 @@ type Applied struct { // person who edits a managed file watches their change vanish every few minutes with nothing // anywhere saying why. Wrote string `json:"wrote,omitempty"` + + // Into is set for a file written into rather than over (novox/hq ADR 0102): the format, what + // each of the mesh's keys held before it set them, which of them were absent, and whether the + // file itself was — so undeclaring it gives the machine back exactly what it had. + Into *Into `json:"into,omitempty"` +} + +// Into is what a file written into held before the mesh's keys. +type Into struct { + Format string `json:"format"` + Before map[string]json.RawMessage `json:"before,omitempty"` + Absent []string `json:"absent,omitempty"` + Created bool `json:"created,omitempty"` + // Added is, for each key whose declared value is a list, exactly the members the mesh added + // to the machine's list — never a member that was already there. Undeclared, only these go, + // and drift is judged on these alone (novox/hq ADR 0102). + Added map[string][]json.RawMessage `json:"added,omitempty"` } // State is the whole of what a node knows about what it has done. @@ -77,6 +94,117 @@ type State struct { // undoing in reverse is the only ordering the host can derive without deciding anything. Resources []Applied `json:"resources"` UpdatedAt time.Time `json:"updated_at"` + + // Held is what this host found on the machine and is keeping as it is, until the module + // declaring it is taken (novox/hq ADR 0100). Never a Resource: nothing here was applied, so + // nothing here is ever removed as an orphan — what is held is not the host's to remove, even + // when its module is unassigned. + Held []Held `json:"held,omitempty"` + + // Firewall is the firewall found on this machine when it was first adopted, and whether the + // mesh has since retired it (novox/hq ADR 0100). Nil on a node that was never adopted. + Firewall *FoundFirewall `json:"firewall,omitempty"` +} + +// FoundFirewall is what the host found filtering this machine, and what it did about it. +type FoundFirewall struct { + // Kind is ufw or none: an unsupported kind is refused adoption, never recorded. + Kind string `json:"kind"` + // WasActive is whether it was in force when found — which is what converging the node + // retires, and returning it to adopted restores. + WasActive bool `json:"was_active,omitempty"` + // DisabledByMesh is set when converging retired it, so returning to adopted enables it again + // and nothing else ever does. + DisabledByMesh bool `json:"disabled_by_mesh,omitempty"` + // Forward is each family's forward policy as it was before the mesh disabled the firewall, + // by the tool that sets it — recorded before, so a retirement retried puts back what the + // machine had. + Forward map[string]string `json:"forward,omitempty"` + FoundAt time.Time `json:"found_at"` +} + +// Held is one thing found on an adopted node — a file, directory or container present at a declared +// path or name, or a service's unit, with no record of this host having made it — kept as it was +// found; or what would reach one: a container mounting found data, an action run in a held +// container (novox/hq ADR 0100, ADR 0103). +type Held struct { + ID string `json:"id"` + Module string `json:"module"` + Kind string `json:"kind"` + Target string `json:"target"` + // Since is when it was first found. It stays held from then until its module is taken, even + // if it disappears: a vanished file is reported, not recreated. + Since time.Time `json:"since"` + + // A file's content as found, by digest; its mode and owner; and where the original was kept + // before anything else could happen to it. + Digest string `json:"digest,omitempty"` + Mode string `json:"mode,omitempty"` + Owner string `json:"owner,omitempty"` + Kept string `json:"kept,omitempty"` + + // A container's id as found, and whether it was running — or a service's unit, whether it + // was running. + Container string `json:"container,omitempty"` + Running bool `json:"running,omitempty"` + + // Why says what was found when it is not the resource's own target: the path or volume a + // container would mount, or the held container an action would run in (novox/hq ADR 0103). + Why string `json:"why,omitempty"` + + // Changed is what something other than the mesh has done to it since it was found — + // rewritten, stopped, replaced or gone — and empty while it is as found. Reported, never + // reverted: that is how a predecessor still writing is caught. + Changed string `json:"changed,omitempty"` + ChangedAt time.Time `json:"changed_at,omitempty"` +} + +// Recorded reports whether this host has a record, of any origin, of putting something of this +// kind at this target. What it has a record of is not found: it wrote it, in this life of the node +// or an earlier one — including a foundation raised from the bundle and adopted as modules later +// (novox/hq ADR 0078). +func (s State) Recorded(kind, target string) bool { + for _, r := range s.Resources { + if r.Type == kind && r.Target == target { + return true + } + } + return false +} + +// HeldAt returns what is held under a resource id. +func (s State) HeldAt(id string) (Held, bool) { + for _, h := range s.Held { + if h.ID == id { + return h, true + } + } + return Held{}, false +} + +// RecordHeld adds or replaces what is held under one id, preserving order. +func (s *State) RecordHeld(h Held) { + for i, existing := range s.Held { + if existing.ID == h.ID { + s.Held[i] = h + return + } + } + s.Held = append(s.Held, h) +} + +// Release drops a hold, once its module is taken and the host has converged what was held. +func (s *State) Release(id string) { + kept := s.Held[:0] + for _, h := range s.Held { + if h.ID != id { + kept = append(kept, h) + } + } + s.Held = kept + if len(s.Held) == 0 { + s.Held = nil + } } // Find returns what was applied under an identity. diff --git a/internal/system/arch.go b/internal/system/arch.go index 178570d..9582757 100644 --- a/internal/system/arch.go +++ b/internal/system/arch.go @@ -246,3 +246,33 @@ func (arch) AddUserToGroup(ctx context.Context, run Runner, name, group string) } return nil } + +// ServiceUnitFile says where the service manager loads a unit from — systemd's FragmentPath. It +// is how the host tells a unit an administrator installed, under /etc or /run, from one a package +// ships under /usr (novox/hq ADR 0103). Empty, with no error, for a unit that loads from nowhere. +func (arch) ServiceUnitFile(ctx context.Context, run Runner, unit string) (string, error) { + out, err := run(ctx, "systemctl", "show", unit, "--property=FragmentPath") + if err != nil { + return "", fmt.Errorf("the service manager did not say where %s comes from: %w", unit, err) + } + for _, line := range strings.Split(out, "\n") { + if path, ok := strings.CutPrefix(strings.TrimSpace(line), "FragmentPath="); ok { + return strings.TrimSpace(path), nil + } + } + return "", nil +} + +// ReloadUnits has systemd read its unit files again. A unit file that changed on disk is otherwise +// ignored: a restart runs the unit systemd already loaded, and the new text only takes effect +// after a reload nobody asked for. +func (arch) ReloadUnits(ctx context.Context, run Runner) error { + _, err := run(ctx, "systemctl", "daemon-reload") + return err +} + +// ReloadService tells a running unit to read its configuration again, without stopping it. +func (arch) ReloadService(ctx context.Context, run Runner, unit string) error { + _, err := run(ctx, "systemctl", "reload", unit) + return err +} diff --git a/internal/system/system.go b/internal/system/system.go index 3f928e6..25ae307 100644 --- a/internal/system/system.go +++ b/internal/system/system.go @@ -178,6 +178,9 @@ func everyShape() []declaration.Type { // it: the mesh's own code runs as a process on the machine, and only software that // genuinely needs isolation asks for a container. declaration.TypeProcess, + // An opening is a rule in the firewall found on the machine, which a partial host neither + // has nor can manage (novox/hq ADR 0100). + declaration.TypeOpening, } }