Files
mesh-host/internal/apply/opening_test.go
T

493 lines
19 KiB
Go

package apply
import (
"context"
"errors"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force; converging the
// node retires it by disabling it, and returning the node to adopted enables it again.
type ufwMachine struct {
installed, active bool
rules []string
ruleset string
asked []string
// forward is iptables' forward policy when set; empty is a machine without iptables. failP
// is how many -P calls fail before one succeeds.
forward string
failP int
}
func (u *ufwMachine) iptables(args []string) (string, error) {
if len(args) == 3 && args[0] == "-P" {
if u.failP > 0 {
u.failP--
return "", errors.New("iptables: resource temporarily unavailable")
}
u.forward = args[2]
return "", nil
}
return "-P FORWARD " + u.forward + "\n-A FORWARD -j DOCKER-USER\n", nil
}
func (u *ufwMachine) run(_ context.Context, name string, args ...string) (string, error) {
u.asked = append(u.asked, name+" "+strings.Join(args, " "))
switch name {
case "nft":
return u.ruleset, nil
case "iptables":
if u.forward == "" {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
return u.iptables(args)
case "ufw":
if !u.installed {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
default:
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
switch args[0] {
case "status":
if u.active {
return "Status: active\n", nil
}
return "Status: inactive\n", nil
case "show":
out := "Added user rules (see 'ufw status' for running firewall):\n"
for _, r := range u.rules {
out += "ufw " + r + "\n"
}
return out, nil
case "--force":
u.active = true
return "", nil
case "disable":
u.active = false
if u.forward != "" {
u.forward = "ACCEPT" // as measured: ufw disable opens the forward policy
}
return "", nil
case "delete":
want := strings.Join(args[1:], " ")
for i, r := range u.rules {
if strings.ReplaceAll(r, "'", "") == want {
u.rules = append(u.rules[:i], u.rules[i+1:]...)
return "", nil
}
}
return "", errors.New("Could not delete non-existent rule")
default:
// Printed back the way it was given, with the comment quoted as ufw does.
line := strings.Join(args[:len(args)-1], " ") + " '" + args[len(args)-1] + "'"
u.rules = append(u.rules, line)
return "", nil
}
}
func (u *ufwMachine) index(prefix string) int {
for i, a := range u.asked {
if strings.HasPrefix(a, prefix) {
return i
}
}
return -1
}
const busOpening = `{"id":"adoption.opening-tcp-5671-incoming","type":"opening","port":5671,"protocol":"tcp","from":"everywhere","path":"incoming"}`
func withConf(dir string) string {
return `{"id":"x.conf","type":"file","path":"` + filepath.Join(dir, "x.conf") + `","content":"x\n"}`
}
func applyWith(t *testing.T, d *declaration.Declaration, known store.State, run Runner) (Report, store.State, error) {
t.Helper()
return ApplyKeeping(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil,
KeepIn(t.TempDir()))
}
func TestAnOpeningOnAMachineWithNoFirewallChangesNothing(t *testing.T) {
dir := t.TempDir()
u := &ufwMachine{}
report, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
o := outcomeOf(report, "adoption.opening-tcp-5671-incoming")
if o.Action != "unchanged" || !strings.Contains(o.Detail, "nothing filters this port") {
t.Errorf("an opening with no firewall: %+v", o)
}
if state.Firewall == nil || state.Firewall.Kind != "none" {
t.Errorf("the firewall found was not recorded: %+v", state.Firewall)
}
}
func TestAnUnsupportedFirewallRefusesTheWholeDeclaration(t *testing.T) {
dir := t.TempDir()
u := &ufwMachine{ruleset: "table inet filter {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t}\n}\n"}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err == nil || !strings.Contains(err.Error(), "no host speaks that firewall") {
t.Fatalf("an unsupported firewall was not refused: %v", err)
}
if _, statErr := os.Stat(filepath.Join(dir, "x.conf")); !errors.Is(statErr, os.ErrNotExist) {
t.Error("part of a refused declaration was applied")
}
if state.Firewall != nil {
t.Errorf("an unsupported firewall was recorded: %+v", state.Firewall)
}
}
func TestConvergingRetiresTheFoundFirewallAndReturningRestoresIt(t *testing.T) {
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
// Adopted: the opening goes through ufw.
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
if len(u.rules) != 2 || !u.active {
t.Fatalf("adopted: rules %v, active %v", u.rules, u.active)
}
if state.Firewall == nil || state.Firewall.Kind != "ufw" || !state.Firewall.WasActive {
t.Fatalf("adopted: firewall recorded as %+v", state.Firewall)
}
// Converged: the derived filter is loaded, the opening's rule goes, and only then is ufw
// disabled — never reset.
u.asked = nil
u.ruleset = "table inet mesh\n"
converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
_, state, err = applyWith(t, converged, state, u.run)
if err != nil {
t.Fatal(err)
}
if u.active || !state.Firewall.DisabledByMesh {
t.Fatalf("converged: ufw still active (%v) or not recorded as retired (%+v)", u.active, state.Firewall)
}
if len(u.rules) != 1 || u.rules[0] != "allow 22/tcp" {
t.Errorf("converged: the operator's rules were touched, or the mesh's left: %v", u.rules)
}
// What protected the adopted node goes last: after the derived filter applied and ufw was
// retired (novox/hq ADR 0103).
if del, dis := u.index("ufw delete"), u.index("ufw disable"); dis < 0 || del < dis {
t.Errorf("converged: the opening was removed before ufw was retired: %v", u.asked)
}
for _, a := range u.asked {
if strings.Contains(a, "reset") {
t.Errorf("converged: ufw was reset: %s", a)
}
}
// Converged again: nothing more to retire.
u.asked = nil
if _, state, err = applyWith(t, converged, state, u.run); err != nil {
t.Fatal(err)
}
if u.index("ufw") >= 0 {
t.Errorf("a converged node kept talking to a retired ufw: %v", u.asked)
}
// Returned to adopted: ufw is enabled before the opening is converged through it.
u.asked = nil
_, state, err = applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), state, u.run)
if err != nil {
t.Fatal(err)
}
if !u.active || state.Firewall.DisabledByMesh {
t.Fatalf("returned: ufw active %v, record %+v", u.active, state.Firewall)
}
if en, add := u.index("ufw --force enable"), u.index("ufw allow"); en < 0 || add < en {
t.Errorf("returned: ufw was not enabled before the opening was added: %v", u.asked)
}
if len(u.rules) != 2 {
t.Errorf("returned: the opening was not converged again: %v", u.rules)
}
}
func TestAConvergedNodeThatWasNeverAdoptedNeverAsksAboutAFirewall(t *testing.T) {
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true}
if _, _, err := applyWith(t, parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`), store.State{}, u.run); err != nil {
t.Fatal(err)
}
if len(u.asked) != 0 {
t.Errorf("a converged apply asked the machine about its firewall: %v", u.asked)
}
}
func TestAnOpeningOnAConvergedNodeIsRefused(t *testing.T) {
if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` + busOpening + `]}`)); err == nil {
t.Error("an opening was accepted on a node the declaration does not say is adopted")
}
}
func TestACarriedApplyOnAnAdoptedNodeLeavesItsFirewallInForce(t *testing.T) {
// The bundle, re-applied by the installer or the one-shot CLI, never says a node is adopted.
// That is not the controller converging it, so ufw must stay enabled (novox/hq ADR 0100).
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
u.asked = nil
carried := parse(t, `{"declaration":1,"resources":[`+withConf(filepath.Join(dir, "bundle"))+`]}`)
_, state, err = ApplyKeeping(context.Background(), archHost(t), carried, state, store.OriginCarried,
u.run, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if !u.active || state.Firewall.DisabledByMesh || u.index("ufw disable") >= 0 {
t.Fatalf("a carried apply retired the found firewall: active %v, record %+v, asked %v",
u.active, state.Firewall, u.asked)
}
}
func TestAFlipThatFailsKeepsTheGuardAndTheOpenings(t *testing.T) {
// Converging a node removes its openings and its guard only once everything else applied and
// the found firewall is retired. A flip that fails part-way keeps them, so the store is never
// left unguarded behind a filter that did not load (novox/hq ADR 0103).
dir := t.TempDir()
guard := filepath.Join(dir, "guard.nft")
guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}`
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+guardFile+","+withConf(dir)),
store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
// The derived filter cannot be written: its path is under a file.
blocked := filepath.Join(dir, "not-a-directory")
if err := os.WriteFile(blocked, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
filter := `{"id":"nftables.config","type":"file","path":"` + filepath.Join(blocked, "nftables.conf") + `","content":"table inet mesh {}\n"}`
converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`,`+filter+`]}`)
u.asked = nil
u.ruleset = "table inet mesh\n" // what the filter's unit loads once the file is written
_, state, err = applyWith(t, converged, state, u.run)
if err == nil {
t.Fatal("the failing flip reported success")
}
if !u.active || u.index("ufw disable") >= 0 {
t.Errorf("the found firewall was retired by a flip that failed: %v", u.asked)
}
if len(u.rules) != 2 || u.index("ufw delete") >= 0 {
t.Errorf("the opening was removed by a flip that failed: %v", u.rules)
}
if _, statErr := os.Stat(guard); statErr != nil {
t.Errorf("the guard was removed by a flip that failed: %v", statErr)
}
for _, id := range []string{"adoption.guard", "adoption.opening-tcp-5671-incoming"} {
if _, ok := state.Find(id); !ok {
t.Errorf("%s was forgotten, so the next flip would never remove it", id)
}
}
// Fixed, the next flip completes: filter, retire, and only then the guard and the openings.
if err := os.Remove(blocked); err != nil {
t.Fatal(err)
}
u.asked = nil
_, state, err = applyWith(t, converged, state, u.run)
if err != nil {
t.Fatal(err)
}
if u.active || len(u.rules) != 1 {
t.Errorf("the completed flip left ufw active %v, rules %v", u.active, u.rules)
}
if _, statErr := os.Stat(guard); !errors.Is(statErr, os.ErrNotExist) {
t.Errorf("the guard outlived the completed flip: %v", statErr)
}
if _, ok := state.Find("adoption.guard"); ok {
t.Error("the guard is still recorded after the completed flip")
}
}
func TestAnOpeningAFoundRuleAnswersIsReportedSatisfied(t *testing.T) {
// novox/hq ADR 0103: the mesh adds nothing beside a rule ufw would take for the same one.
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 5671/tcp"}}
report, _, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
o := outcomeOf(report, "adoption.opening-tcp-5671-incoming")
if o.Action != "unchanged" || !strings.Contains(o.Detail, "satisfied by a rule found in ufw (allow 5671/tcp)") {
t.Errorf("the opening was not reported satisfied: %+v", o)
}
if len(u.rules) != 2 || u.index("ufw allow") >= 0 {
t.Errorf("a rule was added beside the found one: %v", u.rules)
}
}
// Defends novox/hq ADR 0103: returned to adopted, the guard is up before the derived filter's
// orphans go, and stays up if removing them fails.
func TestReturningToAdoptedLoadsTheGuardBeforeRemovingTheFilter(t *testing.T) {
dir := t.TempDir()
guard := filepath.Join(dir, "guard.nft")
guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}`
for _, stopFails := range []bool{false, true} {
_ = os.Remove(guard)
guardUpAtStop := false
run := func(_ context.Context, name string, args ...string) (string, error) {
if name != "systemctl" {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
switch args[0] {
case "show":
return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\n", nil
case "stop":
_, err := os.Stat(guard)
guardUpAtStop = err == nil
if stopFails {
return "", errors.New("the filter would not stop")
}
}
return "", nil
}
converged := store.State{Resources: []store.Applied{
{ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run)
if !guardUpAtStop {
t.Errorf("stop fails %v: the derived filter was stopped before the guard was written", stopFails)
}
if stopFails {
if err == nil {
t.Error("a failed removal was not reported")
}
if _, statErr := os.Stat(guard); statErr != nil {
t.Error("the guard is not up after the filter's removal failed")
}
if _, ok := state.Find("adoption.guard"); !ok {
t.Error("the guard applied before the failure was not recorded")
}
if _, still := state.Find("nftables.load"); !still {
t.Error("the filter that would not stop was forgotten, so nothing would stop it later")
}
} else if err != nil {
t.Fatal(err)
}
}
}
func TestAStaleOpeningOnAnAdoptedNodeIsRemovedAsAnyOrphan(t *testing.T) {
// Only the flip defers the adoption's own orphans; an adopted node drops a stale opening at
// once, before what replaces it is applied.
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
u.asked = nil
other := `{"id":"adoption.opening-tcp-5000-incoming","type":"opening","port":5000,"protocol":"tcp","from":"everywhere","path":"incoming"}`
if _, _, err = applyWith(t, adopted(t, `{"taken":[]}`, other+","+withConf(dir)), state, u.run); err != nil {
t.Fatal(err)
}
if del, add := u.index("ufw delete"), u.index("ufw allow"); del < 0 || add < 0 || del > add {
t.Errorf("the stale opening was not removed before the new one was added: %v", u.asked)
}
}
func TestARetiredFirewallRetriedStillPutsBackTheForwardPolicy(t *testing.T) {
// The forward policy is recorded before ufw is disabled, so a retirement that failed after
// the disable restores what the machine had, not what the disable left (novox/hq ADR 0100).
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, forward: "DROP", failP: 1, ruleset: "table inet mesh\n"}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
if _, state, err = applyWith(t, converged, state, u.run); err == nil {
t.Fatal("the failed restore was not reported")
}
if u.active || u.forward != "ACCEPT" || state.Firewall.Forward["iptables"] != "DROP" {
t.Fatalf("after the failed attempt: active %v, forward %s, recorded %+v", u.active, u.forward, state.Firewall)
}
if _, state, err = applyWith(t, converged, state, u.run); err != nil {
t.Fatal(err)
}
if u.forward != "DROP" || !state.Firewall.DisabledByMesh {
t.Errorf("the retry did not put the forward policy back: %s, %+v", u.forward, state.Firewall)
}
}
func TestAGuardThatFailsLeavesTheDerivedFilterInForce(t *testing.T) {
// Returning to adopted: if the guard cannot be raised, the filter it replaces must not be
// stopped — its stop deletes the mesh's table, and the node would have neither (novox/hq ADR 0103).
dir := t.TempDir()
blocked := filepath.Join(dir, "not-a-directory")
if err := os.WriteFile(blocked, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
guardFile := `{"id":"adoption.guard","type":"file","path":"` + filepath.Join(blocked, "guard.nft") +
`","content":"table inet mesh_guard {}\n"}`
stopped := false
run := func(_ context.Context, name string, args ...string) (string, error) {
if name != "systemctl" {
return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
}
if args[0] == "show" {
return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\n", nil
}
if args[0] == "stop" {
stopped = true
}
return "", nil
}
converged := store.State{Resources: []store.Applied{
{ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run)
if err == nil {
t.Fatal("a guard that could not be written reported success")
}
if stopped {
t.Error("the derived filter was stopped though the guard is not up")
}
if _, gone := state.Find("nftables.load"); !gone {
t.Error("the filter was forgotten, so nothing would ever stop it")
}
}
func TestUfwIsNotRetiredUntilTheMeshsOwnFilterIsLoaded(t *testing.T) {
// The flip retires the found firewall because the mesh's derived filter takes its place. If
// that table is not loaded, retiring would leave the machine filtering nothing (novox/hq ADR 0100).
dir := t.TempDir()
u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
if err != nil {
t.Fatal(err)
}
converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
_, state, err = applyWith(t, converged, state, u.run)
if err == nil || !strings.Contains(err.Error(), "table inet mesh") {
t.Fatalf("ufw was retired with nothing in its place: %v", err)
}
if !u.active || state.Firewall.DisabledByMesh {
t.Errorf("ufw was disabled: active %v, %+v", u.active, state.Firewall)
}
// Once the table is loaded, the same declaration retires it.
u.ruleset = "table inet mesh\n"
if _, state, err = applyWith(t, converged, state, u.run); err != nil {
t.Fatal(err)
}
if u.active || !state.Firewall.DisabledByMesh {
t.Errorf("ufw was not retired once the mesh's filter was loaded: active %v, %+v", u.active, state.Firewall)
}
}