mesh-host/internal/apply/opening_test.go

package apply

import (
	"context"
	"errors"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
)

// Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force; converging the
// node retires it by disabling it, and returning the node to adopted enables it again.

type ufwMachine struct {
	installed, active bool
	rules             []string
	ruleset           string
	asked             []string

	// forward is iptables' forward policy when set; empty is a machine without iptables. failP
	// is how many -P calls fail before one succeeds.
	forward string
	failP   int
}

func (u *ufwMachine) iptables(args []string) (string, error) {
	if len(args) == 3 && args[0] == "-P" {
		if u.failP > 0 {
			u.failP--
			return "", errors.New("iptables: resource temporarily unavailable")
		}
		u.forward = args[2]
		return "", nil
	}
	return "-P FORWARD " + u.forward + "\n-A FORWARD -j DOCKER-USER\n", nil
}

func (u *ufwMachine) run(_ context.Context, name string, args ...string) (string, error) {
	u.asked = append(u.asked, name+" "+strings.Join(args, " "))
	switch name {
	case "nft":
		return u.ruleset, nil
	case "iptables":
		if u.forward == "" {
			return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
		}
		return u.iptables(args)
	case "ufw":
		if !u.installed {
			return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
		}
	default:
		return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
	}
	switch args[0] {
	case "status":
		if u.active {
			return "Status: active\n", nil
		}
		return "Status: inactive\n", nil
	case "show":
		out := "Added user rules (see 'ufw status' for running firewall):\n"
		for _, r := range u.rules {
			out += "ufw " + r + "\n"
		}
		return out, nil
	case "--force":
		u.active = true
		return "", nil
	case "disable":
		u.active = false
		if u.forward != "" {
			u.forward = "ACCEPT" // as measured: ufw disable opens the forward policy
		}
		return "", nil
	case "delete":
		want := strings.Join(args[1:], " ")
		for i, r := range u.rules {
			if strings.ReplaceAll(r, "'", "") == want {
				u.rules = append(u.rules[:i], u.rules[i+1:]...)
				return "", nil
			}
		}
		return "", errors.New("Could not delete non-existent rule")
	default:
		// Printed back the way it was given, with the comment quoted as ufw does.
		line := strings.Join(args[:len(args)-1], " ") + " '" + args[len(args)-1] + "'"
		u.rules = append(u.rules, line)
		return "", nil
	}
}

func (u *ufwMachine) index(prefix string) int {
	for i, a := range u.asked {
		if strings.HasPrefix(a, prefix) {
			return i
		}
	}
	return -1
}

const busOpening = `{"id":"adoption.opening-tcp-5671-incoming","type":"opening","port":5671,"protocol":"tcp","from":"everywhere","path":"incoming"}`

func withConf(dir string) string {
	return `{"id":"x.conf","type":"file","path":"` + filepath.Join(dir, "x.conf") + `","content":"x\n"}`
}

func applyWith(t *testing.T, d *declaration.Declaration, known store.State, run Runner) (Report, store.State, error) {
	t.Helper()
	return ApplyKeeping(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil,
		KeepIn(t.TempDir()))
}

func TestAnOpeningOnAMachineWithNoFirewallChangesNothing(t *testing.T) {
	dir := t.TempDir()
	u := &ufwMachine{}
	report, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
	if err != nil {
		t.Fatal(err)
	}
	o := outcomeOf(report, "adoption.opening-tcp-5671-incoming")
	if o.Action != "unchanged" || !strings.Contains(o.Detail, "nothing filters this port") {
		t.Errorf("an opening with no firewall: %+v", o)
	}
	if state.Firewall == nil || state.Firewall.Kind != "none" {
		t.Errorf("the firewall found was not recorded: %+v", state.Firewall)
	}
}

func TestAnUnsupportedFirewallRefusesTheWholeDeclaration(t *testing.T) {
	dir := t.TempDir()
	u := &ufwMachine{ruleset: "table inet filter {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t}\n}\n"}
	_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
	if err == nil || !strings.Contains(err.Error(), "no host speaks that firewall") {
		t.Fatalf("an unsupported firewall was not refused: %v", err)
	}
	if _, statErr := os.Stat(filepath.Join(dir, "x.conf")); !errors.Is(statErr, os.ErrNotExist) {
		t.Error("part of a refused declaration was applied")
	}
	if state.Firewall != nil {
		t.Errorf("an unsupported firewall was recorded: %+v", state.Firewall)
	}
}

func TestConvergingRetiresTheFoundFirewallAndReturningRestoresIt(t *testing.T) {
	dir := t.TempDir()
	u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}

	// Adopted: the opening goes through ufw.
	_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
	if err != nil {
		t.Fatal(err)
	}
	if len(u.rules) != 2 || !u.active {
		t.Fatalf("adopted: rules %v, active %v", u.rules, u.active)
	}
	if state.Firewall == nil || state.Firewall.Kind != "ufw" || !state.Firewall.WasActive {
		t.Fatalf("adopted: firewall recorded as %+v", state.Firewall)
	}

	// Converged: the opening's rule goes, and only then is ufw disabled — never reset.
	u.asked = nil
	converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
	_, state, err = applyWith(t, converged, state, u.run)
	if err != nil {
		t.Fatal(err)
	}
	if u.active || !state.Firewall.DisabledByMesh {
		t.Fatalf("converged: ufw still active (%v) or not recorded as retired (%+v)", u.active, state.Firewall)
	}
	if len(u.rules) != 1 || u.rules[0] != "allow 22/tcp" {
		t.Errorf("converged: the operator's rules were touched, or the mesh's left: %v", u.rules)
	}
	// What protected the adopted node goes last: after the derived filter applied and ufw was
	// retired (novox/hq ADR 0103).
	if del, dis := u.index("ufw delete"), u.index("ufw disable"); dis < 0 || del < dis {
		t.Errorf("converged: the opening was removed before ufw was retired: %v", u.asked)
	}
	for _, a := range u.asked {
		if strings.Contains(a, "reset") {
			t.Errorf("converged: ufw was reset: %s", a)
		}
	}

	// Converged again: nothing more to retire.
	u.asked = nil
	if _, state, err = applyWith(t, converged, state, u.run); err != nil {
		t.Fatal(err)
	}
	if u.index("ufw") >= 0 {
		t.Errorf("a converged node kept talking to a retired ufw: %v", u.asked)
	}

	// Returned to adopted: ufw is enabled before the opening is converged through it.
	u.asked = nil
	_, state, err = applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), state, u.run)
	if err != nil {
		t.Fatal(err)
	}
	if !u.active || state.Firewall.DisabledByMesh {
		t.Fatalf("returned: ufw active %v, record %+v", u.active, state.Firewall)
	}
	if en, add := u.index("ufw --force enable"), u.index("ufw allow"); en < 0 || add < en {
		t.Errorf("returned: ufw was not enabled before the opening was added: %v", u.asked)
	}
	if len(u.rules) != 2 {
		t.Errorf("returned: the opening was not converged again: %v", u.rules)
	}
}

func TestAConvergedNodeThatWasNeverAdoptedNeverAsksAboutAFirewall(t *testing.T) {
	dir := t.TempDir()
	u := &ufwMachine{installed: true, active: true}
	if _, _, err := applyWith(t, parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`), store.State{}, u.run); err != nil {
		t.Fatal(err)
	}
	if len(u.asked) != 0 {
		t.Errorf("a converged apply asked the machine about its firewall: %v", u.asked)
	}
}

func TestAnOpeningOnAConvergedNodeIsRefused(t *testing.T) {
	if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` + busOpening + `]}`)); err == nil {
		t.Error("an opening was accepted on a node the declaration does not say is adopted")
	}
}

func TestACarriedApplyOnAnAdoptedNodeLeavesItsFirewallInForce(t *testing.T) {
	// The bundle, re-applied by the installer or the one-shot CLI, never says a node is adopted.
	// That is not the controller converging it, so ufw must stay enabled (novox/hq ADR 0100).
	dir := t.TempDir()
	u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
	_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
	if err != nil {
		t.Fatal(err)
	}
	u.asked = nil
	carried := parse(t, `{"declaration":1,"resources":[`+withConf(filepath.Join(dir, "bundle"))+`]}`)
	_, state, err = ApplyKeeping(context.Background(), archHost(t), carried, state, store.OriginCarried,
		u.run, nil, nil, nil)
	if err != nil {
		t.Fatal(err)
	}
	if !u.active || state.Firewall.DisabledByMesh || u.index("ufw disable") >= 0 {
		t.Fatalf("a carried apply retired the found firewall: active %v, record %+v, asked %v",
			u.active, state.Firewall, u.asked)
	}
}

func TestAFlipThatFailsKeepsTheGuardAndTheOpenings(t *testing.T) {
	// Converging a node removes its openings and its guard only once everything else applied and
	// the found firewall is retired. A flip that fails part-way keeps them, so the store is never
	// left unguarded behind a filter that did not load (novox/hq ADR 0103).
	dir := t.TempDir()
	guard := filepath.Join(dir, "guard.nft")
	guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}`
	u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
	_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+guardFile+","+withConf(dir)),
		store.State{}, u.run)
	if err != nil {
		t.Fatal(err)
	}

	// The derived filter cannot be written: its path is under a file.
	blocked := filepath.Join(dir, "not-a-directory")
	if err := os.WriteFile(blocked, []byte("x"), 0o644); err != nil {
		t.Fatal(err)
	}
	filter := `{"id":"nftables.config","type":"file","path":"` + filepath.Join(blocked, "nftables.conf") + `","content":"table inet mesh {}\n"}`
	converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`,`+filter+`]}`)
	u.asked = nil
	_, state, err = applyWith(t, converged, state, u.run)
	if err == nil {
		t.Fatal("the failing flip reported success")
	}
	if !u.active || u.index("ufw disable") >= 0 {
		t.Errorf("the found firewall was retired by a flip that failed: %v", u.asked)
	}
	if len(u.rules) != 2 || u.index("ufw delete") >= 0 {
		t.Errorf("the opening was removed by a flip that failed: %v", u.rules)
	}
	if _, statErr := os.Stat(guard); statErr != nil {
		t.Errorf("the guard was removed by a flip that failed: %v", statErr)
	}
	for _, id := range []string{"adoption.guard", "adoption.opening-tcp-5671-incoming"} {
		if _, ok := state.Find(id); !ok {
			t.Errorf("%s was forgotten, so the next flip would never remove it", id)
		}
	}

	// Fixed, the next flip completes: filter, retire, and only then the guard and the openings.
	if err := os.Remove(blocked); err != nil {
		t.Fatal(err)
	}
	u.asked = nil
	_, state, err = applyWith(t, converged, state, u.run)
	if err != nil {
		t.Fatal(err)
	}
	if u.active || len(u.rules) != 1 {
		t.Errorf("the completed flip left ufw active %v, rules %v", u.active, u.rules)
	}
	if _, statErr := os.Stat(guard); !errors.Is(statErr, os.ErrNotExist) {
		t.Errorf("the guard outlived the completed flip: %v", statErr)
	}
	if _, ok := state.Find("adoption.guard"); ok {
		t.Error("the guard is still recorded after the completed flip")
	}
}

func TestAnOpeningAFoundRuleAnswersIsReportedSatisfied(t *testing.T) {
	// novox/hq ADR 0103: the mesh adds nothing beside a rule ufw would take for the same one.
	dir := t.TempDir()
	u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 5671/tcp"}}
	report, _, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
	if err != nil {
		t.Fatal(err)
	}
	o := outcomeOf(report, "adoption.opening-tcp-5671-incoming")
	if o.Action != "unchanged" || !strings.Contains(o.Detail, "satisfied by a rule found in ufw (allow 5671/tcp)") {
		t.Errorf("the opening was not reported satisfied: %+v", o)
	}
	if len(u.rules) != 2 || u.index("ufw allow") >= 0 {
		t.Errorf("a rule was added beside the found one: %v", u.rules)
	}
}

// Defends novox/hq ADR 0103: returned to adopted, the guard is up before the derived filter's
// orphans go, and stays up if removing them fails.
func TestReturningToAdoptedLoadsTheGuardBeforeRemovingTheFilter(t *testing.T) {
	dir := t.TempDir()
	guard := filepath.Join(dir, "guard.nft")
	guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}`
	for _, stopFails := range []bool{false, true} {
		_ = os.Remove(guard)
		guardUpAtStop := false
		run := func(_ context.Context, name string, args ...string) (string, error) {
			if name != "systemctl" {
				return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
			}
			switch args[0] {
			case "show":
				return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\n", nil
			case "stop":
				_, err := os.Stat(guard)
				guardUpAtStop = err == nil
				if stopFails {
					return "", errors.New("the filter would not stop")
				}
			}
			return "", nil
		}
		converged := store.State{Resources: []store.Applied{
			{ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}}
		_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run)
		if !guardUpAtStop {
			t.Errorf("stop fails %v: the derived filter was stopped before the guard was written", stopFails)
		}
		if stopFails {
			if err == nil {
				t.Error("a failed removal was not reported")
			}
			if _, statErr := os.Stat(guard); statErr != nil {
				t.Error("the guard is not up after the filter's removal failed")
			}
			if _, ok := state.Find("adoption.guard"); !ok {
				t.Error("the guard applied before the failure was not recorded")
			}
			if _, still := state.Find("nftables.load"); !still {
				t.Error("the filter that would not stop was forgotten, so nothing would stop it later")
			}
		} else if err != nil {
			t.Fatal(err)
		}
	}
}

func TestAStaleOpeningOnAnAdoptedNodeIsRemovedAsAnyOrphan(t *testing.T) {
	// Only the flip defers the adoption's own orphans; an adopted node drops a stale opening at
	// once, before what replaces it is applied.
	dir := t.TempDir()
	u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}}
	_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run)
	if err != nil {
		t.Fatal(err)
	}
	u.asked = nil
	other := `{"id":"adoption.opening-tcp-5000-incoming","type":"opening","port":5000,"protocol":"tcp","from":"everywhere","path":"incoming"}`
	if _, _, err = applyWith(t, adopted(t, `{"taken":[]}`, other+","+withConf(dir)), state, u.run); err != nil {
		t.Fatal(err)
	}
	if del, add := u.index("ufw delete"), u.index("ufw allow"); del < 0 || add < 0 || del > add {
		t.Errorf("the stale opening was not removed before the new one was added: %v", u.asked)
	}
}

func TestARetiredFirewallRetriedStillPutsBackTheForwardPolicy(t *testing.T) {
	// The forward policy is recorded before ufw is disabled, so a retirement that failed after
	// the disable restores what the machine had, not what the disable left (novox/hq ADR 0100).
	dir := t.TempDir()
	u := &ufwMachine{installed: true, active: true, forward: "DROP", failP: 1}
	_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)), store.State{}, u.run)
	if err != nil {
		t.Fatal(err)
	}
	converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`)
	if _, state, err = applyWith(t, converged, state, u.run); err == nil {
		t.Fatal("the failed restore was not reported")
	}
	if u.active || u.forward != "ACCEPT" || state.Firewall.Forward["iptables"] != "DROP" {
		t.Fatalf("after the failed attempt: active %v, forward %s, recorded %+v", u.active, u.forward, state.Firewall)
	}
	if _, state, err = applyWith(t, converged, state, u.run); err != nil {
		t.Fatal(err)
	}
	if u.forward != "DROP" || !state.Firewall.DisabledByMesh {
		t.Errorf("the retry did not put the forward policy back: %s, %+v", u.forward, state.Firewall)
	}
}

func TestAGuardThatFailsLeavesTheDerivedFilterInForce(t *testing.T) {
	// Returning to adopted: if the guard cannot be raised, the filter it replaces must not be
	// stopped — its stop deletes the mesh's table, and the node would have neither (novox/hq ADR 0103).
	dir := t.TempDir()
	blocked := filepath.Join(dir, "not-a-directory")
	if err := os.WriteFile(blocked, []byte("x"), 0o644); err != nil {
		t.Fatal(err)
	}
	guardFile := `{"id":"adoption.guard","type":"file","path":"` + filepath.Join(blocked, "guard.nft") +
		`","content":"table inet mesh_guard {}\n"}`
	stopped := false
	run := func(_ context.Context, name string, args ...string) (string, error) {
		if name != "systemctl" {
			return "", &exec.Error{Name: name, Err: exec.ErrNotFound}
		}
		if args[0] == "show" {
			return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\n", nil
		}
		if args[0] == "stop" {
			stopped = true
		}
		return "", nil
	}
	converged := store.State{Resources: []store.Applied{
		{ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}}
	_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run)
	if err == nil {
		t.Fatal("a guard that could not be written reported success")
	}
	if stopped {
		t.Error("the derived filter was stopped though the guard is not up")
	}
	if _, gone := state.Find("nftables.load"); !gone {
		t.Error("the filter was forgotten, so nothing would ever stop it")
	}
}