package apply import ( "context" "errors" "os" "os/exec" "path/filepath" "strings" "testing" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" ) // Defends novox/hq ADR 0100: the firewall found on an adopted node stays in force; converging the // node retires it by disabling it, and returning the node to adopted enables it again. type ufwMachine struct { installed, active bool rules []string ruleset string asked []string // forward is iptables' forward policy when set; empty is a machine without iptables. failP // is how many -P calls fail before one succeeds. forward string failP int } func (u *ufwMachine) iptables(args []string) (string, error) { if len(args) == 3 && args[0] == "-P" { if u.failP > 0 { u.failP-- return "", errors.New("iptables: resource temporarily unavailable") } u.forward = args[2] return "", nil } return "-P FORWARD " + u.forward + "\n-A FORWARD -j DOCKER-USER\n", nil } func (u *ufwMachine) run(_ context.Context, name string, args ...string) (string, error) { u.asked = append(u.asked, name+" "+strings.Join(args, " ")) switch name { case "nft": return u.ruleset, nil case "iptables": if u.forward == "" { return "", &exec.Error{Name: name, Err: exec.ErrNotFound} } return u.iptables(args) case "ufw": if !u.installed { return "", &exec.Error{Name: name, Err: exec.ErrNotFound} } default: return "", &exec.Error{Name: name, Err: exec.ErrNotFound} } switch args[0] { case "status": if u.active { return "Status: active\n", nil } return "Status: inactive\n", nil case "show": out := "Added user rules (see 'ufw status' for running firewall):\n" for _, r := range u.rules { out += "ufw " + r + "\n" } return out, nil case "--force": u.active = true return "", nil case "disable": u.active = false if u.forward != "" { u.forward = "ACCEPT" // as measured: ufw disable opens the forward policy } return "", nil case "delete": want := strings.Join(args[1:], " ") for i, r := range u.rules { if strings.ReplaceAll(r, "'", "") == want { u.rules = append(u.rules[:i], u.rules[i+1:]...) return "", nil } } return "", errors.New("Could not delete non-existent rule") default: // Printed back the way it was given, with the comment quoted as ufw does. line := strings.Join(args[:len(args)-1], " ") + " '" + args[len(args)-1] + "'" u.rules = append(u.rules, line) return "", nil } } func (u *ufwMachine) index(prefix string) int { for i, a := range u.asked { if strings.HasPrefix(a, prefix) { return i } } return -1 } const busOpening = `{"id":"adoption.opening-tcp-5671-incoming","type":"opening","port":5671,"protocol":"tcp","from":"everywhere","path":"incoming"}` func withConf(dir string) string { return `{"id":"x.conf","type":"file","path":"` + filepath.Join(dir, "x.conf") + `","content":"x\n"}` } func applyWith(t *testing.T, d *declaration.Declaration, known store.State, run Runner) (Report, store.State, error) { t.Helper() return ApplyKeeping(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil, KeepIn(t.TempDir())) } func TestAnOpeningOnAMachineWithNoFirewallChangesNothing(t *testing.T) { dir := t.TempDir() u := &ufwMachine{} report, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) if err != nil { t.Fatal(err) } o := outcomeOf(report, "adoption.opening-tcp-5671-incoming") if o.Action != "unchanged" || !strings.Contains(o.Detail, "nothing filters this port") { t.Errorf("an opening with no firewall: %+v", o) } if state.Firewall == nil || state.Firewall.Kind != "none" { t.Errorf("the firewall found was not recorded: %+v", state.Firewall) } } func TestAnUnsupportedFirewallRefusesTheWholeDeclaration(t *testing.T) { dir := t.TempDir() u := &ufwMachine{ruleset: "table inet filter {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t}\n}\n"} _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) if err == nil || !strings.Contains(err.Error(), "no host speaks that firewall") { t.Fatalf("an unsupported firewall was not refused: %v", err) } if _, statErr := os.Stat(filepath.Join(dir, "x.conf")); !errors.Is(statErr, os.ErrNotExist) { t.Error("part of a refused declaration was applied") } if state.Firewall != nil { t.Errorf("an unsupported firewall was recorded: %+v", state.Firewall) } } func TestConvergingRetiresTheFoundFirewallAndReturningRestoresIt(t *testing.T) { dir := t.TempDir() u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} // Adopted: the opening goes through ufw. _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) if err != nil { t.Fatal(err) } if len(u.rules) != 2 || !u.active { t.Fatalf("adopted: rules %v, active %v", u.rules, u.active) } if state.Firewall == nil || state.Firewall.Kind != "ufw" || !state.Firewall.WasActive { t.Fatalf("adopted: firewall recorded as %+v", state.Firewall) } // Converged: the derived filter is loaded, the opening's rule goes, and only then is ufw // disabled — never reset. u.asked = nil u.ruleset = "table inet mesh\n" converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) _, state, err = applyWith(t, converged, state, u.run) if err != nil { t.Fatal(err) } if u.active || !state.Firewall.DisabledByMesh { t.Fatalf("converged: ufw still active (%v) or not recorded as retired (%+v)", u.active, state.Firewall) } if len(u.rules) != 1 || u.rules[0] != "allow 22/tcp" { t.Errorf("converged: the operator's rules were touched, or the mesh's left: %v", u.rules) } // What protected the adopted node goes last: after the derived filter applied and ufw was // retired (novox/hq ADR 0103). if del, dis := u.index("ufw delete"), u.index("ufw disable"); dis < 0 || del < dis { t.Errorf("converged: the opening was removed before ufw was retired: %v", u.asked) } for _, a := range u.asked { if strings.Contains(a, "reset") { t.Errorf("converged: ufw was reset: %s", a) } } // Converged again: nothing more to retire. u.asked = nil if _, state, err = applyWith(t, converged, state, u.run); err != nil { t.Fatal(err) } if u.index("ufw") >= 0 { t.Errorf("a converged node kept talking to a retired ufw: %v", u.asked) } // Returned to adopted: ufw is enabled before the opening is converged through it. u.asked = nil _, state, err = applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), state, u.run) if err != nil { t.Fatal(err) } if !u.active || state.Firewall.DisabledByMesh { t.Fatalf("returned: ufw active %v, record %+v", u.active, state.Firewall) } if en, add := u.index("ufw --force enable"), u.index("ufw allow"); en < 0 || add < en { t.Errorf("returned: ufw was not enabled before the opening was added: %v", u.asked) } if len(u.rules) != 2 { t.Errorf("returned: the opening was not converged again: %v", u.rules) } } func TestAConvergedNodeThatWasNeverAdoptedNeverAsksAboutAFirewall(t *testing.T) { dir := t.TempDir() u := &ufwMachine{installed: true, active: true} if _, _, err := applyWith(t, parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`), store.State{}, u.run); err != nil { t.Fatal(err) } if len(u.asked) != 0 { t.Errorf("a converged apply asked the machine about its firewall: %v", u.asked) } } func TestAnOpeningOnAConvergedNodeIsRefused(t *testing.T) { if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` + busOpening + `]}`)); err == nil { t.Error("an opening was accepted on a node the declaration does not say is adopted") } } func TestACarriedApplyOnAnAdoptedNodeLeavesItsFirewallInForce(t *testing.T) { // The bundle, re-applied by the installer or the one-shot CLI, never says a node is adopted. // That is not the controller converging it, so ufw must stay enabled (novox/hq ADR 0100). dir := t.TempDir() u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) if err != nil { t.Fatal(err) } u.asked = nil carried := parse(t, `{"declaration":1,"resources":[`+withConf(filepath.Join(dir, "bundle"))+`]}`) _, state, err = ApplyKeeping(context.Background(), archHost(t), carried, state, store.OriginCarried, u.run, nil, nil, nil) if err != nil { t.Fatal(err) } if !u.active || state.Firewall.DisabledByMesh || u.index("ufw disable") >= 0 { t.Fatalf("a carried apply retired the found firewall: active %v, record %+v, asked %v", u.active, state.Firewall, u.asked) } } func TestAFlipThatFailsKeepsTheGuardAndTheOpenings(t *testing.T) { // Converging a node removes its openings and its guard only once everything else applied and // the found firewall is retired. A flip that fails part-way keeps them, so the store is never // left unguarded behind a filter that did not load (novox/hq ADR 0103). dir := t.TempDir() guard := filepath.Join(dir, "guard.nft") guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}` u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+guardFile+","+withConf(dir)), store.State{}, u.run) if err != nil { t.Fatal(err) } // The derived filter cannot be written: its path is under a file. blocked := filepath.Join(dir, "not-a-directory") if err := os.WriteFile(blocked, []byte("x"), 0o644); err != nil { t.Fatal(err) } filter := `{"id":"nftables.config","type":"file","path":"` + filepath.Join(blocked, "nftables.conf") + `","content":"table inet mesh {}\n"}` converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`,`+filter+`]}`) u.asked = nil u.ruleset = "table inet mesh\n" // what the filter's unit loads once the file is written _, state, err = applyWith(t, converged, state, u.run) if err == nil { t.Fatal("the failing flip reported success") } if !u.active || u.index("ufw disable") >= 0 { t.Errorf("the found firewall was retired by a flip that failed: %v", u.asked) } if len(u.rules) != 2 || u.index("ufw delete") >= 0 { t.Errorf("the opening was removed by a flip that failed: %v", u.rules) } if _, statErr := os.Stat(guard); statErr != nil { t.Errorf("the guard was removed by a flip that failed: %v", statErr) } for _, id := range []string{"adoption.guard", "adoption.opening-tcp-5671-incoming"} { if _, ok := state.Find(id); !ok { t.Errorf("%s was forgotten, so the next flip would never remove it", id) } } // Fixed, the next flip completes: filter, retire, and only then the guard and the openings. if err := os.Remove(blocked); err != nil { t.Fatal(err) } u.asked = nil _, state, err = applyWith(t, converged, state, u.run) if err != nil { t.Fatal(err) } if u.active || len(u.rules) != 1 { t.Errorf("the completed flip left ufw active %v, rules %v", u.active, u.rules) } if _, statErr := os.Stat(guard); !errors.Is(statErr, os.ErrNotExist) { t.Errorf("the guard outlived the completed flip: %v", statErr) } if _, ok := state.Find("adoption.guard"); ok { t.Error("the guard is still recorded after the completed flip") } } func TestAnOpeningAFoundRuleAnswersIsReportedSatisfied(t *testing.T) { // novox/hq ADR 0103: the mesh adds nothing beside a rule ufw would take for the same one. dir := t.TempDir() u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp", "allow 5671/tcp"}} report, _, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) if err != nil { t.Fatal(err) } o := outcomeOf(report, "adoption.opening-tcp-5671-incoming") if o.Action != "unchanged" || !strings.Contains(o.Detail, "satisfied by a rule found in ufw (allow 5671/tcp)") { t.Errorf("the opening was not reported satisfied: %+v", o) } if len(u.rules) != 2 || u.index("ufw allow") >= 0 { t.Errorf("a rule was added beside the found one: %v", u.rules) } } // Defends novox/hq ADR 0103: returned to adopted, the guard is up before the derived filter's // orphans go, and stays up if removing them fails. func TestReturningToAdoptedLoadsTheGuardBeforeRemovingTheFilter(t *testing.T) { dir := t.TempDir() guard := filepath.Join(dir, "guard.nft") guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}` for _, stopFails := range []bool{false, true} { _ = os.Remove(guard) guardUpAtStop := false run := func(_ context.Context, name string, args ...string) (string, error) { if name != "systemctl" { return "", &exec.Error{Name: name, Err: exec.ErrNotFound} } switch args[0] { case "show": return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\n", nil case "stop": _, err := os.Stat(guard) guardUpAtStop = err == nil if stopFails { return "", errors.New("the filter would not stop") } } return "", nil } converged := store.State{Resources: []store.Applied{ {ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}} _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run) if !guardUpAtStop { t.Errorf("stop fails %v: the derived filter was stopped before the guard was written", stopFails) } if stopFails { if err == nil { t.Error("a failed removal was not reported") } if _, statErr := os.Stat(guard); statErr != nil { t.Error("the guard is not up after the filter's removal failed") } if _, ok := state.Find("adoption.guard"); !ok { t.Error("the guard applied before the failure was not recorded") } if _, still := state.Find("nftables.load"); !still { t.Error("the filter that would not stop was forgotten, so nothing would stop it later") } } else if err != nil { t.Fatal(err) } } } func TestAStaleOpeningOnAnAdoptedNodeIsRemovedAsAnyOrphan(t *testing.T) { // Only the flip defers the adoption's own orphans; an adopted node drops a stale opening at // once, before what replaces it is applied. dir := t.TempDir() u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) if err != nil { t.Fatal(err) } u.asked = nil other := `{"id":"adoption.opening-tcp-5000-incoming","type":"opening","port":5000,"protocol":"tcp","from":"everywhere","path":"incoming"}` if _, _, err = applyWith(t, adopted(t, `{"taken":[]}`, other+","+withConf(dir)), state, u.run); err != nil { t.Fatal(err) } if del, add := u.index("ufw delete"), u.index("ufw allow"); del < 0 || add < 0 || del > add { t.Errorf("the stale opening was not removed before the new one was added: %v", u.asked) } } func TestARetiredFirewallRetriedStillPutsBackTheForwardPolicy(t *testing.T) { // The forward policy is recorded before ufw is disabled, so a retirement that failed after // the disable restores what the machine had, not what the disable left (novox/hq ADR 0100). dir := t.TempDir() u := &ufwMachine{installed: true, active: true, forward: "DROP", failP: 1, ruleset: "table inet mesh\n"} _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)), store.State{}, u.run) if err != nil { t.Fatal(err) } converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) if _, state, err = applyWith(t, converged, state, u.run); err == nil { t.Fatal("the failed restore was not reported") } if u.active || u.forward != "ACCEPT" || state.Firewall.Forward["iptables"] != "DROP" { t.Fatalf("after the failed attempt: active %v, forward %s, recorded %+v", u.active, u.forward, state.Firewall) } if _, state, err = applyWith(t, converged, state, u.run); err != nil { t.Fatal(err) } if u.forward != "DROP" || !state.Firewall.DisabledByMesh { t.Errorf("the retry did not put the forward policy back: %s, %+v", u.forward, state.Firewall) } } func TestAGuardThatFailsLeavesTheDerivedFilterInForce(t *testing.T) { // Returning to adopted: if the guard cannot be raised, the filter it replaces must not be // stopped — its stop deletes the mesh's table, and the node would have neither (novox/hq ADR 0103). dir := t.TempDir() blocked := filepath.Join(dir, "not-a-directory") if err := os.WriteFile(blocked, []byte("x"), 0o644); err != nil { t.Fatal(err) } guardFile := `{"id":"adoption.guard","type":"file","path":"` + filepath.Join(blocked, "guard.nft") + `","content":"table inet mesh_guard {}\n"}` stopped := false run := func(_ context.Context, name string, args ...string) (string, error) { if name != "systemctl" { return "", &exec.Error{Name: name, Err: exec.ErrNotFound} } if args[0] == "show" { return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\n", nil } if args[0] == "stop" { stopped = true } return "", nil } converged := store.State{Resources: []store.Applied{ {ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}} _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run) if err == nil { t.Fatal("a guard that could not be written reported success") } if stopped { t.Error("the derived filter was stopped though the guard is not up") } if _, gone := state.Find("nftables.load"); !gone { t.Error("the filter was forgotten, so nothing would ever stop it") } } func TestUfwIsNotRetiredUntilTheMeshsOwnFilterIsLoaded(t *testing.T) { // The flip retires the found firewall because the mesh's derived filter takes its place. If // that table is not loaded, retiring would leave the machine filtering nothing (novox/hq ADR 0100). dir := t.TempDir() u := &ufwMachine{installed: true, active: true, rules: []string{"allow 22/tcp"}} _, state, err := applyWith(t, adopted(t, `{"taken":[]}`, busOpening+","+withConf(dir)), store.State{}, u.run) if err != nil { t.Fatal(err) } converged := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) _, state, err = applyWith(t, converged, state, u.run) if err == nil || !strings.Contains(err.Error(), "table inet mesh") { t.Fatalf("ufw was retired with nothing in its place: %v", err) } if !u.active || state.Firewall.DisabledByMesh { t.Errorf("ufw was disabled: active %v, %+v", u.active, state.Firewall) } // Once the table is loaded, the same declaration retires it. u.ruleset = "table inet mesh\n" if _, state, err = applyWith(t, converged, state, u.run); err != nil { t.Fatal(err) } if u.active || !state.Firewall.DisabledByMesh { t.Errorf("ufw was not retired once the mesh's filter was loaded: active %v, %+v", u.active, state.Firewall) } }