Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9d13b0593b | ||
|
|
30d548a762 | ||
|
|
550e4c6acb | ||
|
|
1587fd97f9 | ||
|
|
b5df244096 | ||
|
|
db47bb68e9 | ||
|
|
db84142d38 | ||
|
|
c9204591bf | ||
|
|
e8e707e013 | ||
|
|
0c003774ba | ||
|
|
fbc3d320ea | ||
|
|
cf495e315f | ||
|
|
24f024dd74 | ||
|
|
9acb5f1292 |
@@ -95,11 +95,22 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
|
||||
// filter is not sent to one that has not. The anchor reports one, as a real host does; this
|
||||
// fixture lacked it from 2026-09-28 and nothing ran the test (issue 177).
|
||||
Outward: []string{"eth0"},
|
||||
// And what filters it (ADR 0168): its front end, the runtime's own, and a chain a
|
||||
// predecessor left in the runtime's user chain.
|
||||
Filters: anchorFilters,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// anchorFilters is what the adopted anchor says filters it: ufw's chains, the runtime's, and a
|
||||
// predecessor's chain the mesh did not write.
|
||||
var anchorFilters = []link.Filter{
|
||||
{Where: "table ip filter, chain ufw-reject-input", Owner: "found-firewall", Refuses: "reject"},
|
||||
{Where: "table ip filter, chain DOCKER", Owner: "runtime", Refuses: `iifname != "docker0" oifname "docker0" drop`},
|
||||
{Where: "table ip filter, chain DOCKER-USER", Owner: "other", Refuses: `iifname "eth0" tcp dport 6000 drop`},
|
||||
}
|
||||
|
||||
var (
|
||||
heldContainer = link.Held{ID: "hello-web.server", Module: "hello-web", Kind: "container",
|
||||
Target: "hello-web", Since: time.Now()}
|
||||
@@ -264,6 +275,20 @@ func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T)
|
||||
if strings.Contains(preview, "15672") {
|
||||
t.Errorf("a loopback listener is in the preview:\n%s", preview)
|
||||
}
|
||||
// What filters the machine now, and the fate of each (novox/hq ADR 0168): the predecessor's
|
||||
// chain is named as not the mesh's and left, so the reader knows before the flip.
|
||||
for _, want := range []string{
|
||||
"table ip filter, chain DOCKER-USER",
|
||||
"NOT THE MESH'S; left in force",
|
||||
`iifname "eth0" tcp dport 6000 drop`,
|
||||
"table ip filter, chain ufw-reject-input",
|
||||
"the found firewall's; retired with it",
|
||||
"the container runtime's own; left",
|
||||
} {
|
||||
if !strings.Contains(preview, want) {
|
||||
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
||||
}
|
||||
}
|
||||
for _, line := range strings.Split(preview, "\n") {
|
||||
if strings.Contains(line, "5000") && !strings.Contains(line, "WILL CLOSE") {
|
||||
t.Errorf("an undeclared published port is not said to close: %s", line)
|
||||
|
||||
@@ -29,6 +29,10 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node
|
||||
if said, err := inv.AdoptionOf(ctx, node.Name); err == nil && len(said.Strays) > 0 {
|
||||
showStrays(said.Strays)
|
||||
}
|
||||
// And what filters it, truthfully (novox/hq ADR 0168): the mesh alone, or not.
|
||||
if filtering, err := inv.FilteringOf(ctx, node.Name); err == nil {
|
||||
showFiltering(filtering, false)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fmt.Printf(" mode adopted since %s\n",
|
||||
@@ -72,10 +76,63 @@ func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node
|
||||
}
|
||||
}
|
||||
showStrays(said.Strays)
|
||||
if filtering, err := inv.FilteringOf(ctx, node.Name); err == nil {
|
||||
showFiltering(filtering, true)
|
||||
}
|
||||
fmt.Printf(" as of %s\n", said.At.Local().Format(time.DateTime))
|
||||
return nil
|
||||
}
|
||||
|
||||
// showFiltering says what filters a machine, with owners (novox/hq ADR 0168), and for a converged
|
||||
// machine the state of the firewall it was found with. A machine that has not said is not said to
|
||||
// be filtered by anything.
|
||||
func showFiltering(f inventory.Filtering, adopted bool) {
|
||||
if len(f.Filters) == 0 && f.FoundFirewall == nil {
|
||||
return
|
||||
}
|
||||
if fw := f.FoundFirewall; fw != nil && !adopted {
|
||||
switch {
|
||||
case fw.Active:
|
||||
fmt.Printf(" found firewall %s is ACTIVE on this converged machine; the next apply retires it again\n", fw.Kind)
|
||||
case fw.RetiredBy == inventory.FilterMesh || fw.RetiredBy == "mesh":
|
||||
fmt.Printf(" found firewall %s, retired by the mesh; its configuration stays on disk\n", fw.Kind)
|
||||
case fw.RetiredBy != "":
|
||||
fmt.Printf(" found firewall %s, found inactive — not by the mesh\n", fw.Kind)
|
||||
default:
|
||||
fmt.Printf(" found firewall %s, inactive\n", fw.Kind)
|
||||
}
|
||||
}
|
||||
if len(f.Filters) == 0 {
|
||||
return
|
||||
}
|
||||
if f.Alone() {
|
||||
fmt.Printf(" filtered by the mesh alone (%s)\n", filterSummary(f.Filters))
|
||||
return
|
||||
}
|
||||
fmt.Printf(" filtered by NOT the mesh alone: %d rule set(s) the mesh did not write refuse traffic here\n", len(f.Others()))
|
||||
for _, x := range f.Filters {
|
||||
if x.Owner == inventory.FilterOther || x.Owner == inventory.FilterFoundFirewall {
|
||||
fmt.Printf(" %-17s %s — %s: %s\n", "", x.Where, x.Owner, x.Refuses)
|
||||
}
|
||||
}
|
||||
fmt.Printf(" %-17s and its own: %s\n", "", filterSummary(f.Filters))
|
||||
}
|
||||
|
||||
// filterSummary counts a machine's filters by owner: "mesh 2, runtime 3, ban 1".
|
||||
func filterSummary(filters []inventory.Filter) string {
|
||||
counts := map[string]int{}
|
||||
for _, x := range filters {
|
||||
counts[x.Owner]++
|
||||
}
|
||||
var parts []string
|
||||
for _, owner := range []string{inventory.FilterMesh, inventory.FilterRuntime, inventory.FilterBan, inventory.FilterFoundFirewall, inventory.FilterOther} {
|
||||
if n := counts[owner]; n > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%s %d", owner, n))
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
// showStrays says what a machine runs that the mesh neither wrote nor holds (ADR 0163).
|
||||
func showStrays(strays []inventory.Stray) {
|
||||
if len(strays) == 0 {
|
||||
@@ -661,7 +718,11 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
}
|
||||
derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
|
||||
outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks}
|
||||
preview, saw := previewOf(node, reported, derived, plan, taken, filter, runs[filter])
|
||||
filtering, err := inv.FilteringOf(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
preview, saw := previewOf(node, reported, filtering, derived, plan, taken, filter, runs[filter])
|
||||
preview += "\n\n preview " + saw
|
||||
if !yes {
|
||||
return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes %s` to do "+
|
||||
@@ -731,7 +792,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
// previewOf is what converging a node will change, before it changes it, and a short digest of
|
||||
// what it said: every reachable thing and its fate, the modules the flip takes and the filter. The
|
||||
// digest is what the flip is asked to act on, so it changes whenever any of those would.
|
||||
func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
|
||||
func previewOf(node string, reported inventory.Adoption, filtering inventory.Filtering, derived derivedFilter,
|
||||
plan catalogue.Resolution, taken []string, filter string, filterAssigned bool) (string, string) {
|
||||
var said []string
|
||||
var b strings.Builder
|
||||
@@ -823,6 +884,30 @@ func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
|
||||
fmt.Fprintf(&b, " the found firewall (%s) is disabled, never flushed: its configuration stays on disk\n", fw)
|
||||
}
|
||||
said = append(said, fmt.Sprintf("filter %s assigned=%t firewall=%s", filter, filterAssigned, fw))
|
||||
// What filters the machine now, and the fate of each (novox/hq ADR 0168): the found firewall
|
||||
// retired, the runtime's own and bans left, and what the mesh did not write left and named —
|
||||
// so the reader knows before the flip that the machine will not be filtered by the mesh alone.
|
||||
if len(filtering.Filters) > 0 {
|
||||
b.WriteString("\n what filters the machine now, and what the flip does to each:\n")
|
||||
for _, x := range filtering.Filters {
|
||||
fate := "left: " + x.Owner + "'s"
|
||||
switch x.Owner {
|
||||
case inventory.FilterMesh:
|
||||
fate = "the mesh's guard; replaced by its filter"
|
||||
case inventory.FilterFoundFirewall:
|
||||
fate = "the found firewall's; retired with it"
|
||||
case inventory.FilterRuntime:
|
||||
fate = "the container runtime's own; left"
|
||||
case inventory.FilterBan:
|
||||
fate = "a ban list; left"
|
||||
case inventory.FilterOther:
|
||||
fate = "NOT THE MESH'S; left in force — the machine is not filtered by the mesh alone until you remove it"
|
||||
}
|
||||
fmt.Fprintf(&b, " %-50s %s\n", x.Where, fate)
|
||||
fmt.Fprintf(&b, " %-50s %s\n", "", x.Refuses)
|
||||
said = append(said, "filter "+x.Owner+" "+x.Where)
|
||||
}
|
||||
}
|
||||
// Sorted: the same account, reported in another order, is the same preview.
|
||||
sort.Strings(said)
|
||||
sum := sha256.Sum256([]byte(strings.Join(said, "\n")))
|
||||
|
||||
@@ -607,6 +607,10 @@ type answers struct {
|
||||
// a consequence of the refusals above: a node that does not resolve is not on the network, and
|
||||
// a mesh whose hub is that node has no hub.
|
||||
network string
|
||||
// filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
||||
// 0168): what filters it beyond the mesh's own, the runtime's plumbing and bans, by name — a
|
||||
// predecessor's chain, a found firewall in force again. Such a machine is not "all well".
|
||||
filtered map[string]inventory.Filtering
|
||||
// untaken is, per machine, each assigned module whose resources the machine is holding as it
|
||||
// found them, and how many — a module that was assigned, sent, and is running none of what it
|
||||
// declares because nothing has taken it (novox/hq ADR 0100, 04-ISSUES/125).
|
||||
|
||||
@@ -352,10 +352,14 @@ func settingsCommand(ctx context.Context, args []string) error {
|
||||
switch args[0] {
|
||||
case "set":
|
||||
if len(positionals) != 2 {
|
||||
return errors.New("settings set <module> <settings.json> [--node <node>]")
|
||||
return errors.New("settings set <module> <settings.json | {…}> [--node <node>]")
|
||||
}
|
||||
raw, err := os.ReadFile(positionals[1])
|
||||
if err != nil {
|
||||
// A file, or the values themselves when they begin with `{` — which is how the mesh's own
|
||||
// `settings` tool passes them, having no file to hand over (novox/hq issue 198).
|
||||
var raw []byte
|
||||
if strings.HasPrefix(strings.TrimSpace(positionals[1]), "{") {
|
||||
raw = []byte(positionals[1])
|
||||
} else if raw, err = os.ReadFile(positionals[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
var values map[string]any
|
||||
|
||||
@@ -397,6 +397,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
return sendable{}, err
|
||||
}
|
||||
return sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}, nil
|
||||
}
|
||||
|
||||
|
||||
+23
-10
@@ -393,11 +393,7 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
fmt.Printf("\n%d node(s) told\n", len(sending))
|
||||
// And each machine's memberships, as every other send does (ADR 0160): a push is the one most
|
||||
// operators run, and on 2026-10-01 it was the one path that issued none.
|
||||
var told []string
|
||||
for _, s := range sending {
|
||||
told = append(told, s.node)
|
||||
}
|
||||
if err := issueMemberships(ctx, open, server, told); err != nil {
|
||||
if err := issueMemberships(ctx, open, server, sending); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -715,11 +711,15 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
// And every assignment on those machines its membership (novox/hq ADR 0160): composed from the
|
||||
// same records the bus's accounts are, so what a runtime serves and what its account may are one
|
||||
// composition. Issued after the declaration, because the runtime it is for arrives with it.
|
||||
return issueMemberships(ctx, open, server, names)
|
||||
return issueMemberships(ctx, open, server, sending)
|
||||
}
|
||||
|
||||
// issueMemberships publishes the membership of every module on the named machines.
|
||||
func issueMemberships(ctx context.Context, open *stores, server *link.Server, names []string) error {
|
||||
// issueMemberships publishes the membership of every module on the machines just sent.
|
||||
//
|
||||
// Each carries what its module receives and the private network's addresses, from the same
|
||||
// composition as the declaration it was sent (novox/hq ADR 0167): a provider reads what it is
|
||||
// given on the bus, and the file written beside it says the same thing.
|
||||
func issueMemberships(ctx context.Context, open *stores, server *link.Server, sent []readyNode) error {
|
||||
records, err := open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -734,9 +734,22 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, na
|
||||
// the push stands, the first failure is named once, and the next push tries again.
|
||||
issued, failed := 0, 0
|
||||
var first error
|
||||
for _, node := range names {
|
||||
for _, s := range sent {
|
||||
node := s.node
|
||||
for _, d := range records.Assigned[node] {
|
||||
body, err := json.Marshal(broker.MembershipFor(node, d, where))
|
||||
membership := broker.MembershipFor(node, d, where)
|
||||
membership.Mesh = s.declared.Mesh
|
||||
for requirement, given := range s.declared.Received[d.Module] {
|
||||
raw, err := json.Marshal(given)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if membership.Receives == nil {
|
||||
membership.Receives = map[string]json.RawMessage{}
|
||||
}
|
||||
membership.Receives[requirement] = raw
|
||||
}
|
||||
body, err := json.Marshal(membership)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
@@ -66,6 +67,21 @@ type meshStatus struct {
|
||||
// **A document without this said an outage was a well mesh.** Read from what each machine
|
||||
// reported, so it is the machine's account and not the mesh's take-time listing.
|
||||
Untaken []machineUntaken `json:"untaken,omitempty"`
|
||||
// Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
||||
// 0168), one entry per rule set the mesh did not write — the found firewall in force again,
|
||||
// or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh
|
||||
// alone. A document without this called a machine well while a predecessor's chain refused
|
||||
// what the mesh declared open.
|
||||
Filtered []machineFiltered `json:"filtered,omitempty"`
|
||||
}
|
||||
|
||||
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||
// refuses traffic: where it is, whose the host reads it as, and what it refuses.
|
||||
type machineFiltered struct {
|
||||
Node string `json:"node"`
|
||||
Where string `json:"where"`
|
||||
Owner string `json:"owner"`
|
||||
Refuses string `json:"refuses"`
|
||||
}
|
||||
|
||||
// machineUntaken is one module a machine is holding rather than running, and how many resources of
|
||||
@@ -173,6 +189,21 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
machineUntaken{Node: name, Module: m, Held: asked.untaken[name][m]})
|
||||
}
|
||||
}
|
||||
filteredNodes := make([]string, 0, len(asked.filtered))
|
||||
for name := range asked.filtered {
|
||||
filteredNodes = append(filteredNodes, name)
|
||||
}
|
||||
sort.Strings(filteredNodes)
|
||||
for _, name := range filteredNodes {
|
||||
f := asked.filtered[name]
|
||||
if fw := f.FoundFirewall; fw != nil && fw.Active {
|
||||
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: "the found firewall",
|
||||
Owner: inventory.FilterFoundFirewall, Refuses: fw.Kind + " is in force again"})
|
||||
}
|
||||
for _, x := range f.Others() {
|
||||
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: x.Where, Owner: x.Owner, Refuses: x.Refuses})
|
||||
}
|
||||
}
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
|
||||
@@ -167,3 +167,46 @@ func TestAMachineFailingTheSameWayIsSaidToBeStuck(t *testing.T) {
|
||||
t.Fatalf("one failure is not stuck: %v", once)
|
||||
}
|
||||
}
|
||||
|
||||
// A converged machine something other than the mesh filters is named, per rule set, and is not
|
||||
// well (novox/hq ADR 0168); one filtered by the mesh alone is not in the list.
|
||||
func TestAMachineNotFilteredByTheMeshAloneIsNamedAndNotWell(t *testing.T) {
|
||||
alone := inventory.Filtering{Filters: []inventory.Filter{
|
||||
{Where: "table inet mesh, chain input", Owner: inventory.FilterMesh, Refuses: "policy drop"},
|
||||
{Where: "table ip filter, chain DOCKER", Owner: inventory.FilterRuntime, Refuses: "drop"},
|
||||
{Where: "table ip filter, chain f2b-sshd", Owner: inventory.FilterBan, Refuses: "ip saddr 192.0.2.1 reject"},
|
||||
}}
|
||||
if !alone.Alone() {
|
||||
t.Fatal("the mesh's own, the runtime's and a ban are not the mesh alone")
|
||||
}
|
||||
notAlone := inventory.Filtering{
|
||||
Filters: append(alone.Filters, inventory.Filter{Where: "chain HAL-MESH-ONLY (iptables-legacy)",
|
||||
Owner: inventory.FilterOther, Refuses: `-A HAL-MESH-ONLY -m comment --comment "not public" -j DROP`}),
|
||||
FoundFirewall: &inventory.FoundFirewall{Kind: "ufw", Active: true},
|
||||
}
|
||||
asked := answers{nodes: []inventory.Node{{Name: "home-server"}, {Name: "laptop"}},
|
||||
filtered: map[string]inventory.Filtering{"home-server": notAlone}}
|
||||
if asked.well() {
|
||||
t.Fatal("a machine not filtered by the mesh alone reads as well")
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var parsed struct {
|
||||
Filtered []map[string]string `json:"filtered"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(parsed.Filtered) != 2 {
|
||||
t.Fatalf("filtered: %v", parsed.Filtered)
|
||||
}
|
||||
if parsed.Filtered[0]["node"] != "home-server" || parsed.Filtered[0]["owner"] != inventory.FilterFoundFirewall ||
|
||||
parsed.Filtered[1]["where"] != "chain HAL-MESH-ONLY (iptables-legacy)" || parsed.Filtered[1]["owner"] != inventory.FilterOther {
|
||||
t.Fatalf("filtered: %v", parsed.Filtered)
|
||||
}
|
||||
if body, _ := statusAsJSON(answers{nodes: asked.nodes}); strings.Contains(string(body), `"filtered"`) {
|
||||
t.Fatal("a mesh filtered by itself alone carries a filtered list")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,6 +129,25 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
// Half of either shape: the command says its usage, which names both shapes, and that is
|
||||
// the answer the caller needs.
|
||||
return []string{"rotate"}, nil
|
||||
case "settings":
|
||||
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
|
||||
// because a tool has no file to hand the command; the command reads either.
|
||||
if err := need("module"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"settings", "set", str("module")}
|
||||
switch {
|
||||
case str("clear") == "true":
|
||||
argv = []string{"settings", "clear", str("module")}
|
||||
case str("values") != "":
|
||||
argv = append(argv, str("values"))
|
||||
}
|
||||
// Neither values nor clear: the command says its usage, which names both, and that is the
|
||||
// answer the caller needs — the same as `rotate` given half of either shape.
|
||||
if n := str("node"); n != "" {
|
||||
argv = append(argv, "--node", n)
|
||||
}
|
||||
return argv, nil
|
||||
case "issue":
|
||||
// The same act as `module issue` at a shell (novox/hq design 25 §4): the account is minted
|
||||
// into the mesh's records and delivered at the machine's next push, which is the caller's to
|
||||
|
||||
@@ -73,6 +73,22 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// `settings` is `settings set|clear` at a shell, with the values passed inline (novox/hq issue 198).
|
||||
func TestSettingsSetsOrClearsALayer(t *testing.T) {
|
||||
argv, err := argvFor("settings", map[string]any{"module": "dnsmasq", "values": `{"a":1}`, "node": "ace"})
|
||||
if err != nil || strings.Join(argv, " ") != `settings set dnsmasq {"a":1} --node ace` {
|
||||
t.Fatalf("set on a machine: %v %v", argv, err)
|
||||
}
|
||||
argv, _ = argvFor("settings", map[string]any{"module": "dnsmasq", "clear": "true"})
|
||||
if strings.Join(argv, " ") != "settings clear dnsmasq" {
|
||||
t.Fatalf("clear for the mesh: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("settings", map[string]any{"module": "dnsmasq"})
|
||||
if strings.Join(argv, " ") != "settings set dnsmasq" {
|
||||
t.Fatalf("a set with no values falls to the command's usage: %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
// `issue` is `module issue` at a shell: the module and the machine, and nothing that would push. A
|
||||
// module's bus account was mintable only from the controller's command line, so an agent working
|
||||
// through the tools could not finish a rollout that gave a module one (novox/hq issue 191).
|
||||
|
||||
@@ -25,6 +25,12 @@ type sendable struct {
|
||||
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
||||
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
||||
Adoption *adoptionEnvelope
|
||||
|
||||
// Received and Mesh are not sent in the declaration. They are what this machine's memberships
|
||||
// are issued with on the bus (novox/hq ADR 0167): each module's received contributions, from
|
||||
// the same composition as its received files, and every machine's private-network address.
|
||||
Received map[string]map[string][]catalogue.Contribution
|
||||
Mesh []string
|
||||
// LeftOut is every module of the machine's set left out of this declaration because a stored
|
||||
// setting cannot compose with its definition (novox/hq ADR 0163, rule 6), sorted. The host
|
||||
// keeps that module's held things and touches none of its containers; a machine is told
|
||||
|
||||
@@ -227,6 +227,28 @@ func printStatus(asked answers) error {
|
||||
" not readable from a commit; that needs a version the host reports as ordered\n\n")
|
||||
}
|
||||
|
||||
if len(asked.filtered) > 0 {
|
||||
// A converged machine is filtered by the mesh alone, and the mesh says truthfully which
|
||||
// (novox/hq ADR 0168). One that is not — a predecessor's chain still refusing, a found
|
||||
// firewall in force again — is said here, and is not well.
|
||||
machines := make([]string, 0, len(asked.filtered))
|
||||
for name := range asked.filtered {
|
||||
machines = append(machines, name)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
fmt.Printf("%d converged machine(s) are not filtered by the mesh alone:\n", len(machines))
|
||||
for _, name := range machines {
|
||||
f := asked.filtered[name]
|
||||
if fw := f.FoundFirewall; fw != nil && fw.Active {
|
||||
fmt.Printf(" %-12s the found firewall (%s) is in force again; the next apply retires it\n", name, fw.Kind)
|
||||
}
|
||||
for _, x := range f.Others() {
|
||||
fmt.Printf(" %-12s %s (%s): %s\n", name, x.Where, x.Owner, x.Refuses)
|
||||
}
|
||||
}
|
||||
fmt.Printf("\n the mesh wrote none of these and removes none; `node show <node>` lists every filter with its owner\n\n")
|
||||
}
|
||||
|
||||
if len(asked.untaken) > 0 {
|
||||
// **Before the adopted line, and it breaks "all well".** An adopted machine is a state
|
||||
// somebody chose and can leave alone; a module assigned to one and never taken is work
|
||||
@@ -357,6 +379,13 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
// And which converged machines something other than the mesh filters (novox/hq ADR 0168), as
|
||||
// each last reported — the account that was missing when a predecessor's chain refused what the
|
||||
// mesh declared open for eleven hours (04-ISSUES/144, 145).
|
||||
out.filtered, err = filteredMachines(ctx, inv, out.nodes)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
@@ -406,6 +435,30 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
//
|
||||
// A machine that reports no holds contributes nothing, so a converged mesh answers an empty map and
|
||||
// the caller prints nothing.
|
||||
// filteredMachines is every converged machine not filtered by the mesh alone, with what it last said
|
||||
// filters it (novox/hq ADR 0168). An adopted machine keeps its found firewall by design and is not
|
||||
// counted; a machine that has not said is not said to be filtered by anything.
|
||||
func filteredMachines(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) (
|
||||
map[string]inventory.Filtering, error) {
|
||||
out := map[string]inventory.Filtering{}
|
||||
for _, n := range nodes {
|
||||
if n.Adopted {
|
||||
continue
|
||||
}
|
||||
f, err := inv.FilteringOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what filters %s cannot be read: %w", n.Name, err)
|
||||
}
|
||||
if len(f.Filters) == 0 && f.FoundFirewall == nil {
|
||||
continue
|
||||
}
|
||||
if !f.Alone() {
|
||||
out[n.Name] = f
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) (
|
||||
map[string]map[string]int, error) {
|
||||
|
||||
@@ -442,7 +495,8 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
|
||||
// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
|
||||
func (a answers) well() bool {
|
||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||
len(a.filtered) == 0
|
||||
}
|
||||
|
||||
// hostSplit is which machines report which host version, for every version more than one machine
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// What the mesh issued this proxy, read on the bus (novox/hq ADR 0160, ADR 0167).
|
||||
//
|
||||
// **The proxy is told, not left to work it out.** Its membership carries the routes it is given —
|
||||
// the same contributions its file is written from — and every machine's address on the private
|
||||
// network, which is who may be served an internal name. Read once at connect and followed, so a
|
||||
// route added or a machine joining reaches a running proxy without a restart.
|
||||
|
||||
// credential is the bus account the mesh delivered as this module's own secret named broker.
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
// followMembership connects with the credential in path and applies every membership the mesh
|
||||
// issues this proxy. It retries the first connection for as long as it takes: a proxy that started
|
||||
// before the bus keeps serving the file, and takes the bus when it answers.
|
||||
func followMembership(path string, held *table, fromBus *atomic.Bool) {
|
||||
for {
|
||||
err := followOnce(path, held, fromBus)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
log.Printf("cannot follow this proxy's membership, serving the file meanwhile: %v", err)
|
||||
time.Sleep(30 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func followOnce(path string, held *table, fromBus *atomic.Bool) error {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var cred credential
|
||||
if err := json.Unmarshal(raw, &cred); err != nil {
|
||||
return fmt.Errorf("the broker credential is not one: %w", err)
|
||||
}
|
||||
if cred.Node == "" || cred.Module == "" {
|
||||
return fmt.Errorf("the broker credential names no node or module, so it has no membership")
|
||||
}
|
||||
|
||||
opts := []nats.Option{
|
||||
nats.Name(cred.Node + "." + cred.Module),
|
||||
nats.UserInfo(cred.User, cred.Password),
|
||||
// Its own inbox, and nothing wider: every principal is granted `_INBOX.<its user>.>` alone.
|
||||
nats.CustomInboxPrefix("_INBOX." + cred.User),
|
||||
// The bus being restarted is an upgrade, not a reason to stop following.
|
||||
nats.MaxReconnects(-1),
|
||||
}
|
||||
if strings.TrimSpace(cred.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(broker.PinnedToFingerprint(cred.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(cred.URL, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connecting to the bus at %s: %w", broker.BareAddress(cred.URL), err)
|
||||
}
|
||||
|
||||
subject := broker.MembershipSubject(cred.Node, cred.Module)
|
||||
apply := func(body []byte) {
|
||||
var issued broker.Membership
|
||||
if err := json.Unmarshal(body, &issued); err != nil {
|
||||
log.Printf("a membership arrived that is not one: %v", err)
|
||||
return
|
||||
}
|
||||
if took := applyMembership(issued, held); took && !fromBus.Swap(true) {
|
||||
log.Printf("routes now come from this proxy's membership on %s", subject)
|
||||
}
|
||||
}
|
||||
|
||||
// Followed first, read second: an issue landing between the two is applied, not missed.
|
||||
if _, err := conn.Subscribe(subject, func(m *nats.Msg) { apply(m.Data) }); err != nil {
|
||||
conn.Close()
|
||||
return fmt.Errorf("cannot follow %s: %w", subject, err)
|
||||
}
|
||||
// The subject-addressed direct get: the one request this account may make of the stream.
|
||||
got, err := conn.Request("$JS.API.DIRECT.GET."+broker.AssignmentsStream+"."+subject, nil, 5*time.Second)
|
||||
switch {
|
||||
case err != nil:
|
||||
log.Printf("cannot read the membership issued on %s yet (%v); following it", subject, err)
|
||||
case got.Header.Get("Status") != "" || len(got.Data) == 0:
|
||||
log.Printf("no membership issued on %s yet; serving the file until one is", subject)
|
||||
default:
|
||||
apply(got.Data)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyMembership serves what a membership says, and says whether it said anything about routes.
|
||||
//
|
||||
// A membership with no routes in it is one from a controller older than ADR 0167, and the file stays
|
||||
// the source rather than every route being withdrawn because a field was absent.
|
||||
func applyMembership(issued broker.Membership, held *table) bool {
|
||||
raw, carries := issued.Receives["route"]
|
||||
if !carries {
|
||||
return false
|
||||
}
|
||||
var contributions []contribution
|
||||
if err := json.Unmarshal(raw, &contributions); err != nil {
|
||||
log.Printf("the routes in this proxy's membership are not contributions, keeping what is served: %v", err)
|
||||
return false
|
||||
}
|
||||
inside, err := sourcesOf(issued.Mesh)
|
||||
if err != nil {
|
||||
log.Printf("the mesh in this proxy's membership is unreadable, keeping what is served: %v", err)
|
||||
return false
|
||||
}
|
||||
routes, public := routesOf(contributions)
|
||||
held.set(routes, public)
|
||||
held.setInside(inside)
|
||||
log.Printf("serving %d route(s) from the membership, internal names to %d machine(s): %s",
|
||||
len(routes), len(inside), strings.Join(held.names(), ", "))
|
||||
return true
|
||||
}
|
||||
+175
-29
@@ -54,12 +54,14 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/acme"
|
||||
@@ -196,6 +198,63 @@ type table struct {
|
||||
// pass ACME's own validation (it has no public DNS to prove it against), so asking for it is
|
||||
// not merely pointless but the failing order onlyWhatTheMeshSaid exists to prevent.
|
||||
public map[string]bool
|
||||
// inside is where a request must come from to be served a name that is only internal: every
|
||||
// machine's address on the private network, as the mesh issued it in this proxy's membership
|
||||
// (novox/hq ADR 0167). Empty until it is issued, and then only the machine itself is inside.
|
||||
inside sources
|
||||
}
|
||||
|
||||
// sources is who may be served an internal name: the private network's addresses as the mesh
|
||||
// issued them. The machine itself is always inside — anything on a machine may call anything on it
|
||||
// (novox/hq ADR 0144) — so loopback needs no entry.
|
||||
type sources []netip.Prefix
|
||||
|
||||
// sourcesOf reads the addresses the mesh issued, each a single address or a range. One that does
|
||||
// not parse is an error, not an entry skipped: the proxy would otherwise serve internal names to
|
||||
// fewer machines than the mesh said, and say nothing.
|
||||
func sourcesOf(mesh []string) (sources, error) {
|
||||
var out sources
|
||||
for _, entry := range mesh {
|
||||
entry = strings.TrimSpace(entry)
|
||||
if prefix, err := netip.ParsePrefix(entry); err == nil {
|
||||
out = append(out, prefix.Masked())
|
||||
continue
|
||||
}
|
||||
addr, err := netip.ParseAddr(entry)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%q is not an address on the private network", entry)
|
||||
}
|
||||
addr = addr.Unmap()
|
||||
out = append(out, netip.PrefixFrom(addr, addr.BitLen()))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// holds says whether a request from this remote address came from the mesh or the machine itself.
|
||||
//
|
||||
// **By source, which the mesh's guard deliberately is not** — it names interfaces, because a source
|
||||
// address can be claimed by whoever sends the packet. The proxy cannot see the interface a request
|
||||
// arrived on, and here the claim does not carry: a connection needs its replies, and replies to a
|
||||
// mesh address leave by the tunnel, never back to the claimant.
|
||||
func (s sources) holds(remote string) bool {
|
||||
host := remote
|
||||
if h, _, err := net.SplitHostPort(remote); err == nil {
|
||||
host = h
|
||||
}
|
||||
addr, err := netip.ParseAddr(host)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
addr = addr.Unmap()
|
||||
if addr.IsLoopback() {
|
||||
return true
|
||||
}
|
||||
for _, prefix := range s {
|
||||
if prefix.Contains(addr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *table) set(routes map[string][]rule, public map[string]bool) {
|
||||
@@ -314,6 +373,41 @@ func bareHost(host string) string {
|
||||
return strings.ToLower(host)
|
||||
}
|
||||
|
||||
// hiddenFrom says whether this host must look unrouted to a request from this address: it is
|
||||
// only an internal name, and the request did not come from the private network.
|
||||
//
|
||||
// **The proxy is the only way in to a routed endpoint, so it is what makes `internal` true**
|
||||
// (novox/hq ADR 0138, issue 191). It answers public names on the same listeners, so a request from
|
||||
// anywhere can carry any Host header; a name being internal keeps nobody out unless this check does.
|
||||
// Answered exactly as a name that was never routed, so an outsider learns nothing from asking.
|
||||
func (t *table) hiddenFrom(host, remote string) bool {
|
||||
if !t.eligibleForInternalACME(host) {
|
||||
return false
|
||||
}
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
return !t.inside.holds(remote)
|
||||
}
|
||||
|
||||
// setInside replaces who the mesh is, as the membership said.
|
||||
func (t *table) setInside(inside sources) {
|
||||
t.mu.Lock()
|
||||
t.inside = inside
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
// namesSeenFrom is what this proxy says it serves to a request from this address — every routed
|
||||
// name, less the internal-only ones when the request came from outside.
|
||||
func (t *table) namesSeenFrom(remote string) []string {
|
||||
out := []string{}
|
||||
for _, name := range t.names() {
|
||||
if !t.hiddenFrom(name, remote) {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (t *table) names() []string {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
@@ -343,7 +437,20 @@ func run() error {
|
||||
}
|
||||
|
||||
held := newTable()
|
||||
// **The bus first, the file until it has spoken** (novox/hq ADR 0167). The membership carries
|
||||
// the routes and who the mesh is; the file carries the routes alone, so while the proxy reads
|
||||
// it an internal name is served to this machine and to nobody else — refused, never opened.
|
||||
fromBus := &atomic.Bool{}
|
||||
if credential := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); credential != "" {
|
||||
go followMembership(credential, held, fromBus)
|
||||
} else {
|
||||
log.Printf("MESH_BROKER_FILE is not set: routes come from %s alone, and a name that is only "+
|
||||
"internal is served to this machine alone", path)
|
||||
}
|
||||
read := func() {
|
||||
if fromBus.Load() {
|
||||
return
|
||||
}
|
||||
routes, public, err := routesFrom(path)
|
||||
if err != nil {
|
||||
// Kept serving what it had. A file being rewritten is momentarily unreadable, and
|
||||
@@ -422,19 +529,7 @@ func run() error {
|
||||
}()
|
||||
|
||||
tlsConfig := publicManager.TLSConfig()
|
||||
if internalManager != nil {
|
||||
// Dispatched by which authority may certify this name at all — the same question
|
||||
// eligibleForInternalACME already answers, asked once more at handshake time rather than
|
||||
// only when an order is placed, since a cached certificate is served here on every request
|
||||
// and never goes through HostPolicy again.
|
||||
fromPublic, fromInternal := tlsConfig.GetCertificate, internalManager.TLSConfig().GetCertificate
|
||||
tlsConfig.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
if held.eligibleForInternalACME(hello.ServerName) {
|
||||
return fromInternal(hello)
|
||||
}
|
||||
return fromPublic(hello)
|
||||
}
|
||||
}
|
||||
tlsConfig.GetCertificate = certificateFor(held, tlsConfig.GetCertificate, internalManager)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: secure,
|
||||
@@ -592,6 +687,33 @@ func forThisAuthority(cache, directory string, root []byte) string {
|
||||
return filepath.Join(cache, hex.EncodeToString(sum[:])[:16])
|
||||
}
|
||||
|
||||
// certificateFor picks the certificate a handshake is answered with.
|
||||
//
|
||||
// Dispatched by which authority may certify this name at all — the same question
|
||||
// eligibleForInternalACME already answers, asked once more at handshake time rather than only when
|
||||
// an order is placed, since a cached certificate is served here on every request and never goes
|
||||
// through HostPolicy again. And refused, exactly as an unrouted name is, to a client outside the
|
||||
// private network asking for a name that is only internal: the certificate would name it.
|
||||
func certificateFor(held *table, fromPublic func(*tls.ClientHelloInfo) (*tls.Certificate, error),
|
||||
internalManager *autocert.Manager) func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
var fromInternal func(*tls.ClientHelloInfo) (*tls.Certificate, error)
|
||||
if internalManager != nil {
|
||||
fromInternal = internalManager.TLSConfig().GetCertificate
|
||||
}
|
||||
return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
if held.eligibleForInternalACME(hello.ServerName) {
|
||||
if hello.Conn != nil && held.hiddenFrom(hello.ServerName, hello.Conn.RemoteAddr().String()) {
|
||||
return nil, fmt.Errorf("no public route for %q in this mesh, so no certificate is asked for",
|
||||
hello.ServerName)
|
||||
}
|
||||
if fromInternal != nil {
|
||||
return fromInternal(hello)
|
||||
}
|
||||
}
|
||||
return fromPublic(hello)
|
||||
}
|
||||
}
|
||||
|
||||
// newTable is an empty routing table.
|
||||
func newTable() *table {
|
||||
return &table{to: map[string][]rule{}}
|
||||
@@ -600,8 +722,9 @@ func newTable() *table {
|
||||
// handler is the proxy itself, separated so it can be driven by a test without a listener.
|
||||
func handler(held *table) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hidden := held.hiddenFrom(r.Host, r.RemoteAddr)
|
||||
matched, known := held.find(r.Host, r.URL.Path)
|
||||
if !known {
|
||||
if hidden || !known {
|
||||
// **Named, not a bare 404.** A route that was withdrawn and a name that never existed
|
||||
// are different things, and a proxy that says only "not found" makes an operator go
|
||||
// and read the mesh to tell them apart. What it is serving is the answer to both.
|
||||
@@ -611,13 +734,13 @@ func handler(held *table) http.Handler {
|
||||
// contradiction an operator would have to disbelieve the proxy to get past.
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
if held.routed(r.Host) {
|
||||
if !hidden && held.routed(r.Host) {
|
||||
fmt.Fprintf(w, "%s is served here, but no route covers %q.\n",
|
||||
bareHost(r.Host), r.URL.Path)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, "no route for %q in this mesh.\nserving: %s\n",
|
||||
r.Host, strings.Join(held.names(), ", "))
|
||||
r.Host, strings.Join(held.namesSeenFrom(r.RemoteAddr), ", "))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -716,6 +839,12 @@ func boolByte(b bool) byte {
|
||||
// routesFrom reads what the mesh wrote and turns it into host → the rules for that host, and
|
||||
// which of those hosts is a public name — the second is `name`, ACME-eligible; a host reached
|
||||
// only through `internal-name` never appears there.
|
||||
//
|
||||
// **A route may carry either name, or both** (novox/hq ADR 0138). How far an endpoint reaches
|
||||
// decides which names the mesh composes, so an endpoint that reaches only the private network
|
||||
// arrives with an `internal-name` and no `name`. That is a whole route, not a malformed one: it is
|
||||
// served under its internal name and certified by the internal authority. Only a route with
|
||||
// neither name has nothing to be served under (novox/hq issue 191).
|
||||
func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
@@ -725,17 +854,29 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
|
||||
if err := json.Unmarshal(raw, &said); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
routes, public := routesOf(said.Given)
|
||||
return routes, public, nil
|
||||
}
|
||||
|
||||
// routesOf turns what the mesh gave into host → the rules for that host, and which hosts are public
|
||||
// names — the same whether the contributions came in the file or in the membership.
|
||||
func routesOf(contributions []contribution) (map[string][]rule, map[string]bool) {
|
||||
out := map[string][]rule{}
|
||||
public := map[string]bool{}
|
||||
for _, c := range said.Given {
|
||||
for _, c := range contributions {
|
||||
name, _ := c.Values["name"].(string)
|
||||
if name == "" {
|
||||
name = strings.TrimSpace(name)
|
||||
internal, _ := c.Values["internal-name"].(string)
|
||||
internal = strings.TrimSpace(internal)
|
||||
if name == "" && internal == "" {
|
||||
log.Printf("%s on %s asked for a route and named nothing; skipped", c.From, c.Node)
|
||||
continue
|
||||
}
|
||||
host := strings.ToLower(name)
|
||||
public[host] = true
|
||||
// What the route is called in a log line: its public name when it has one.
|
||||
called := name
|
||||
if called == "" {
|
||||
called = internal
|
||||
}
|
||||
|
||||
made := rule{path: asPath(c.Values["path"])}
|
||||
if p, ok := asWhole(c.Values["priority"]); ok {
|
||||
@@ -752,7 +893,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
|
||||
if looksLikeACredential(named) {
|
||||
log.Printf("%s on %s declared route %q with a credential in the declaration rather "+
|
||||
"than the name of a secret; the whole route is refused (novox/hq ADR 0108)",
|
||||
c.From, c.Node, name)
|
||||
c.From, c.Node, called)
|
||||
continue
|
||||
}
|
||||
users, err := usersFrom(named)
|
||||
@@ -770,7 +911,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
|
||||
port, ok := asPort(c.Values["port"])
|
||||
if !ok {
|
||||
log.Printf("%s on %s asked for route %q and gave no usable port; skipped",
|
||||
c.From, c.Node, name)
|
||||
c.From, c.Node, called)
|
||||
continue
|
||||
}
|
||||
// Where the mesh says that machine is. Empty means it is this one — a workload beside
|
||||
@@ -791,7 +932,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
|
||||
}
|
||||
if scheme != "http" && scheme != "https" {
|
||||
log.Printf("%s on %s asked for route %q with scheme %q, which is neither http "+
|
||||
"nor https; skipped", c.From, c.Node, name, scheme)
|
||||
"nor https; skipped", c.From, c.Node, called, scheme)
|
||||
continue
|
||||
}
|
||||
made.insecure, _ = c.Values["insecure"].(bool)
|
||||
@@ -802,7 +943,7 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
|
||||
bytes, whole := asWhole(asked)
|
||||
if !whole || bytes <= 0 {
|
||||
log.Printf("%s on %s asked for route %q with a max-request-body of %v, which is "+
|
||||
"not a whole positive number of bytes; skipped", c.From, c.Node, name, asked)
|
||||
"not a whole positive number of bytes; skipped", c.From, c.Node, called, asked)
|
||||
continue
|
||||
}
|
||||
made.maxRequestBody = int64(bytes)
|
||||
@@ -810,19 +951,24 @@ func routesFrom(path string) (map[string][]rule, map[string]bool, error) {
|
||||
made.target = fmt.Sprintf("%s://%s:%d", scheme, at, port)
|
||||
}
|
||||
|
||||
out[host] = append(out[host], made)
|
||||
if name != "" {
|
||||
host := strings.ToLower(name)
|
||||
out[host] = append(out[host], made)
|
||||
public[host] = true
|
||||
}
|
||||
|
||||
// The internal-network alias, the same rule under a second host — a predecessor proxy
|
||||
// The internal-network name, the same rule under a second host — a predecessor proxy
|
||||
// answered both for one route, as a convenience (reaching a service over the VPN without a
|
||||
// public TLS round trip), not as an access boundary; composing it here restores exactly
|
||||
// that, nothing more. Absent whenever the node composed no internal name (novox/hq ADR
|
||||
// 0056's internalDomain half) — the same "nothing to join a label to" case the public name
|
||||
// already has.
|
||||
if internal, _ := c.Values["internal-name"].(string); strings.TrimSpace(internal) != "" {
|
||||
// already has. And the only name, when the endpoint reaches no further than the private
|
||||
// network.
|
||||
if internal != "" {
|
||||
out[strings.ToLower(internal)] = append(out[strings.ToLower(internal)], made)
|
||||
}
|
||||
}
|
||||
return out, public, nil
|
||||
return out, public
|
||||
}
|
||||
|
||||
// asWhole is any whole number the mesh wrote, whatever its magnitude.
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// behind is a workload the proxy can send to, and a table routing one public name and one
|
||||
// internal-only name to it, with the mesh's machines as the membership would issue them.
|
||||
func behind(t *testing.T, mesh ...string) *table {
|
||||
t.Helper()
|
||||
workload := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
io.WriteString(w, "the workload")
|
||||
}))
|
||||
t.Cleanup(workload.Close)
|
||||
at, _ := url.Parse(workload.URL)
|
||||
host, port, _ := net.SplitHostPort(at.Host)
|
||||
|
||||
routes, public, err := routesFrom(write(t, fmt.Sprintf(`{"given":[
|
||||
{"from":"app","node":"anchor","at":%q,
|
||||
"values":{"name":"app.example","internal-name":"app.anchor.internal","port":%s}},
|
||||
{"from":"admin","node":"anchor","at":%q,
|
||||
"values":{"internal-name":"admin.anchor.internal","port":%s}}
|
||||
]}`, host, port, host, port)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := newTable()
|
||||
inside, err := sourcesOf(mesh)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held.setInside(inside)
|
||||
held.set(routes, public)
|
||||
return held
|
||||
}
|
||||
|
||||
// askFrom is what the proxy answers a request for host coming from remote.
|
||||
func askFrom(held *table, host, remote string) (int, string) {
|
||||
r := httptest.NewRequest(http.MethodGet, "http://"+host+"/", nil)
|
||||
r.RemoteAddr = remote
|
||||
w := httptest.NewRecorder()
|
||||
handler(held).ServeHTTP(w, r)
|
||||
return w.Code, w.Body.String()
|
||||
}
|
||||
|
||||
// **An internal-only name is served to the private network and to nobody else** (novox/hq ADR
|
||||
// 0138, issue 191). The proxy answers public names on the same listeners, so without this a name
|
||||
// being internal kept nobody out: a request from the internet only had to carry it.
|
||||
func TestAnInternalOnlyNameIsServedOnlyInsideThePrivateNetwork(t *testing.T) {
|
||||
held := behind(t, "10.10.0.1", "10.10.0.7")
|
||||
|
||||
if code, body := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK ||
|
||||
body != "the workload" {
|
||||
t.Errorf("a request from the private network was not served: %d %q", code, body)
|
||||
}
|
||||
if code, body := askFrom(held, "admin.anchor.internal", "127.0.0.1:51000"); code != http.StatusOK {
|
||||
t.Errorf("a request from the machine itself was not served: %d %q", code, body)
|
||||
}
|
||||
|
||||
code, body := askFrom(held, "admin.anchor.internal", "203.0.113.9:51000")
|
||||
if code != http.StatusNotFound {
|
||||
t.Fatalf("a request from outside the private network reached an internal-only name: %d %q",
|
||||
code, body)
|
||||
}
|
||||
// Answered as a name never routed, and the list of what is served does not name it either —
|
||||
// otherwise the refusal would tell an outsider exactly what to ask for from inside.
|
||||
if strings.Contains(strings.SplitN(body, "\n", 2)[1], "admin.anchor.internal") {
|
||||
t.Errorf("the refusal names the internal-only route to an outsider: %q", body)
|
||||
}
|
||||
if !strings.Contains(body, "app.example") {
|
||||
t.Errorf("the refusal stopped listing the public names: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
// The internal name of a route that also has a public one is internal too: served inside, and to
|
||||
// an outsider only under the public name. Nothing is lost — the outsider has the public name — and a
|
||||
// name stays one thing whichever route it came from.
|
||||
func TestAnInternalAliasOfAPublicRouteIsServedInsideOnly(t *testing.T) {
|
||||
held := behind(t, "10.10.0.1", "10.10.0.7")
|
||||
if code, body := askFrom(held, "app.anchor.internal", "10.10.0.7:51000"); code != http.StatusOK {
|
||||
t.Errorf("the internal alias stopped answering the private network: %d %q", code, body)
|
||||
}
|
||||
if code, _ := askFrom(held, "app.anchor.internal", "203.0.113.9:51000"); code != http.StatusNotFound {
|
||||
t.Errorf("the internal alias was served to an outsider: %d", code)
|
||||
}
|
||||
if code, _ := askFrom(held, "app.example", "203.0.113.9:51000"); code != http.StatusOK {
|
||||
t.Errorf("the public name was refused to an outsider: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// Before a membership has said who the mesh is, only the machine itself is inside — refused to
|
||||
// everyone else, never served to everyone.
|
||||
func TestUntilTheMeshIsIssuedAnInternalOnlyNameIsServedToTheMachineAlone(t *testing.T) {
|
||||
held := behind(t)
|
||||
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:51000"); code != http.StatusNotFound {
|
||||
t.Errorf("an internal-only name was served with no private network said: %d", code)
|
||||
}
|
||||
if code, _ := askFrom(held, "admin.anchor.internal", "[::1]:51000"); code != http.StatusOK {
|
||||
t.Errorf("an internal-only name was refused to the machine itself: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
type from struct {
|
||||
net.Conn
|
||||
remote net.Addr
|
||||
}
|
||||
|
||||
func (c from) RemoteAddr() net.Addr { return c.remote }
|
||||
|
||||
// The handshake refuses an internal-only name to an outsider too: the certificate would name it,
|
||||
// and serving it would answer the question the routing refuses to.
|
||||
func TestTheHandshakeRefusesAnInternalOnlyNameToAnOutsider(t *testing.T) {
|
||||
held := behind(t, "10.10.0.1", "10.10.0.7")
|
||||
served := &tls.Certificate{}
|
||||
pick := certificateFor(held, func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return served, nil }, nil)
|
||||
hello := func(name, remote string) *tls.ClientHelloInfo {
|
||||
addr, _ := net.ResolveTCPAddr("tcp", remote)
|
||||
return &tls.ClientHelloInfo{ServerName: name, Conn: from{remote: addr}}
|
||||
}
|
||||
|
||||
if _, err := pick(hello("admin.anchor.internal", "203.0.113.9:443")); err == nil {
|
||||
t.Error("an outsider was handed a certificate for an internal-only name")
|
||||
}
|
||||
if got, err := pick(hello("admin.anchor.internal", "10.10.0.7:443")); err != nil || got != served {
|
||||
t.Errorf("a client on the private network was refused: %v", err)
|
||||
}
|
||||
if got, err := pick(hello("app.example", "203.0.113.9:443")); err != nil || got != served {
|
||||
t.Errorf("a public name was refused to an outsider: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The mesh is issued as machines' addresses; a range is read as well. One that does not parse is
|
||||
// refused rather than skipped, so a typo never quietly narrows or widens who is inside.
|
||||
func TestTheMeshIsReadAsAddressesAndRanges(t *testing.T) {
|
||||
if _, err := sourcesOf([]string{"10.10.0.1", "not-an-address"}); err == nil {
|
||||
t.Error("an entry that is not an address was accepted")
|
||||
}
|
||||
inside, err := sourcesOf([]string{"10.10.0.1", "fd00::1", "10.20.0.0/24"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for remote, want := range map[string]bool{
|
||||
"10.10.0.1:1": true,
|
||||
"[::ffff:10.10.0.1]:1": true,
|
||||
"[fd00::1]:1": true,
|
||||
"10.20.0.200:1": true,
|
||||
"10.10.0.2:1": false,
|
||||
"192.168.1.10:1": false,
|
||||
"not-an-address": false,
|
||||
} {
|
||||
if inside.holds(remote) != want {
|
||||
t.Errorf("%s inside the mesh: got %v, want %v", remote, !want, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the mesh issues is what is served: the routes in the membership, internal names to the
|
||||
// machines it names (novox/hq ADR 0167).
|
||||
func TestAMembershipIsServedAsIssued(t *testing.T) {
|
||||
held := newTable()
|
||||
took := applyMembership(broker.Membership{
|
||||
Receives: map[string]json.RawMessage{"route": json.RawMessage(`[
|
||||
{"from":"admin","node":"anchor","at":"anchor.internal",
|
||||
"values":{"internal-name":"admin.anchor.internal","port":8080}}]`)},
|
||||
Mesh: []string{"10.10.0.7"},
|
||||
}, held)
|
||||
if !took {
|
||||
t.Fatal("a membership carrying routes was not applied")
|
||||
}
|
||||
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.7:1"); code == http.StatusNotFound {
|
||||
t.Error("a machine the membership names was refused the internal-only route")
|
||||
}
|
||||
if code, _ := askFrom(held, "admin.anchor.internal", "10.10.0.9:1"); code != http.StatusNotFound {
|
||||
t.Errorf("a machine the membership does not name was served the internal-only route: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// A membership that says nothing about routes is one from a controller that does not issue them,
|
||||
// and changes nothing: the file stays the source rather than every route being withdrawn.
|
||||
func TestAMembershipWithoutRoutesLeavesTheFileServing(t *testing.T) {
|
||||
held := behind(t, "10.10.0.7")
|
||||
before := held.names()
|
||||
if applyMembership(broker.Membership{Mesh: []string{"10.10.0.7"}}, held) {
|
||||
t.Error("a membership without routes was taken as the source of routes")
|
||||
}
|
||||
if got := held.names(); strings.Join(got, ",") != strings.Join(before, ",") {
|
||||
t.Errorf("a membership without routes changed what is served: %v, was %v", got, before)
|
||||
}
|
||||
if applyMembership(broker.Membership{
|
||||
Receives: map[string]json.RawMessage{"route": json.RawMessage(`[]`)},
|
||||
Mesh: []string{"not-an-address"},
|
||||
}, held) {
|
||||
t.Error("a membership whose mesh cannot be read was applied")
|
||||
}
|
||||
}
|
||||
@@ -90,6 +90,50 @@ func TestARouteWithAnInternalNameIsReachableUnderBoth(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A route whose endpoint reaches only the private network carries an internal name and no public
|
||||
// one (novox/hq ADR 0138), and is served under that name rather than skipped as naming nothing —
|
||||
// skipping it left every internal-only module unreachable by name (novox/hq issue 191).
|
||||
func TestARouteWithOnlyAnInternalNameIsServed(t *testing.T) {
|
||||
routes, public, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"app","node":"anchor","at":"anchor.internal",
|
||||
"values":{"internal-name":"App.Anchor.Internal","port":8443,"scheme":"https","insecure":true}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if targetOf(routes, "app.anchor.internal") != "https://anchor.internal:8443" {
|
||||
t.Fatalf("the internal-only route is not served: %v", routes)
|
||||
}
|
||||
if len(routes) != 1 {
|
||||
t.Errorf("an internal-only route made hosts it never named: %v", routes)
|
||||
}
|
||||
if len(public) != 0 {
|
||||
t.Errorf("an internal-only route made a name eligible for a public certificate: %v", public)
|
||||
}
|
||||
|
||||
held := newTable()
|
||||
held.set(routes, public)
|
||||
if err := onlyInternalNamesTheMeshSaid(held)(context.Background(), "app.anchor.internal"); err != nil {
|
||||
t.Errorf("the internal authority refused the internal-only route's name: %v", err)
|
||||
}
|
||||
if err := onlyWhatTheMeshSaid(held)(context.Background(), "app.anchor.internal"); err == nil {
|
||||
t.Error("a public certificate was ordered for an internal-only name")
|
||||
}
|
||||
}
|
||||
|
||||
// A route with neither name has nothing to be served under, and is still skipped.
|
||||
func TestARouteWithNeitherNameIsSkipped(t *testing.T) {
|
||||
routes, public, err := routesFrom(write(t, `{"given":[
|
||||
{"from":"app","node":"anchor","at":"anchor.internal","values":{"internal-name":" ","port":8080}}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(routes) != 0 || len(public) != 0 {
|
||||
t.Errorf("a route that named nothing was served: %v %v", routes, public)
|
||||
}
|
||||
}
|
||||
|
||||
// A route with no internal-name composed gets no second host — the ordinary case, unchanged.
|
||||
func TestARouteWithNoInternalNameGetsNoAlias(t *testing.T) {
|
||||
routes, _, err := routesFrom(write(t, `{"given":[
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
@@ -33,6 +34,17 @@ type Membership struct {
|
||||
Reaches map[string][]string `json:"reaches,omitempty"`
|
||||
// Tools is where this instance answers what it serves — the runtime's one verb of its own.
|
||||
Tools string `json:"tools"`
|
||||
// Receives is what this assignment is given for each requirement it receives, by requirement:
|
||||
// the contributions of every module that asked for it, as the catalogue composed them (novox/hq
|
||||
// ADR 0167). The same list its received file is written from, so the two cannot disagree; a
|
||||
// requirement nobody contributed to is an empty list, never absent. Kept as JSON because the
|
||||
// catalogue owns the shape of a contribution and the bus only carries it.
|
||||
Receives map[string]json.RawMessage `json:"receives,omitempty"`
|
||||
// Mesh is every machine's address on the private network — what a rule saying "from the mesh"
|
||||
// resolves to in the packet filter, issued here from the same list (novox/hq ADR 0167). A
|
||||
// module that must tell the mesh from the world, the route proxy serving an internal name, reads
|
||||
// it here rather than keeping a definition of its own.
|
||||
Mesh []string `json:"mesh,omitempty"`
|
||||
}
|
||||
|
||||
// Served is one address a tool is answered on.
|
||||
|
||||
@@ -241,9 +241,14 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||
// Owner is kept beside the resources because a resource id cannot be split back into its module:
|
||||
// a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard —
|
||||
// has no owner.
|
||||
//
|
||||
// Received is what each module on the machine is given for each requirement it receives — the same
|
||||
// contributions its received file is written from, kept beside it so the mesh can also issue them
|
||||
// on the bus in the module's membership (novox/hq ADR 0167). By module, then requirement.
|
||||
type Composed struct {
|
||||
Resources []map[string]any
|
||||
Owner map[string]string
|
||||
Received map[string]map[string][]Contribution
|
||||
// LeftOut is every module of this machine's set that was left out of its declaration, and
|
||||
// why (novox/hq ADR 0163, rule 6): a setting stored for it that its definition can no longer
|
||||
// compose. Its held things are kept and its containers untouched — the machine is told so —
|
||||
@@ -267,8 +272,9 @@ func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string
|
||||
// Compose is Declaration with the owner of every resource said.
|
||||
func (r Resolution) Compose(with Rendering) (Composed, error) {
|
||||
owner := map[string]string{}
|
||||
received := map[string]map[string][]Contribution{}
|
||||
leftOut := map[string]string{}
|
||||
resources, err := r.compose(with, owner, leftOut)
|
||||
resources, err := r.compose(with, owner, received, leftOut)
|
||||
if err != nil {
|
||||
return Composed{}, err
|
||||
}
|
||||
@@ -280,7 +286,7 @@ func (r Resolution) Compose(with Rendering) (Composed, error) {
|
||||
"sealed": with.BusMembership, "mode": "0600",
|
||||
})
|
||||
}
|
||||
return Composed{Resources: resources, Owner: owner, LeftOut: leftOut}, nil
|
||||
return Composed{Resources: resources, Owner: owner, Received: received, LeftOut: leftOut}, nil
|
||||
}
|
||||
|
||||
// BusMembershipID names the resource carrying a machine's membership for the new bus, and
|
||||
@@ -289,7 +295,8 @@ func BusMembershipID() string { return "bus-membership" }
|
||||
|
||||
const BusMembershipPath = "/var/lib/mesh/membership-next.json"
|
||||
|
||||
func (r Resolution) compose(with Rendering, owner map[string]string, leftOut map[string]string) ([]map[string]any, error) {
|
||||
func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
received map[string]map[string][]Contribution, leftOut map[string]string) ([]map[string]any, error) {
|
||||
// **A setting is judged where it is stored, and an impossible one costs a module, not a
|
||||
// machine** (novox/hq ADR 0163, rule 6). A definition that moved under a stored setting makes
|
||||
// this module uncomposable; it is left out of the declaration — its held things kept, its
|
||||
@@ -633,6 +640,12 @@ func (r Resolution) compose(with Rendering, owner map[string]string, leftOut map
|
||||
return nil, err
|
||||
}
|
||||
first = append(first, file)
|
||||
if received[m.Module] == nil {
|
||||
received[m.Module] = map[string][]Contribution{}
|
||||
}
|
||||
// Empty rather than absent when nobody contributed, for the reason the file is
|
||||
// written empty: "nothing asked" and "never told" want different responses.
|
||||
received[m.Module][to] = append([]Contribution{}, given[to]...)
|
||||
}
|
||||
if m.Keeps != "" && with.Kept != nil {
|
||||
file, err := keptFile(m.Keeps, with.Kept)
|
||||
|
||||
@@ -444,6 +444,18 @@ func AsNftables(rules []Rule, mesh []string, outward bool, foundation []int,
|
||||
b.WriteString("\t\t# this machine's own guests reaching outward: not a port opened to anybody\n")
|
||||
b.WriteString(fmt.Sprintf("\t\tiifname != { %s } accept\n", inward))
|
||||
}
|
||||
// **The mesh passing through, not arriving.** A machine the mesh routes through — the hub, for
|
||||
// every path between machines that are not co-located (novox/hq ADR 0007) — relays a packet that
|
||||
// came in on the tunnel and leaves on it again, addressed to another machine of the mesh. That is
|
||||
// no port of this machine's: the machine it is for filters it against its own rules. Without
|
||||
// this, the chain below judged a relayed packet by this machine's own published ports, so two
|
||||
// machines behind the hub reached each other only on ports the hub happened to publish for itself
|
||||
// (novox/hq issue 196). In and out on the tunnel both: a packet off the tunnel for this machine's
|
||||
// own containers leaves by a bridge, and still meets the rules below.
|
||||
if tunnel != "" {
|
||||
b.WriteString("\t\t# the mesh passing through to another of its machines, which filters it itself\n")
|
||||
b.WriteString(fmt.Sprintf("\t\tiifname %q oifname %q accept\n", tunnel, tunnel))
|
||||
}
|
||||
|
||||
if len(rules) > 0 {
|
||||
b.WriteString("\n")
|
||||
|
||||
@@ -887,3 +887,34 @@ func TestAPublicPortNeedsNoGuestLine(t *testing.T) {
|
||||
t.Fatalf("a public port was given a guest line it does not need:\n%s", nft)
|
||||
}
|
||||
}
|
||||
|
||||
// **The hub relays the mesh** (novox/hq ADR 0007, issue 196). Two machines that are not co-located
|
||||
// reach each other through the hub, so the hub forwards a packet that arrives on the tunnel and
|
||||
// leaves on it. The forward chain judged that packet by the hub's own published ports, and two
|
||||
// machines behind the hub reached each other only on the ports the hub happened to publish.
|
||||
//
|
||||
// Measured: from one home machine to another through the hub, 17 of 55 ports answered, and they
|
||||
// were exactly the hub's own; the SYN for the rest never left the hub.
|
||||
func TestTheMeshPassingThroughIsRelayedNotJudgedAsThisMachines(t *testing.T) {
|
||||
nft := AsNftables(nil, []string{"10.42.0.1", "10.42.0.2"}, false, nil, []string{"eth0"}, "mesh0")
|
||||
relay := `iifname "mesh0" oifname "mesh0" accept`
|
||||
if !strings.Contains(chainBody(t, nft, "forward"), relay) {
|
||||
t.Errorf("the forward chain does not relay the mesh through this machine:\n%s", chainBody(t, nft, "forward"))
|
||||
}
|
||||
// Relaying is not receiving: nothing in the input chain opens because of it.
|
||||
if strings.Contains(chainBody(t, nft, "input"), "oifname") {
|
||||
t.Errorf("the input chain names an outgoing interface, which no packet for this machine has:\n%s",
|
||||
chainBody(t, nft, "input"))
|
||||
}
|
||||
// And off the tunnel into this machine's own containers is still judged: the tunnel is not
|
||||
// accepted wholesale, only in and out on it.
|
||||
if strings.Contains(chainBody(t, nft, "forward"), `iifname "mesh0" accept`) {
|
||||
t.Errorf("the forward chain accepts everything off the tunnel:\n%s", chainBody(t, nft, "forward"))
|
||||
}
|
||||
|
||||
// A machine with no tunnel relays nothing, and names no interface it does not have.
|
||||
alone := AsNftables(nil, nil, false, nil, []string{"eth0"}, "")
|
||||
if strings.Contains(alone, "oifname") {
|
||||
t.Errorf("a machine with no tunnel was given a relay rule:\n%s", alone)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1810,6 +1810,12 @@ func (m Manifest) undeclaredMounts() []string {
|
||||
claim(p)
|
||||
}
|
||||
}
|
||||
// The file a filter module's rule set is written to is declared by `filtering.into`: the mesh
|
||||
// writes it, the module loads it, and the module's runtime may read it back to reload the
|
||||
// mesh's own table (novox/hq ADR 0169).
|
||||
if m.Filtering != nil {
|
||||
claim(m.Filtering.Into)
|
||||
}
|
||||
// Under a declared directory is declared: a module that says where its data lives has said so
|
||||
// for what it puts inside.
|
||||
covers := func(path string) bool {
|
||||
|
||||
@@ -98,3 +98,13 @@ func TestAMountOfABoundFactIsAccepted(t *testing.T) {
|
||||
t.Fatalf("a mount of the file the mesh writes a binding to was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The file a filter module's rule set is written to is declared by `filtering.into` (novox/hq ADR
|
||||
// 0169): the module's runtime mounts it to reload the mesh's own table, and nothing else declares it.
|
||||
func TestAMountOfTheFilterFileIsDeclaredByFilteringInto(t *testing.T) {
|
||||
_, err := ParseManifest([]byte(`{"module":"nftables","filtering":{"into":"/etc/nftables.conf"},` +
|
||||
`"resources":[` + strings.Replace(aContainerMounting, "%s", "/etc/nftables.conf", 1) + `]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("a filter module mounting its own filter file was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// What a provider receives is composed once, and issued twice: as its received file, and in its
|
||||
// membership on the bus (novox/hq ADR 0167). The two are the same list, so a proxy reading the bus
|
||||
// and one reading the file serve the same routes — including the port the machine published, which
|
||||
// is the same-node fix the file already carries.
|
||||
func TestWhatAProviderReceivesIsTheSameOnTheBusAsInItsFile(t *testing.T) {
|
||||
gitea := Manifest{
|
||||
Module: "gitea", Version: "1",
|
||||
Listens: []Listening{{Port: 3000, Protocol: "tcp", From: FromMesh}},
|
||||
Contributes: map[string]map[string]any{"route": {"label": "git", "port": 3000}},
|
||||
Resources: []map[string]any{{
|
||||
"id": "server", "type": "container", "name": "gitea", "ports": []any{"3000"},
|
||||
}},
|
||||
}
|
||||
r, err := Resolve(shelf(gitea, routeProxy(), stepCA()),
|
||||
[]string{"gitea", "route-proxy", "step-ca"}, reachable(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composed, err := r.Compose(Rendering{Ports: map[string]map[int]int{"gitea": {3000: 20000}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
file := fileNamed(composed.Resources, "route-proxy.received-route")
|
||||
if file == nil {
|
||||
t.Fatal("the proxy was given no routes file")
|
||||
}
|
||||
var written struct {
|
||||
Given []Contribution `json:"given"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(file["content"].(string)), &written); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
issued, said := composed.Received["route-proxy"]["route"]
|
||||
if !said {
|
||||
t.Fatalf("nothing is issued for the proxy to receive on the bus: %v", composed.Received)
|
||||
}
|
||||
// Compared as JSON, which is what both are once they leave the controller.
|
||||
a, _ := json.Marshal(written.Given)
|
||||
b, _ := json.Marshal(issued)
|
||||
var fromFile, fromBus any
|
||||
_ = json.Unmarshal(a, &fromFile)
|
||||
_ = json.Unmarshal(b, &fromBus)
|
||||
if !reflect.DeepEqual(fromFile, fromBus) {
|
||||
t.Errorf("the bus and the file disagree about the routes:\nfile %s\nbus %s", a, b)
|
||||
}
|
||||
if len(issued) != 1 {
|
||||
t.Fatalf("expected one route on the bus, got %v", issued)
|
||||
}
|
||||
if port, ok := asPort(issued[0].Values["port"]); !ok || port != 20000 {
|
||||
t.Errorf("the bus carries a port nothing listens on: %v", issued[0].Values["port"])
|
||||
}
|
||||
|
||||
// A module that receives nothing is issued nothing to receive.
|
||||
if _, any := composed.Received["gitea"]; any {
|
||||
t.Errorf("a module that receives nothing was issued something: %v", composed.Received["gitea"])
|
||||
}
|
||||
}
|
||||
@@ -100,7 +100,25 @@ var defaultSeats = []Seat{
|
||||
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0121"},
|
||||
{Name: "node-dns-resolver", Scope: ScopeNode, Decision: "novox/hq ADR 0121"},
|
||||
{Name: "node-intrusion-prevention", Scope: ScopeNode, Decision: "novox/hq ADR 0121"},
|
||||
{Name: "node-packet-filter", Scope: ScopeNode, Decision: "novox/hq ADR 0121"},
|
||||
// The packet filter's verbs (novox/hq ADR 0169): what a person asks a machine's filter whatever
|
||||
// filter answers — the rules as enforced, reload the mesh's own, remove one thing the mesh did
|
||||
// not write. Every holder serves all three; what differs by filter is the holder's own tools.
|
||||
{Name: "node-packet-filter", Scope: ScopeNode, Decision: "novox/hq ADR 0121",
|
||||
Serves: []Verb{
|
||||
{Name: "rules", Description: "The packet filter as this machine enforces it now: the nftables " +
|
||||
"ruleset and, where the tool exists, the legacy filter's listings. Narrowed to one table or " +
|
||||
"chain when asked.",
|
||||
Input: schema(map[string]string{"table": "one nftables table, as `family name` (optional)",
|
||||
"chain": "one chain of that table (optional)"}, nil)},
|
||||
{Name: "reload", Description: "Load the mesh's own filter again from the file the mesh writes, " +
|
||||
"and answer with the mesh's table as loaded.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
{Name: "remove", Description: "Remove one rule set the mesh did not write, named exactly as the " +
|
||||
"host reports it (novox/hq ADR 0168) — `chain X (iptables-legacy)` or `table ip6 filter, chain " +
|
||||
"DOCKER-USER`. Refuses the mesh's tables, the runtime's own chains, a built-in chain and an " +
|
||||
"active found firewall's chains. An operator's act, by name, never a flush.",
|
||||
Input: schema(map[string]string{"where": "the rule set, as `node show` lists it"}, []string{"where"})},
|
||||
}},
|
||||
// Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client
|
||||
// model change, not a rename, so it stays until that is built.
|
||||
{Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
||||
|
||||
@@ -136,6 +136,15 @@ var ControllerVerbs = []Verb{
|
||||
"node": "the machine that runs the module",
|
||||
"module": "the module's name",
|
||||
}, []string{"node", "module"})},
|
||||
{Name: "settings", Description: "Set what an assignment is configured with: a module's settings for the whole mesh, " +
|
||||
"or for one machine. Replaces that layer whole — what it does not name, it no longer sets — and takes effect " +
|
||||
"at the next push. With clear, removes the layer and the module is back to what its definition says.",
|
||||
Input: schema(map[string]string{
|
||||
"module": "the module's name",
|
||||
"values": "the settings as a JSON object, for set",
|
||||
"node": "one machine; the whole mesh when absent",
|
||||
"clear": "\"true\" to remove the layer instead of setting it",
|
||||
}, []string{"module"})},
|
||||
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
|
||||
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
|
||||
Input: schema(map[string]string{
|
||||
|
||||
@@ -178,6 +178,103 @@ type Stray struct {
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// A Filter is one place on a machine that refuses traffic, with its owner (novox/hq ADR 0168).
|
||||
type Filter struct {
|
||||
Where string `json:"where"`
|
||||
Owner string `json:"owner"`
|
||||
Refuses string `json:"refuses"`
|
||||
}
|
||||
|
||||
// Owners of a filter, as the host names them (ADR 0168).
|
||||
const (
|
||||
FilterMesh = "mesh"
|
||||
FilterFoundFirewall = "found-firewall"
|
||||
FilterRuntime = "runtime"
|
||||
FilterBan = "ban"
|
||||
FilterOther = "other"
|
||||
)
|
||||
|
||||
// FoundFirewall is the state of a converged machine's found firewall (ADR 0168): in force now or
|
||||
// not, and how it came to be inactive.
|
||||
type FoundFirewall struct {
|
||||
Kind string `json:"kind"`
|
||||
Active bool `json:"active"`
|
||||
RetiredBy string `json:"retired_by,omitempty"`
|
||||
}
|
||||
|
||||
// Filtering is what a machine last said filters it (ADR 0168).
|
||||
type Filtering struct {
|
||||
Filters []Filter
|
||||
FoundFirewall *FoundFirewall
|
||||
}
|
||||
|
||||
// Alone is whether the machine is filtered by the mesh alone: nothing in its list but the mesh's
|
||||
// own, the runtime's plumbing and bans, and no found firewall in force.
|
||||
func (f Filtering) Alone() bool {
|
||||
for _, x := range f.Filters {
|
||||
if x.Owner == FilterOther || x.Owner == FilterFoundFirewall {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return f.FoundFirewall == nil || !f.FoundFirewall.Active
|
||||
}
|
||||
|
||||
// Others is every filter that is neither the mesh's, the runtime's nor a ban.
|
||||
func (f Filtering) Others() []Filter {
|
||||
var out []Filter
|
||||
for _, x := range f.Filters {
|
||||
if x.Owner == FilterOther || x.Owner == FilterFoundFirewall {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RecordFiltering keeps what a machine last said filters it, replacing what was there (ADR 0168).
|
||||
func (i *Inventory) RecordFiltering(ctx context.Context, nodeID string, filters []Filter, found *FoundFirewall) error {
|
||||
raw, err := json.Marshal(nonNil(filters))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var foundRaw any
|
||||
if found != nil {
|
||||
b, err := json.Marshal(found)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
foundRaw = string(b)
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`update node set filters = $2, found_firewall = $3 where id = $1`, nodeID, raw, foundRaw)
|
||||
return err
|
||||
}
|
||||
|
||||
// FilteringOf is what a machine last said filters it; empty for a machine that never said.
|
||||
func (i *Inventory) FilteringOf(ctx context.Context, name string) (Filtering, error) {
|
||||
var filtersRaw, foundRaw []byte
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select filters, found_firewall from node where name = $1`, name).Scan(&filtersRaw, &foundRaw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Filtering{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
||||
}
|
||||
if err != nil {
|
||||
return Filtering{}, err
|
||||
}
|
||||
var out Filtering
|
||||
if len(filtersRaw) > 0 {
|
||||
if err := json.Unmarshal(filtersRaw, &out.Filters); err != nil {
|
||||
return Filtering{}, err
|
||||
}
|
||||
}
|
||||
if len(foundRaw) > 0 {
|
||||
out.FoundFirewall = &FoundFirewall{}
|
||||
if err := json.Unmarshal(foundRaw, out.FoundFirewall); err != nil {
|
||||
return Filtering{}, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Reach is one thing reachable on an adopted node: a listening socket or a published port.
|
||||
type Reach struct {
|
||||
Protocol string `json:"protocol"`
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
-- What filters a machine, with owners, as the host reports it with every apply (novox/hq ADR 0168):
|
||||
-- every table and chain that refuses traffic — the mesh's, the found firewall's, the runtime's own,
|
||||
-- a ban, or other — so the mesh says truthfully what filters a converged machine and names what it
|
||||
-- did not write. And the state of the firewall a converged machine was found with: in force now or
|
||||
-- not, and who retired it.
|
||||
alter table node add column filters jsonb;
|
||||
alter table node add column found_firewall jsonb;
|
||||
@@ -305,6 +305,23 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
// What filters the machine, and the state of its found firewall (novox/hq ADR 0168), whenever
|
||||
// it says — every apply of a host that knows how, adopted or converged; never cleared by a
|
||||
// report that carries none, which is every bare word that the node is there.
|
||||
if len(report.Filters) > 0 || report.FoundFirewall != nil {
|
||||
filters := make([]inventory.Filter, 0, len(report.Filters))
|
||||
for _, f := range report.Filters {
|
||||
filters = append(filters, inventory.Filter{Where: f.Where, Owner: f.Owner, Refuses: f.Refuses})
|
||||
}
|
||||
var found *inventory.FoundFirewall
|
||||
if report.FoundFirewall != nil {
|
||||
found = &inventory.FoundFirewall{Kind: report.FoundFirewall.Kind, Active: report.FoundFirewall.Active,
|
||||
RetiredBy: report.FoundFirewall.RetiredBy}
|
||||
}
|
||||
if err := e.Inventory.RecordFiltering(ctx, node.ID, filters, found); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
// Which of its links face outside (novox/hq ADR 0140), whenever it says so. Recorded on every
|
||||
// report that carries it, adopted or converged, because the filter the mesh composes is written
|
||||
// around it — and never cleared by a report that carries none, which is every bare word that the
|
||||
|
||||
@@ -218,3 +218,45 @@ func TestWhatAnAdoptedNodeHoldsIsKeptAndAnAliveWordDoesNotWipeIt(t *testing.T) {
|
||||
t.Fatalf("a report from an adopted node holding nothing did not empty held: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// What filters a machine, and the state of its found firewall, are kept from every report that
|
||||
// carries them and never cleared by one that does not (novox/hq ADR 0168).
|
||||
func TestWhatFiltersAMachineIsKeptFromItsReport(t *testing.T) {
|
||||
inv, _, _ := heardFrom(t, link.Report{
|
||||
Node: "home-server", Applied: []string{"a"},
|
||||
Filters: []link.Filter{
|
||||
{Where: "table inet mesh, chain forward", Owner: "mesh", Refuses: "policy drop"},
|
||||
{Where: "chain HAL-MESH-ONLY (iptables-legacy)", Owner: "other", Refuses: "-j DROP"},
|
||||
},
|
||||
FoundFirewall: &link.FoundFirewall{Kind: "ufw", Active: false, RetiredBy: "found-inactive"},
|
||||
})
|
||||
ctx := context.Background()
|
||||
f, err := inv.FilteringOf(ctx, "home-server")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.Filters) != 2 || f.Filters[1].Owner != inventory.FilterOther || f.Alone() {
|
||||
t.Fatalf("recorded %+v", f)
|
||||
}
|
||||
if f.FoundFirewall == nil || f.FoundFirewall.RetiredBy != "found-inactive" || f.FoundFirewall.Active {
|
||||
t.Fatalf("the found firewall's state: %+v", f.FoundFirewall)
|
||||
}
|
||||
if len(f.Others()) != 1 || f.Others()[0].Where != "chain HAL-MESH-ONLY (iptables-legacy)" {
|
||||
t.Fatalf("others: %+v", f.Others())
|
||||
}
|
||||
// A bare word that the node is there clears nothing.
|
||||
if _, err := (link.Enrolment{Inventory: inv}).Heard(ctx, link.Report{Node: "home-server"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _ := inv.FilteringOf(ctx, "home-server"); len(again.Filters) != 2 {
|
||||
t.Fatalf("a bare report cleared what filters the machine: %+v", again)
|
||||
}
|
||||
// The next full report replaces it: the chain removed by hand is gone from the record.
|
||||
if _, err := (link.Enrolment{Inventory: inv}).Heard(ctx, link.Report{Node: "home-server", Applied: []string{"a"},
|
||||
Filters: []link.Filter{{Where: "table inet mesh, chain forward", Owner: "mesh", Refuses: "policy drop"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _ := inv.FilteringOf(ctx, "home-server"); len(again.Filters) != 1 || !again.Alone() {
|
||||
t.Fatalf("the next report did not replace what filters the machine: %+v", again)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -197,6 +197,15 @@ type Report struct {
|
||||
// Strays is what runs on the machine that the mesh neither wrote nor holds (ADR 0163).
|
||||
Strays []Stray `json:"strays,omitempty"`
|
||||
|
||||
// Filters is what filters the machine now: every table and chain that refuses traffic, with
|
||||
// its owner — the mesh's, the found firewall's, the container runtime's own, a ban, or other
|
||||
// (novox/hq ADR 0168). Every machine reports it, adopted or converged; absent from a host older
|
||||
// than this.
|
||||
Filters []Filter `json:"filters,omitempty"`
|
||||
// FoundFirewall is the state of the firewall a converged machine was found with: in force now
|
||||
// or not, and how it came to be inactive — the mesh disabled it, or it was found so (ADR 0168).
|
||||
FoundFirewall *FoundFirewall `json:"found_firewall,omitempty"`
|
||||
|
||||
// Profile is what the machine can do, detected again by this apply (novox/hq ADR 0161): the
|
||||
// same shape enrolment sends, so a machine that gained or lost a capability — switched its
|
||||
// network manager — is known at its next push and not at its next enrolment. Absent from a host
|
||||
@@ -278,6 +287,21 @@ type Held struct {
|
||||
Facts map[string]any `json:"facts,omitempty"`
|
||||
}
|
||||
|
||||
// A Filter is one place on a machine that refuses traffic, with its owner (novox/hq ADR 0168):
|
||||
// the host's own shape, carried as data.
|
||||
type Filter struct {
|
||||
Where string `json:"where"`
|
||||
Owner string `json:"owner"`
|
||||
Refuses string `json:"refuses"`
|
||||
}
|
||||
|
||||
// FoundFirewall is the state of a converged machine's found firewall (ADR 0168).
|
||||
type FoundFirewall struct {
|
||||
Kind string `json:"kind"`
|
||||
Active bool `json:"active"`
|
||||
RetiredBy string `json:"retired_by,omitempty"`
|
||||
}
|
||||
|
||||
// A Stray is a container a machine runs that the mesh neither wrote nor holds (ADR 0163).
|
||||
type Stray struct {
|
||||
Kind string `json:"kind"`
|
||||
|
||||
Reference in New Issue
Block a user