package main import ( "context" "crypto/sha256" "encoding/hex" "errors" "flag" "fmt" "slices" "sort" "strings" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/overlay" ) // A node is adopted or converged (novox/hq ADR 0100), and it is said to be adopted wherever the // mesh reports a node's state: node list, node show, status and the board. // showMode is the node show lines about a node's mode and what was taken on it. func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node) error { if !node.Adopted { fmt.Printf(" mode converged\n") // A converged machine holds nothing, and can still run what nobody asked for // (novox/hq ADR 0163): what it reports as strays is said whatever its mode. if said, err := inv.AdoptionOf(ctx, node.Name); err == nil && len(said.Strays) > 0 { showStrays(said.Strays) } // And what filters it, truthfully (novox/hq ADR 0168): the mesh alone, or not. if filtering, err := inv.FilteringOf(ctx, node.Name); err == nil { showFiltering(filtering, false) } return nil } fmt.Printf(" mode adopted since %s\n", node.AdoptedSince.Local().Format(time.DateTime)) taken, err := inv.Taken(ctx, node.Name) if err != nil { return err } if len(taken) == 0 { fmt.Printf(" taken nothing yet\n") } else { fmt.Printf(" taken %s\n", strings.Join(taken, ", ")) } said, err := inv.AdoptionOf(ctx, node.Name) if err != nil { return err } if said.At.IsZero() { fmt.Printf(" it has not yet said what it found\n") return nil } fmt.Printf(" firewall found %s\n", orNone(said.Firewall)) if err := showTunnel(ctx, inv, node.Name); err != nil { return err } if len(said.Held) == 0 { fmt.Printf(" holding nothing found\n") } for _, h := range said.Held { // A held thing that changed is how a predecessor still writing is caught: said first. line := fmt.Sprintf(" holds %-11s %s %s, for %s", h.Kind, h.Target, h.ID, h.Module) if h.Changed != "" { line += " — " + strings.ToUpper(h.Changed) + " by something other than the mesh" } fmt.Println(line) if h.Kept != "" { fmt.Printf(" %-17s original kept at %s\n", "", h.Kept) } for _, f := range comparisonLines(h) { fmt.Printf(" %-17s %s\n", "", f) } } showStrays(said.Strays) if filtering, err := inv.FilteringOf(ctx, node.Name); err == nil { showFiltering(filtering, true) } fmt.Printf(" as of %s\n", said.At.Local().Format(time.DateTime)) return nil } // showFiltering says what filters a machine, with owners (novox/hq ADR 0168), and for a converged // machine the state of the firewall it was found with. A machine that has not said is not said to // be filtered by anything. func showFiltering(f inventory.Filtering, adopted bool) { if len(f.Filters) == 0 && f.FoundFirewall == nil { return } if fw := f.FoundFirewall; fw != nil && !adopted { switch { case fw.Active: fmt.Printf(" found firewall %s is ACTIVE on this converged machine; the next apply retires it again\n", fw.Kind) case fw.RetiredBy == "removed": fmt.Printf(" found firewall %s, removed: the mesh's filter is what filters this machine (novox/hq ADR 0180)\n", fw.Kind) case fw.RetiredBy == inventory.FilterMesh || fw.RetiredBy == "mesh": fmt.Printf(" found firewall %s, retired by the mesh; its configuration stays on disk\n", fw.Kind) case fw.RetiredBy != "": fmt.Printf(" found firewall %s, found inactive — not by the mesh\n", fw.Kind) default: fmt.Printf(" found firewall %s, inactive\n", fw.Kind) } } if len(f.Filters) == 0 { return } if f.Alone() { fmt.Printf(" filtered by the mesh alone (%s)\n", filterSummary(f.Filters)) return } fmt.Printf(" filtered by NOT the mesh alone: %d rule set(s) the mesh did not write refuse traffic here\n", len(f.Others())) for _, x := range f.Filters { if x.Owner == inventory.FilterOther || x.Owner == inventory.FilterFoundFirewall { fmt.Printf(" %-17s %s — %s: %s\n", "", x.Where, x.Owner, x.Refuses) } } fmt.Printf(" %-17s and its own: %s\n", "", filterSummary(f.Filters)) } // filterSummary counts a machine's filters by owner: "mesh 2, runtime 3, ban 1". func filterSummary(filters []inventory.Filter) string { counts := map[string]int{} for _, x := range filters { counts[x.Owner]++ } var parts []string for _, owner := range []string{inventory.FilterMesh, inventory.FilterRuntime, inventory.FilterBan, inventory.FilterFoundFirewall, inventory.FilterOther} { if n := counts[owner]; n > 0 { parts = append(parts, fmt.Sprintf("%s %d", owner, n)) } } return strings.Join(parts, ", ") } // showStrays says what a machine runs that the mesh neither wrote nor holds (ADR 0163). func showStrays(strays []inventory.Stray) { if len(strays) == 0 { return } fmt.Printf(" strays %d container(s) the mesh neither wrote nor holds:\n", len(strays)) for _, s := range strays { fmt.Printf(" %-17s %s (%s)\n", "", s.Name, s.Detail) } } // showTunnel is the node show lines about the tunnel an adopted node found and carried (novox/hq // ADR 0105): what it presented at enrolment, and what it last said about taking it over. func showTunnel(ctx context.Context, inv *inventory.Inventory, name string) error { tunnel, err := inv.TunnelOf(ctx, name) if errors.Is(err, inventory.ErrNoTunnel) { return nil } if err != nil { return err } fmt.Printf(" tunnel found %s on port %d, %s in %s, %d peer(s)\n", tunnel.Interface, tunnel.Port, tunnel.Address, tunnel.Range, len(tunnel.Peers)) carried, said, err := inv.CarriedTunnelOf(ctx, name) if err != nil { return err } switch { case !said: fmt.Printf(" %-17s not yet taken over — the node has not said so\n", "") case carried.State == inventory.CarriedTaken: fmt.Printf(" %-17s taken over: %s is down and disabled, never flushed; the mesh's interface "+ "runs with its key, port and %d peer(s)\n", "", carried.Interface, carried.Peers) case carried.State == inventory.CarriedDown: fmt.Printf(" %-17s TUNNEL DOWN: %s is stopped and the mesh's interface is not up — the peers "+ "reach nothing. On the machine: systemctl start %s\n", "", carried.Interface, "wg-quick@"+carried.Interface) default: fmt.Printf(" %-17s NOT taken over: %s is still the interface the peers reach\n", "", carried.Interface) } if said && carried.Note != "" { fmt.Printf(" %-17s %s\n", "", carried.Note) } if said && carried.Kept != "" { fmt.Printf(" %-17s its configuration's original kept at %s\n", "", carried.Kept) } return nil } func orNone(s string) string { if s == "" { return "none reported" } return s } // adoptedNodes are the names of every adopted node, in the order given. func adoptedNodes(nodes []inventory.Node) []string { var out []string for _, n := range nodes { if n.Adopted { out = append(out, n.Name) } } return out } // The operator's acts on an adopted node (novox/hq ADR 0100). Called by the command line and the // command API alike, so a refusal is the same refusal in the same words at both (ADR 0035). // sendNodes sends the named machines what they should be now. A variable so a test can see what // an act would send without a broker. var sendNodes = sendTo // DefaultFilter is the module converging a node assigns to load the mesh's derived filter. const DefaultFilter = "nftables" // take is a module's cutover on an adopted node: the operator's act, done when that module's data // has moved. From the next push its resources converge there like any other, replacing what the // node found and holds for it. // // **Previewed, and the preview is a comparison** (novox/hq ADR 0163): for every held thing the // module would replace, what runs beside what the module declares, and the difference; the // module's secrets on the machine and where each came from; its settings on the machine. Without // --yes the comparison is printed and nothing changes. `--yes ` cuts over exactly what was // previewed, the way the flip is confirmed: the preview ends with a digest of what it said, and a // take naming an older one, or acting on an account of the machine older than the flip allows, is // refused. A module the machine holds nothing for has nothing to compare, and `--yes` suffices. // takeOptions is what a take was told about the differences it may pass (novox/hq ADR 0163). type takeOptions struct { Yes bool // Digest is the preview's, named with --yes; required whenever the machine holds something // for the module. Digest string Downgrade bool Replace map[string]bool // Mint names the secrets the service shall take a new value for, although the mesh minted // one and the service already has its own (rule 2). Mint map[string]bool } func take(ctx context.Context, open *stores, node, module string, opts takeOptions) (string, error) { inv := open.inventory assigned, err := inv.Assigned(ctx, node) if err != nil { return "", err } if !slices.Contains(assigned, module) { if plan, _, err := planFor(ctx, open, node); err == nil { if why, runs := plan.Because[module]; runs { return "", fmt.Errorf("%w: %s runs on %s because %s — assign it to %s to take it", inventory.ErrNotAssigned, module, node, why, node) } } } // The comparison first (novox/hq ADR 0163): every held thing the module would replace, beside // what the module declares, and the differences that refuse unless named. c, err := comparisonFor(ctx, open, node, module) if err != nil { return "", err } preview, refusals, saw := comparisonOf(module, c, opts) if len(refusals) > 0 { return "", fmt.Errorf("taking %s on %s is refused:\n %s\n%s", module, node, strings.Join(refusals, "\n "), preview) } holds := len(heldOf(c.reported, module)) > 0 if holds { preview += "\n preview " + saw } if !opts.Yes { if !holds { return preview + fmt.Sprintf("\nnothing taken; `take %s %s --yes` declares it as the mesh's own", node, module), nil } return preview + fmt.Sprintf("\nnothing taken; `take %s %s --yes %s` cuts it over as previewed", node, module, saw), nil } if holds { // The take acts on the preview the operator saw, and on an account of the machine that // is still the machine: the same two refusals the flip makes. if age := time.Since(c.reported.At); age > reportFreshFor { return preview, fmt.Errorf("%s last said what it holds %s ago, and a take acts only on "+ "an account newer than %s: run `push %s --wait 2m`, then preview again", node, age.Round(time.Second), reportFreshFor, node) } if opts.Digest == "" { return preview, fmt.Errorf("taking %s on %s acts on the preview you saw: name its digest, "+ "`take %s %s --yes %s`, once you have read it", module, node, node, module, saw) } if opts.Digest != saw { return preview, fmt.Errorf("what taking %s on %s would replace has changed since preview %s "+ "(it is now %s): read the preview above, and run `take %s %s --yes %s` if it is "+ "what you want", module, node, opts.Digest, saw, node, module, saw) } } if err := inv.Take(ctx, node, module); err != nil { return "", err } said := fmt.Sprintf("%s is taken on %s", module, node) if holds { said += "; the next push replaces what the node found and holds for it:\n" + preview } return said + fmt.Sprintf("\n run `push %s` to cut it over", node), nil } // comparison is everything a take puts beside what the module declares: the machine's account of // what it holds and what is reachable on it, the module's secrets on the machine, its settings // there, and which found networks a setting keeps for each of its containers (by held id). type comparison struct { reported inventory.Adoption secrets []inventory.SecretState layers []catalogue.Layer keeps map[string][]string // settingsRefused is why the module's settings cannot compose with its definition, when // they cannot — the module would be left out of the declaration (rule 6). settingsRefused string } func comparisonFor(ctx context.Context, open *stores, node, module string) (comparison, error) { inv := open.inventory var c comparison var err error if c.reported, err = inv.AdoptionOf(ctx, node); err != nil { return c, err } if c.secrets, err = inv.SecretsOf(ctx, node, module); err != nil { return c, err } if c.layers, err = inv.SettingsFor(ctx, node, module); err != nil { return c, err } shelf, err := inv.Catalogue(ctx) if err != nil { return c, err } if m, known := shelf[module]; known && len(c.layers) > 0 { if err := catalogue.JudgeSettings(m, c.layers, true); err != nil { c.settingsRefused = err.Error() } if kept, err := catalogue.KeptNetworks(m, c.layers, true); err == nil && len(kept) > 0 { c.keeps = map[string][]string{} for id, networks := range kept { c.keeps[module+"."+id] = networks } } } return c, nil } // heldOf is what a node holds for one module. func heldOf(reported inventory.Adoption, module string) []inventory.Held { var out []inventory.Held for _, h := range reported.Held { if h.Module == module { out = append(out, h) } } return out } // comparisonOf is a take's preview: for every held thing of the module, what runs beside what the // module declares; its secrets and its settings on the machine; and the refusals the differences // earn unless the take named them (novox/hq ADR 0163): an image older than the one running, a // declared file that differs from the found one, a secret the mesh minted for a service whose data // was found. A narrowed port and a shared network are said and not refused. The digest is of what // the preview says, so anything in it changing changes the digest. func comparisonOf(module string, c comparison, opts takeOptions) (preview string, refusals []string, digest string) { var b strings.Builder held := heldOf(c.reported, module) foundData := false for _, h := range held { if h.Kind == "container" || h.Kind == "directory" { foundData = true } fmt.Fprintf(&b, " %s", heldLine(h)) if h.Kept != "" { fmt.Fprintf(&b, ", original kept at %s", h.Kept) } b.WriteString("\n") for _, line := range comparisonLinesWith(h, c.keeps[h.ID], c.reported) { fmt.Fprintf(&b, " %s\n", line) } f := factsOf(h) if f.downgrade && !opts.Downgrade { refusals = append(refusals, fmt.Sprintf("%s: the module's image (%s, made %s) is older than the one running (%s, made %s) — "+ "a service that migrated its data forward may not start on it; `--downgrade` to take it anyway", h.Target, f.declaredImage, day(f.declaredCreated), f.image, day(f.imageCreated))) } if f.differs && !opts.Replace[h.Target] && !opts.Replace["*"] { refusals = append(refusals, fmt.Sprintf("%s: the module's content differs from the file found; the lines above "+ "marked - are lost by taking it; `--replace %s` to replace it anyway, or declare the file partially", h.Target, h.Target)) } } // The module's secrets on the machine (rule 2 and 3): a service whose data was found already // has a value for each, so one the mesh minted and nobody accepted refuses unless --mint says // the service shall take a new one. for _, sec := range c.secrets { name := sec.Name if sec.Local != "" { name += " (" + sec.Local + ")" } what := "own secret" accept := fmt.Sprintf("`secret accept %s %s`", module, sec.Name) if !sec.Own() { what = "secret from " + sec.Provider accept = fmt.Sprintf("`secret accept %s %s --provider %s`", module, sec.Name, sec.Provider) if sec.Local != "" { accept = strings.TrimSuffix(accept, "`") + " --local " + sec.Local + "`" } } switch { case sec.Origin == inventory.OriginAccepted: fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name) case opts.Mint[sec.Name]: fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name) case foundData: fmt.Fprintf(&b, " %s %s: MINTED by the mesh and not accepted — the running service already has one\n", what, name) refusals = append(refusals, fmt.Sprintf("%s: the mesh minted a value and the service whose data was found "+ "already uses its own; %s carries the existing value in, or `--mint %s` says the service shall take "+ "the new one", name, accept, sec.Name)) default: fmt.Fprintf(&b, " %s %s: minted by the mesh\n", what, name) } } // And its settings on this machine, composed against its definition (rule 1, rule 6). for _, layer := range c.layers { keys := make([]string, 0, len(layer.Values)) for k := range layer.Values { keys = append(keys, k) } sort.Strings(keys) fmt.Fprintf(&b, " settings from %s: %s\n", layer.From, strings.Join(keys, ", ")) } if c.settingsRefused != "" { fmt.Fprintf(&b, " SETTINGS DO NOT COMPOSE with the module's definition, so the push leaves it out: %s\n", c.settingsRefused) } preview = strings.TrimRight(b.String(), "\n") sum := sha256.Sum256([]byte(preview)) return preview, refusals, hex.EncodeToString(sum[:])[:12] } // facts is a held thing's facts as the preview reads them. type facts struct { image, imageCreated, declaredImage, declaredCreated string downgrade, differs bool networks map[string][]string mounts, ports, declaredPorts, declaredVolumes []string difference []string } func factsOf(h inventory.Held) facts { var f facts if h.Facts == nil { return f } str := func(k string) string { s, _ := h.Facts[k].(string); return s } list := func(k string) []string { var out []string if raw, ok := h.Facts[k].([]any); ok { for _, x := range raw { if s, ok := x.(string); ok { out = append(out, s) } } } return out } f.image, f.imageCreated = str("image"), str("image_created") f.declaredImage, f.declaredCreated = str("declared_image"), str("declared_image_created") f.downgrade, _ = h.Facts["downgrade"].(bool) f.differs, _ = h.Facts["differs"].(bool) f.mounts, f.ports = list("mounts"), list("ports") f.declaredPorts, f.declaredVolumes, f.difference = list("declared_ports"), list("declared_volumes"), list("difference") if raw, ok := h.Facts["networks"].(map[string]any); ok { f.networks = map[string][]string{} for name, members := range raw { var out []string if ms, ok := members.([]any); ok { for _, m := range ms { if s, ok := m.(string); ok { out = append(out, s) } } } f.networks[name] = out } } return f } // comparisonLines says a held thing's facts the way a person weighs them. func comparisonLines(h inventory.Held) []string { return comparisonLinesWith(h, nil, inventory.Adoption{}) } // comparisonLinesWith is comparisonLines knowing which found networks this machine's setting keeps // for the container (rule 4) and what the machine reports reachable, so a published port's reach // is said beside the port (rule 1). func comparisonLinesWith(h inventory.Held, keeps []string, reported inventory.Adoption) []string { f := factsOf(h) var out []string if f.image != "" || f.declaredImage != "" { line := fmt.Sprintf("runs %s", orNone(f.image)) if f.imageCreated != "" { line += " (made " + day(f.imageCreated) + ")" } line += "; the module declares " + orNone(f.declaredImage) switch { case f.declaredCreated != "": line += " (made " + day(f.declaredCreated) + ")" case f.declaredImage != "": line += " (not on the machine yet, so its age is unknown)" } if f.downgrade { line += " — DOWNGRADE" } out = append(out, line) } names := make([]string, 0, len(f.networks)) for n := range f.networks { names = append(names, n) } sort.Strings(names) for _, n := range names { members := f.networks[n] if len(members) == 0 { continue } if slices.Contains(keeps, n) { out = append(out, fmt.Sprintf("on the network %s with %s — kept by this machine's setting, so they still reach it by name once taken", n, strings.Join(members, ", "))) continue } out = append(out, fmt.Sprintf("on the network %s with %s, which may reach it by name and will not once it moves to the module's own network"+ " (`settings set %s --node ` with {%q: {: [%q]}} keeps it)", n, strings.Join(members, ", "), h.Module, catalogue.NetworksSetting, n)) } for _, n := range keeps { if _, found := f.networks[n]; !found { out = append(out, fmt.Sprintf("keeps the network %s by this machine's setting, which the found container is not on", n)) } } if len(f.ports) > 0 || len(f.declaredPorts) > 0 { out = append(out, fmt.Sprintf("publishes %s; the module declares %s", orNone(strings.Join(f.ports, " ")), orNone(strings.Join(f.declaredPorts, " ")))) // How far each published port reaches now, as the machine reported it: the listener the // runtime publishes for this container. The found firewall's and the guard's rules are // not read; what they let through is said as what was reported reachable. var reach []string for _, r := range reported.Reachable { if r.By == h.Target && r.Published { reach = append(reach, fmt.Sprintf("%s:%d (%s, container port %d)", r.Address, r.Port, r.Protocol, r.ContainerPort)) } } switch { case len(reach) > 0: line := "reachable now at " + strings.Join(reach, ", ") if reported.Firewall != "" && reported.Firewall != "none" { line += ", behind the found firewall (" + reported.Firewall + "), whose rules are not read" } out = append(out, line) case len(f.ports) > 0 && len(reported.Reachable) > 0: out = append(out, "not reported reachable on the machine") } } if len(f.mounts) > 0 || len(f.declaredVolumes) > 0 { out = append(out, fmt.Sprintf("mounts %s; the module declares %s", orNone(strings.Join(f.mounts, " ")), orNone(strings.Join(f.declaredVolumes, " ")))) } if f.differs { out = append(out, "the declared content differs from the file found (- lost, + new):") for _, d := range f.difference { out = append(out, " "+d) } } return out } // day is a timestamp as a person reads it in a preview: its date. func day(stamp string) string { if len(stamp) >= 10 { return stamp[:10] } return stamp } // reportFreshFor is how old a node's account of itself may be for the flip to act on it. A // variable so a test can age a report without waiting. var reportFreshFor = 15 * time.Minute // converge previews, and with yes makes, the flip of an adopted node to converged: every module it // runs is taken, the filter module is assigned to load the mesh's derived filter in place of the // guard, and the found firewall is retired — disabled, never flushed — by the host. // // Refused while an assigned module still holds a found container: each service is taken on its // own, when its data has moved, never by the flip. And refused on a preview that would be stale: // what is reachable is the node's last account, so that account must be of what it was last sent. // // **The flip acts on the preview the operator saw** and on nothing else. The preview ends with a // short digest of what it said — every reachable thing and its fate, the modules the flip takes and // the filter — and yes must name that digest: if anything the preview would say has changed since, // the flip is refused rather than done on a preview nobody read. And it is refused on an account // older than reportFreshFor: what was reachable then is not evidence of what is reachable now. func converge(ctx context.Context, open *stores, node string, yes bool, digest string, filter string) (string, error) { inv := open.inventory if filter == "" { filter = DefaultFilter } if yes { // Held from the checks to the send, so no push composed before the flip is sent after it // and returns the node to adopted. held, release, err := holdNodes(ctx, open, []string{node}) if err != nil { return "", err } defer release() ctx = held } record, err := inv.NodeByName(ctx, node) if err != nil { return "", err } if !record.Adopted { return "", fmt.Errorf("%s is converged already; there is nothing to flip", node) } reports, err := inv.LastReports(ctx) if err != nil { return "", err } current := false for _, r := range reports { if r.Node == node { current = r.Current } } reported, err := inv.AdoptionOf(ctx, node) if err != nil { return "", err } if !current || reported.At.IsZero() { return "", fmt.Errorf("%s has not reported on the declaration it was last sent, so what it "+ "says is reachable may not be the machine as it is: run `push %s --wait 2m` and "+ "converge once it has applied", node, node) } assignedWhenPreviewed, err := inv.Assigned(ctx, node) if err != nil { return "", err } plan, settings, err := planFor(ctx, open, node) if err != nil { return "", err } runs := map[string]bool{} for _, m := range plan.Modules { runs[m.Module] = true } var holding []string for _, h := range reported.Held { if h.Kind == "container" && runs[h.Module] { holding = append(holding, fmt.Sprintf(" %s holds the found container %s — take %s %s "+ "once its data has moved", h.Module, h.Target, node, h.Module)) } } if len(holding) > 0 { sort.Strings(holding) return "", fmt.Errorf("%s still holds what it found, and a service is taken on its own, "+ "never by the flip:\n%s", node, strings.Join(holding, "\n")) } // And refused while a peer of the tunnel this hub took over has not enrolled (novox/hq ADR // 0105): the flip loads the derived filter and retires the found firewall, and a machine the // mesh has no record of is not one the filter admits — it would go dark. if _, hubName, adopted, err := inv.AdoptedTunnel(ctx); err != nil { return "", err } else if adopted && hubName == node { carried, err := inv.CarriedPeers(ctx) if err != nil { return "", err } var waiting []string for _, c := range carried { if c.EnrolledAs == "" { waiting = append(waiting, fmt.Sprintf(" %s at %s", overlay.CarriedName(c.PublicKey), c.Address)) } } if len(waiting) > 0 { return "", fmt.Errorf("%s carries peers of the tunnel it took over that have not enrolled, and "+ "converging would cut them off — enrol each first (`overlay show` says which are enrolled):\n%s", node, strings.Join(waiting, "\n")) } } shelf, err := inv.Catalogue(ctx) if err != nil { return "", err } filterModule, known := shelf[filter] if !known { return "", fmt.Errorf("%w: %s — converging assigns it to load the mesh's filter; "+ "name another with --filter", inventory.ErrNoSuchModule, filter) } if filterModule.Filtering == nil { return "", fmt.Errorf("%s loads no filter of the mesh's; name a module that does with --filter", filter) } gens, err := generators(ctx, open) if err != nil { return "", err } with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading) if err != nil { return "", err } rules, err := plan.Rules(with) if err != nil { return "", err } taken, err := inv.Taken(ctx, node) if err != nil { return "", err } derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh, outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks} filtering, err := inv.FilteringOf(ctx, node) if err != nil { return "", err } preview, saw := previewOf(node, reported, filtering, derived, plan, taken, filter, runs[filter]) preview += "\n\n preview " + saw if !yes { return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes %s` to do "+ "it.", node, saw), nil } // An account naming nothing reachable is not an account of a machine: every machine answers // on ssh, and the host's collectors failing — `ss` refusing, or the container runtime not // answering, which drops every published port at once — leaves exactly this. Flipping on it // would close ports the preview never named. if yes && countReachable(reported) == 0 { return preview, fmt.Errorf("%s says nothing is reachable on it, which no machine that is "+ "up ever is: its account looks partial — whatever reads what is listening, or what "+ "the container runtime publishes, did not answer. Fix that on the machine and run "+ "`push %s --wait 2m`, then preview again", node, node) } if age := time.Since(reported.At); age > reportFreshFor { return preview, fmt.Errorf("%s last said what is reachable on it %s ago, and the flip acts "+ "only on an account newer than %s: wait for its next report, or run `push %s --wait 2m`, "+ "then preview again", node, age.Round(time.Second), reportFreshFor, node) } if digest == "" { return preview, fmt.Errorf("converging %s acts on the preview you saw: name its digest, "+ "`converge %s --yes %s`, once you have read it", node, node, saw) } if digest != saw { return preview, fmt.Errorf("what converging %s would do has changed since preview %s "+ "(it is now %s): read the preview above, and run `converge %s --yes %s` if it is "+ "what you want", node, digest, saw, node, saw) } // The flip. The filter first, and only kept if the node still resolves with it: a node that // cannot be worked out would be sent nothing, and would sit with its guard and no filter. assigned, err := inv.Assigned(ctx, node) if err != nil { return "", err } // And nothing assigned since the preview was composed: the flip takes every module the node // runs, and one assigned in between would be taken without ever having been previewed. if !slices.Equal(assigned, assignedWhenPreviewed) { return preview, fmt.Errorf("what %s runs changed while this was converging (it is now %s): "+ "the flip takes every module on the node, so read the preview again", node, strings.Join(assigned, ", ")) } if !slices.Contains(assigned, filter) { if _, err := inv.Assign(ctx, node, filter); err != nil { return "", err } if _, _, err := planFor(ctx, open, node); err != nil { _ = inv.Unassign(ctx, node, filter) return "", fmt.Errorf("%s cannot run %s, so it was not converged: %w", node, filter, err) } } took, err := inv.Converge(ctx, node) if err != nil { return "", err } said := preview + fmt.Sprintf("\n\n%s is converged", node) if len(took) > 0 { said += "; took " + strings.Join(took, ", ") } if err := sendNodes(ctx, open, []string{node}); err != nil { return said + "\n and it could not be sent: run `push " + node + "`", err } return said + "\n sent: the host loads the mesh's filter and disables the firewall it found", nil } // previewOf is what converging a node will change, before it changes it, and a short digest of // what it said: every reachable thing and its fate, the modules the flip takes and the filter. The // digest is what the flip is asked to act on, so it changes whenever any of those would. func previewOf(node string, reported inventory.Adoption, filtering inventory.Filtering, derived derivedFilter, plan catalogue.Resolution, taken []string, filter string, filterAssigned bool) (string, string) { var said []string var b strings.Builder fmt.Fprintf(&b, "converging %s\n", node) fmt.Fprintf(&b, "\n reachable on the machine now, as it reported at %s:\n", reported.At.Local().Format(time.DateTime)) for _, r := range reported.Reachable { if loopback(r.Address) { continue } what := fmt.Sprintf("%s/%d", r.Protocol, r.Port) if r.By != "" { what += " " + r.By } if r.Published { what += fmt.Sprintf(" (published, container port %d)", r.ContainerPort) } fate := derived.fate(r) fmt.Fprintf(&b, " %-44s %s\n", what, fate) said = append(said, fmt.Sprintf("reach %s %s %s", r.Address, what, fate)) } if countReachable(reported) == 0 { // Said as what it is: no machine that is up is reachable on nothing, so this is an // account that did not come back, not a machine with nothing on it. b.WriteString(" nothing reported — this account looks partial, and the flip is " + "refused on it\n") said = append(said, "reach nothing reported") } // What the machine routes for others is not a listener and not a published port, so nothing // above can show it; the derived filter's forward chain drops it all the same. b.WriteString(" not previewed: traffic the machine routes that is not a published port " + "(a tunnel, NAT in the found firewall) — the derived filter drops it unless a module " + "declares it\n") // Which links the filter constrains, said rather than left to the sentence above (novox/hq ADR // 0140). Everything arriving anywhere else is this machine's own guest and keeps working — which // is what a reader most wants to know, because the previous shape of this filter cut a machine's // guests off at the flip without saying so, and that is how this was found. if len(derived.outwardLinks) > 0 { b.WriteString(fmt.Sprintf(" it filters what arrives on: %s, and on the private network "+ "— everything its own guests send keeps working\n", strings.Join(derived.outwardLinks, ", "))) } else { b.WriteString(" it has reported no link facing outside, so no filter can be composed " + "for it — the flip is refused until it reports one\n") } isTaken := map[string]bool{} for _, m := range taken { isTaken[m] = true } var takes []string for _, m := range plan.Modules { if !isTaken[m.Module] { takes = append(takes, m.Module) } } if !filterAssigned && !isTaken[filter] { takes = append(takes, filter) } sort.Strings(takes) b.WriteString("\n the flip takes:\n") if len(takes) == 0 { b.WriteString(" nothing — every module is taken already\n") } for _, m := range takes { fmt.Fprintf(&b, " %s\n", m) said = append(said, "take "+m) // Every kind it holds — a directory, a service, an archive, a process, a user as well as a // file (novox/hq ADR 0103) — each said, and each part of what the flip is asked to act on. for _, h := range reported.Held { if h.Module != m { continue } fmt.Fprintf(&b, " replacing the found %s", heldLine(h)) if h.Kept != "" { fmt.Fprintf(&b, ", original kept at %s", h.Kept) } b.WriteString("\n") said = append(said, "replace "+m+" "+heldLine(h)+" "+h.Kept) } } if !filterAssigned { fmt.Fprintf(&b, "\n and assigns %s, which loads the mesh's filter in place of its guard\n", filter) } fw := reported.Firewall if fw == "" || fw == "none" { b.WriteString(" no firewall was found on the machine; the mesh's filter is its first\n") } else { fmt.Fprintf(&b, " the found firewall (%s) is disabled, never flushed: its configuration stays on disk\n", fw) } said = append(said, fmt.Sprintf("filter %s assigned=%t firewall=%s", filter, filterAssigned, fw)) // What filters the machine now, and the fate of each (novox/hq ADR 0168): the found firewall // retired, the runtime's own and bans left, and what the mesh did not write left and named — // so the reader knows before the flip that the machine will not be filtered by the mesh alone. if len(filtering.Filters) > 0 { b.WriteString("\n what filters the machine now, and what the flip does to each:\n") for _, x := range filtering.Filters { fate := "left: " + x.Owner + "'s" switch x.Owner { case inventory.FilterMesh: fate = "the mesh's guard; replaced by its filter" case inventory.FilterFoundFirewall: fate = "the found firewall's; retired with it" case inventory.FilterRuntime: fate = "the container runtime's own; left" case inventory.FilterBan: fate = "a ban list; left" case inventory.FilterOther: fate = "NOT THE MESH'S; left in force — the machine is not filtered by the mesh alone until you remove it" } fmt.Fprintf(&b, " %-50s %s\n", x.Where, fate) fmt.Fprintf(&b, " %-50s %s\n", "", x.Refuses) said = append(said, "filter "+x.Owner+" "+x.Where) } } // Sorted: the same account, reported in another order, is the same preview. sort.Strings(said) sum := sha256.Sum256([]byte(strings.Join(said, "\n"))) return strings.TrimRight(b.String(), "\n"), hex.EncodeToString(sum[:])[:12] } // derivedFilter is what the filter the flip loads is rendered from, as AsNftables renders it. type derivedFilter struct { rules []catalogue.Rule foundation []int // mesh is every address on the private network; outward says the machine faces outside. mesh []string outward bool // outwardLinks is the links this machine reported as facing outside it (novox/hq ADR 0140). // The filter constrains what arrives on them; everything arriving elsewhere is this machine's // own guest and is not filtered. outwardLinks []string } // closesOutside is what a narrowing from everywhere to the private network is called: it closes. const closesOutside = "WILL CLOSE to everything outside the private network" // fate is what the derived filter does to one reachable thing: which module declares it and from // where, or that it will close — wholly, or to everything outside the private network. Rendered // exactly as AsNftables admits it, ssh included. func (d derivedFilter) fate(r inventory.Reach) string { // Bound to an address on the private network, it was never reachable from outside it, so // admitting it from the mesh narrows nothing. onMesh := slices.Contains(d.mesh, strings.Trim(r.Address, "[]")) if r.Protocol == "tcp" && r.Port == catalogue.SSHPort { // From everywhere only when the machine faces outward or the mesh has no addresses to // narrow it to; otherwise from the private network only. if d.outward || len(d.mesh) == 0 || onMesh { return "stays open — ssh is never closed" } return closesOutside + " — ssh stays open from the mesh, never closed there" } for _, port := range d.foundation { if r.Protocol == "tcp" && r.Port == port { return "stays open — the mesh's own, from anywhere" } } // This machine's own guests ask it for an address and for names, and those two arrive here // (novox/hq ADR 0140). Admitted by the link they arrive on, so a listener bound anywhere but an // outward link keeps answering them. if (r.Protocol == "udp" && (r.Port == 53 || r.Port == 67)) || (r.Protocol == "tcp" && r.Port == 53) { return "stays open — this machine's own guests asking it for an address and for names" } for _, rule := range d.rules { if rule.Port != r.Port || rule.Protocol != r.Protocol { continue } by := strings.Join(rule.Because, ", ") switch rule.From { case catalogue.FromMachine: return fmt.Sprintf("WILL CLOSE to the network — declared by %s for this machine only", by) case catalogue.FromMesh: if len(d.mesh) == 0 { return fmt.Sprintf("WILL CLOSE — declared by %s from the mesh, and this node "+ "knows no mesh addresses", by) } if !onMesh { return fmt.Sprintf("%s — declared by %s from the mesh only", closesOutside, by) } } return fmt.Sprintf("declared by %s (from %s)", by, rule.From) } return "WILL CLOSE — no module assigned here declares it" } // countReachable is how much of a node's account of itself names something off the machine. // Loopback is left out for the same reason the preview leaves it out: nothing outside reaches it, // so a report of loopback alone says nothing about what the filter would close. func countReachable(reported inventory.Adoption) int { n := 0 for _, r := range reported.Reachable { if !loopback(r.Address) { n++ } } return n } // heldLine is one thing a node holds as found, as take and the converge preview both say it. func heldLine(h inventory.Held) string { return fmt.Sprintf("%s %s (%s)", h.Kind, h.Target, h.ID) } // loopback is an address nothing off the machine reaches. func loopback(address string) bool { a := strings.Trim(address, "[]") return strings.HasPrefix(a, "127.") || a == "::1" || a == "localhost" } // adopt returns a converged node to adopted: the mesh's filter is unloaded, the guard restored, // the found firewall enabled again and the openings converged through it once more. What was // taken stays taken. func adopt(ctx context.Context, open *stores, node string) (string, error) { inv := open.inventory record, err := inv.NodeByName(ctx, node) if err != nil { return "", err } if record.Adopted { return "", fmt.Errorf("%s is adopted already", node) } held, release, err := holdNodes(ctx, open, []string{node}) if err != nil { return "", err } defer release() ctx = held if err := inv.SetAdopted(ctx, node, true); err != nil { return "", err } said := fmt.Sprintf("%s is adopted; what was taken on it stays taken", node) if err := sendNodes(ctx, open, []string{node}); err != nil { return said + "\n and it could not be sent: run `push " + node + "`", err } return said + "\n sent: the host unloads the mesh's filter and enables the firewall it found", nil } // takeCommand, convergeCommand and adoptCommand are the command line's adapters to the acts above. func takeCommand(ctx context.Context, args []string) error { set := flag.NewFlagSet("take", flag.ContinueOnError) yes := set.Bool("yes", false, "cut over as previewed, naming the digest the preview printed after it; "+ "without it the comparison is printed and nothing is taken") downgrade := set.Bool("downgrade", false, "take it although the module's image is older than the one running") var replace, mint stringList set.Var(&replace, "replace", "a found file's path whose content the module may replace although it differs (repeatable; * for every one)") set.Var(&mint, "mint", "a secret the service shall take the mesh's minted value for, although it already has its own (repeatable)") positionals, err := parseAround(set, args) if err != nil { return err } if len(positionals) < 2 || len(positionals) > 3 || (len(positionals) == 3 && !*yes) { return errors.New("take [--yes ] [--downgrade] [--replace ]... [--mint ]...") } opts := takeOptions{Yes: *yes, Downgrade: *downgrade, Replace: map[string]bool{}, Mint: map[string]bool{}} if len(positionals) == 3 { opts.Digest = positionals[2] } for _, r := range replace { opts.Replace[r] = true } for _, m := range mint { opts.Mint[m] = true } return runAct(ctx, func(open *stores) (string, error) { return take(ctx, open, positionals[0], positionals[1], opts) }) } // stringList is a repeatable flag. type stringList []string func (l *stringList) String() string { return strings.Join(*l, ",") } func (l *stringList) Set(v string) error { *l = append(*l, v); return nil } func convergeCommand(ctx context.Context, args []string) error { set := flag.NewFlagSet("converge", flag.ContinueOnError) yes := set.String("yes", "", "do it, naming the digest the preview printed; without it, only "+ "the preview") filter := set.String("filter", DefaultFilter, "the module that loads the mesh's filter") positionals, err := parseAround(set, args) if err != nil { return err } if len(positionals) != 1 { return errors.New("converge [--yes ] [--filter nftables]") } return runAct(ctx, func(open *stores) (string, error) { return converge(ctx, open, positionals[0], *yes != "", *yes, *filter) }) } func adoptCommand(ctx context.Context, args []string) error { if len(args) != 1 { return errors.New("adopt ") } return runAct(ctx, func(open *stores) (string, error) { return adopt(ctx, open, args[0]) }) } func runAct(ctx context.Context, act func(*stores) (string, error)) error { open, err := openStores(ctx) if err != nil { return err } defer open.Close() said, err := act(open) if said != "" { fmt.Println(said) } if err != nil && said != "" { fmt.Println() } return err }