jschoubben
fbc3d320ea
take ends its preview with a digest and --yes names it, as the flip does; a changed preview or an account older than the flip allows is refused. A module the machine holds nothing for has nothing to compare, and --yes suffices. A published port's reach is said as the machine reported it. Every secret the module holds on the machine is listed with where it came from, and one the mesh minted for a service whose data was found refuses unless --mint names it. One judgement of a module's settings against its definition, in the catalogue: settings set refuses what cannot compose or reaches nothing, naming node, module, layer and key; Compose leaves out a module whose definition moved under a stored setting, the envelope says so (left_out), plan and push say it by name, and the machine is told everything else. A stray setting no longer refuses the whole machine where it is read (issue 096). The per-machine setting networks keeps a found network for a taken container, on an adopted machine only; the container's declaration carries it and the preview names it (rule 4).
1022 lines
39 KiB
Go
1022 lines
39 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/overlay"
|
|
)
|
|
|
|
// A node is adopted or converged (novox/hq ADR 0100), and it is said to be adopted wherever the
|
|
// mesh reports a node's state: node list, node show, status and the board.
|
|
|
|
// showMode is the node show lines about a node's mode and what was taken on it.
|
|
func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node) error {
|
|
if !node.Adopted {
|
|
fmt.Printf(" mode converged\n")
|
|
// A converged machine holds nothing, and can still run what nobody asked for
|
|
// (novox/hq ADR 0163): what it reports as strays is said whatever its mode.
|
|
if said, err := inv.AdoptionOf(ctx, node.Name); err == nil && len(said.Strays) > 0 {
|
|
showStrays(said.Strays)
|
|
}
|
|
return nil
|
|
}
|
|
fmt.Printf(" mode adopted since %s\n",
|
|
node.AdoptedSince.Local().Format(time.DateTime))
|
|
taken, err := inv.Taken(ctx, node.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(taken) == 0 {
|
|
fmt.Printf(" taken nothing yet\n")
|
|
} else {
|
|
fmt.Printf(" taken %s\n", strings.Join(taken, ", "))
|
|
}
|
|
said, err := inv.AdoptionOf(ctx, node.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if said.At.IsZero() {
|
|
fmt.Printf(" it has not yet said what it found\n")
|
|
return nil
|
|
}
|
|
fmt.Printf(" firewall found %s\n", orNone(said.Firewall))
|
|
if err := showTunnel(ctx, inv, node.Name); err != nil {
|
|
return err
|
|
}
|
|
if len(said.Held) == 0 {
|
|
fmt.Printf(" holding nothing found\n")
|
|
}
|
|
for _, h := range said.Held {
|
|
// A held thing that changed is how a predecessor still writing is caught: said first.
|
|
line := fmt.Sprintf(" holds %-11s %s %s, for %s", h.Kind, h.Target, h.ID, h.Module)
|
|
if h.Changed != "" {
|
|
line += " — " + strings.ToUpper(h.Changed) + " by something other than the mesh"
|
|
}
|
|
fmt.Println(line)
|
|
if h.Kept != "" {
|
|
fmt.Printf(" %-17s original kept at %s\n", "", h.Kept)
|
|
}
|
|
for _, f := range comparisonLines(h) {
|
|
fmt.Printf(" %-17s %s\n", "", f)
|
|
}
|
|
}
|
|
showStrays(said.Strays)
|
|
fmt.Printf(" as of %s\n", said.At.Local().Format(time.DateTime))
|
|
return nil
|
|
}
|
|
|
|
// showStrays says what a machine runs that the mesh neither wrote nor holds (ADR 0163).
|
|
func showStrays(strays []inventory.Stray) {
|
|
if len(strays) == 0 {
|
|
return
|
|
}
|
|
fmt.Printf(" strays %d container(s) the mesh neither wrote nor holds:\n", len(strays))
|
|
for _, s := range strays {
|
|
fmt.Printf(" %-17s %s (%s)\n", "", s.Name, s.Detail)
|
|
}
|
|
}
|
|
|
|
// showTunnel is the node show lines about the tunnel an adopted node found and carried (novox/hq
|
|
// ADR 0105): what it presented at enrolment, and what it last said about taking it over.
|
|
func showTunnel(ctx context.Context, inv *inventory.Inventory, name string) error {
|
|
tunnel, err := inv.TunnelOf(ctx, name)
|
|
if errors.Is(err, inventory.ErrNoTunnel) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf(" tunnel found %s on port %d, %s in %s, %d peer(s)\n",
|
|
tunnel.Interface, tunnel.Port, tunnel.Address, tunnel.Range, len(tunnel.Peers))
|
|
carried, said, err := inv.CarriedTunnelOf(ctx, name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch {
|
|
case !said:
|
|
fmt.Printf(" %-17s not yet taken over — the node has not said so\n", "")
|
|
case carried.State == inventory.CarriedTaken:
|
|
fmt.Printf(" %-17s taken over: %s is down and disabled, never flushed; the mesh's interface "+
|
|
"runs with its key, port and %d peer(s)\n", "", carried.Interface, carried.Peers)
|
|
case carried.State == inventory.CarriedDown:
|
|
fmt.Printf(" %-17s TUNNEL DOWN: %s is stopped and the mesh's interface is not up — the peers "+
|
|
"reach nothing. On the machine: systemctl start %s\n", "", carried.Interface,
|
|
"wg-quick@"+carried.Interface)
|
|
default:
|
|
fmt.Printf(" %-17s NOT taken over: %s is still the interface the peers reach\n", "", carried.Interface)
|
|
}
|
|
if said && carried.Note != "" {
|
|
fmt.Printf(" %-17s %s\n", "", carried.Note)
|
|
}
|
|
if said && carried.Kept != "" {
|
|
fmt.Printf(" %-17s its configuration's original kept at %s\n", "", carried.Kept)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func orNone(s string) string {
|
|
if s == "" {
|
|
return "none reported"
|
|
}
|
|
return s
|
|
}
|
|
|
|
// adoptedNodes are the names of every adopted node, in the order given.
|
|
func adoptedNodes(nodes []inventory.Node) []string {
|
|
var out []string
|
|
for _, n := range nodes {
|
|
if n.Adopted {
|
|
out = append(out, n.Name)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// The operator's acts on an adopted node (novox/hq ADR 0100). Called by the command line and the
|
|
// command API alike, so a refusal is the same refusal in the same words at both (ADR 0035).
|
|
|
|
// sendNodes sends the named machines what they should be now. A variable so a test can see what
|
|
// an act would send without a broker.
|
|
var sendNodes = sendTo
|
|
|
|
// DefaultFilter is the module converging a node assigns to load the mesh's derived filter.
|
|
const DefaultFilter = "nftables"
|
|
|
|
// take is a module's cutover on an adopted node: the operator's act, done when that module's data
|
|
// has moved. From the next push its resources converge there like any other, replacing what the
|
|
// node found and holds for it.
|
|
//
|
|
// **Previewed, and the preview is a comparison** (novox/hq ADR 0163): for every held thing the
|
|
// module would replace, what runs beside what the module declares, and the difference; the
|
|
// module's secrets on the machine and where each came from; its settings on the machine. Without
|
|
// --yes the comparison is printed and nothing changes. `--yes <digest>` cuts over exactly what was
|
|
// previewed, the way the flip is confirmed: the preview ends with a digest of what it said, and a
|
|
// take naming an older one, or acting on an account of the machine older than the flip allows, is
|
|
// refused. A module the machine holds nothing for has nothing to compare, and `--yes` suffices.
|
|
// takeOptions is what a take was told about the differences it may pass (novox/hq ADR 0163).
|
|
type takeOptions struct {
|
|
Yes bool
|
|
// Digest is the preview's, named with --yes; required whenever the machine holds something
|
|
// for the module.
|
|
Digest string
|
|
Downgrade bool
|
|
Replace map[string]bool
|
|
// Mint names the secrets the service shall take a new value for, although the mesh minted
|
|
// one and the service already has its own (rule 2).
|
|
Mint map[string]bool
|
|
}
|
|
|
|
func take(ctx context.Context, open *stores, node, module string, opts takeOptions) (string, error) {
|
|
inv := open.inventory
|
|
assigned, err := inv.Assigned(ctx, node)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if !slices.Contains(assigned, module) {
|
|
if plan, _, err := planFor(ctx, open, node); err == nil {
|
|
if why, runs := plan.Because[module]; runs {
|
|
return "", fmt.Errorf("%w: %s runs on %s because %s — assign it to %s to take it",
|
|
inventory.ErrNotAssigned, module, node, why, node)
|
|
}
|
|
}
|
|
}
|
|
// The comparison first (novox/hq ADR 0163): every held thing the module would replace, beside
|
|
// what the module declares, and the differences that refuse unless named.
|
|
c, err := comparisonFor(ctx, open, node, module)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
preview, refusals, saw := comparisonOf(module, c, opts)
|
|
if len(refusals) > 0 {
|
|
return "", fmt.Errorf("taking %s on %s is refused:\n %s\n%s", module, node,
|
|
strings.Join(refusals, "\n "), preview)
|
|
}
|
|
holds := len(heldOf(c.reported, module)) > 0
|
|
if holds {
|
|
preview += "\n preview " + saw
|
|
}
|
|
if !opts.Yes {
|
|
if !holds {
|
|
return preview + fmt.Sprintf("\nnothing taken; `take %s %s --yes` declares it as the mesh's own", node, module), nil
|
|
}
|
|
return preview + fmt.Sprintf("\nnothing taken; `take %s %s --yes %s` cuts it over as previewed", node, module, saw), nil
|
|
}
|
|
if holds {
|
|
// The take acts on the preview the operator saw, and on an account of the machine that
|
|
// is still the machine: the same two refusals the flip makes.
|
|
if age := time.Since(c.reported.At); age > reportFreshFor {
|
|
return preview, fmt.Errorf("%s last said what it holds %s ago, and a take acts only on "+
|
|
"an account newer than %s: run `push %s --wait 2m`, then preview again",
|
|
node, age.Round(time.Second), reportFreshFor, node)
|
|
}
|
|
if opts.Digest == "" {
|
|
return preview, fmt.Errorf("taking %s on %s acts on the preview you saw: name its digest, "+
|
|
"`take %s %s --yes %s`, once you have read it", module, node, node, module, saw)
|
|
}
|
|
if opts.Digest != saw {
|
|
return preview, fmt.Errorf("what taking %s on %s would replace has changed since preview %s "+
|
|
"(it is now %s): read the preview above, and run `take %s %s --yes %s` if it is "+
|
|
"what you want", module, node, opts.Digest, saw, node, module, saw)
|
|
}
|
|
}
|
|
if err := inv.Take(ctx, node, module); err != nil {
|
|
return "", err
|
|
}
|
|
said := fmt.Sprintf("%s is taken on %s", module, node)
|
|
if holds {
|
|
said += "; the next push replaces what the node found and holds for it:\n" + preview
|
|
}
|
|
return said + fmt.Sprintf("\n run `push %s` to cut it over", node), nil
|
|
}
|
|
|
|
// comparison is everything a take puts beside what the module declares: the machine's account of
|
|
// what it holds and what is reachable on it, the module's secrets on the machine, its settings
|
|
// there, and which found networks a setting keeps for each of its containers (by held id).
|
|
type comparison struct {
|
|
reported inventory.Adoption
|
|
secrets []inventory.SecretState
|
|
layers []catalogue.Layer
|
|
keeps map[string][]string
|
|
// settingsRefused is why the module's settings cannot compose with its definition, when
|
|
// they cannot — the module would be left out of the declaration (rule 6).
|
|
settingsRefused string
|
|
}
|
|
|
|
func comparisonFor(ctx context.Context, open *stores, node, module string) (comparison, error) {
|
|
inv := open.inventory
|
|
var c comparison
|
|
var err error
|
|
if c.reported, err = inv.AdoptionOf(ctx, node); err != nil {
|
|
return c, err
|
|
}
|
|
if c.secrets, err = inv.SecretsOf(ctx, node, module); err != nil {
|
|
return c, err
|
|
}
|
|
if c.layers, err = inv.SettingsFor(ctx, node, module); err != nil {
|
|
return c, err
|
|
}
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return c, err
|
|
}
|
|
if m, known := shelf[module]; known && len(c.layers) > 0 {
|
|
if err := catalogue.JudgeSettings(m, c.layers, true); err != nil {
|
|
c.settingsRefused = err.Error()
|
|
}
|
|
if kept, err := catalogue.KeptNetworks(m, c.layers, true); err == nil && len(kept) > 0 {
|
|
c.keeps = map[string][]string{}
|
|
for id, networks := range kept {
|
|
c.keeps[module+"."+id] = networks
|
|
}
|
|
}
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
// heldOf is what a node holds for one module.
|
|
func heldOf(reported inventory.Adoption, module string) []inventory.Held {
|
|
var out []inventory.Held
|
|
for _, h := range reported.Held {
|
|
if h.Module == module {
|
|
out = append(out, h)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// comparisonOf is a take's preview: for every held thing of the module, what runs beside what the
|
|
// module declares; its secrets and its settings on the machine; and the refusals the differences
|
|
// earn unless the take named them (novox/hq ADR 0163): an image older than the one running, a
|
|
// declared file that differs from the found one, a secret the mesh minted for a service whose data
|
|
// was found. A narrowed port and a shared network are said and not refused. The digest is of what
|
|
// the preview says, so anything in it changing changes the digest.
|
|
func comparisonOf(module string, c comparison, opts takeOptions) (preview string, refusals []string, digest string) {
|
|
var b strings.Builder
|
|
held := heldOf(c.reported, module)
|
|
foundData := false
|
|
for _, h := range held {
|
|
if h.Kind == "container" || h.Kind == "directory" {
|
|
foundData = true
|
|
}
|
|
fmt.Fprintf(&b, " %s", heldLine(h))
|
|
if h.Kept != "" {
|
|
fmt.Fprintf(&b, ", original kept at %s", h.Kept)
|
|
}
|
|
b.WriteString("\n")
|
|
for _, line := range comparisonLinesWith(h, c.keeps[h.ID], c.reported) {
|
|
fmt.Fprintf(&b, " %s\n", line)
|
|
}
|
|
f := factsOf(h)
|
|
if f.downgrade && !opts.Downgrade {
|
|
refusals = append(refusals, fmt.Sprintf("%s: the module's image (%s, made %s) is older than the one running (%s, made %s) — "+
|
|
"a service that migrated its data forward may not start on it; `--downgrade` to take it anyway",
|
|
h.Target, f.declaredImage, day(f.declaredCreated), f.image, day(f.imageCreated)))
|
|
}
|
|
if f.differs && !opts.Replace[h.Target] && !opts.Replace["*"] {
|
|
refusals = append(refusals, fmt.Sprintf("%s: the module's content differs from the file found; the lines above "+
|
|
"marked - are lost by taking it; `--replace %s` to replace it anyway, or declare the file partially",
|
|
h.Target, h.Target))
|
|
}
|
|
}
|
|
// The module's secrets on the machine (rule 2 and 3): a service whose data was found already
|
|
// has a value for each, so one the mesh minted and nobody accepted refuses unless --mint says
|
|
// the service shall take a new one.
|
|
for _, sec := range c.secrets {
|
|
name := sec.Name
|
|
if sec.Local != "" {
|
|
name += " (" + sec.Local + ")"
|
|
}
|
|
what := "own secret"
|
|
accept := fmt.Sprintf("`secret accept <node> %s %s`", module, sec.Name)
|
|
if !sec.Own() {
|
|
what = "secret from " + sec.Provider
|
|
accept = fmt.Sprintf("`secret accept <node> %s %s --provider %s`", module, sec.Name, sec.Provider)
|
|
if sec.Local != "" {
|
|
accept = strings.TrimSuffix(accept, "`") + " --local " + sec.Local + "`"
|
|
}
|
|
}
|
|
switch {
|
|
case sec.Origin == inventory.OriginAccepted:
|
|
fmt.Fprintf(&b, " %s %s: accepted from a person, carried in as it is\n", what, name)
|
|
case opts.Mint[sec.Name]:
|
|
fmt.Fprintf(&b, " %s %s: minted by the mesh; the service takes the new value, as --mint said\n", what, name)
|
|
case foundData:
|
|
fmt.Fprintf(&b, " %s %s: MINTED by the mesh and not accepted — the running service already has one\n", what, name)
|
|
refusals = append(refusals, fmt.Sprintf("%s: the mesh minted a value and the service whose data was found "+
|
|
"already uses its own; %s carries the existing value in, or `--mint %s` says the service shall take "+
|
|
"the new one", name, accept, sec.Name))
|
|
default:
|
|
fmt.Fprintf(&b, " %s %s: minted by the mesh\n", what, name)
|
|
}
|
|
}
|
|
// And its settings on this machine, composed against its definition (rule 1, rule 6).
|
|
for _, layer := range c.layers {
|
|
keys := make([]string, 0, len(layer.Values))
|
|
for k := range layer.Values {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
fmt.Fprintf(&b, " settings from %s: %s\n", layer.From, strings.Join(keys, ", "))
|
|
}
|
|
if c.settingsRefused != "" {
|
|
fmt.Fprintf(&b, " SETTINGS DO NOT COMPOSE with the module's definition, so the push leaves it out: %s\n", c.settingsRefused)
|
|
}
|
|
preview = strings.TrimRight(b.String(), "\n")
|
|
sum := sha256.Sum256([]byte(preview))
|
|
return preview, refusals, hex.EncodeToString(sum[:])[:12]
|
|
}
|
|
|
|
// facts is a held thing's facts as the preview reads them.
|
|
type facts struct {
|
|
image, imageCreated, declaredImage, declaredCreated string
|
|
downgrade, differs bool
|
|
networks map[string][]string
|
|
mounts, ports, declaredPorts, declaredVolumes []string
|
|
difference []string
|
|
}
|
|
|
|
func factsOf(h inventory.Held) facts {
|
|
var f facts
|
|
if h.Facts == nil {
|
|
return f
|
|
}
|
|
str := func(k string) string { s, _ := h.Facts[k].(string); return s }
|
|
list := func(k string) []string {
|
|
var out []string
|
|
if raw, ok := h.Facts[k].([]any); ok {
|
|
for _, x := range raw {
|
|
if s, ok := x.(string); ok {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
f.image, f.imageCreated = str("image"), str("image_created")
|
|
f.declaredImage, f.declaredCreated = str("declared_image"), str("declared_image_created")
|
|
f.downgrade, _ = h.Facts["downgrade"].(bool)
|
|
f.differs, _ = h.Facts["differs"].(bool)
|
|
f.mounts, f.ports = list("mounts"), list("ports")
|
|
f.declaredPorts, f.declaredVolumes, f.difference = list("declared_ports"), list("declared_volumes"), list("difference")
|
|
if raw, ok := h.Facts["networks"].(map[string]any); ok {
|
|
f.networks = map[string][]string{}
|
|
for name, members := range raw {
|
|
var out []string
|
|
if ms, ok := members.([]any); ok {
|
|
for _, m := range ms {
|
|
if s, ok := m.(string); ok {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
}
|
|
f.networks[name] = out
|
|
}
|
|
}
|
|
return f
|
|
}
|
|
|
|
// comparisonLines says a held thing's facts the way a person weighs them.
|
|
func comparisonLines(h inventory.Held) []string {
|
|
return comparisonLinesWith(h, nil, inventory.Adoption{})
|
|
}
|
|
|
|
// comparisonLinesWith is comparisonLines knowing which found networks this machine's setting keeps
|
|
// for the container (rule 4) and what the machine reports reachable, so a published port's reach
|
|
// is said beside the port (rule 1).
|
|
func comparisonLinesWith(h inventory.Held, keeps []string, reported inventory.Adoption) []string {
|
|
f := factsOf(h)
|
|
var out []string
|
|
if f.image != "" || f.declaredImage != "" {
|
|
line := fmt.Sprintf("runs %s", orNone(f.image))
|
|
if f.imageCreated != "" {
|
|
line += " (made " + day(f.imageCreated) + ")"
|
|
}
|
|
line += "; the module declares " + orNone(f.declaredImage)
|
|
switch {
|
|
case f.declaredCreated != "":
|
|
line += " (made " + day(f.declaredCreated) + ")"
|
|
case f.declaredImage != "":
|
|
line += " (not on the machine yet, so its age is unknown)"
|
|
}
|
|
if f.downgrade {
|
|
line += " — DOWNGRADE"
|
|
}
|
|
out = append(out, line)
|
|
}
|
|
names := make([]string, 0, len(f.networks))
|
|
for n := range f.networks {
|
|
names = append(names, n)
|
|
}
|
|
sort.Strings(names)
|
|
for _, n := range names {
|
|
members := f.networks[n]
|
|
if len(members) == 0 {
|
|
continue
|
|
}
|
|
if slices.Contains(keeps, n) {
|
|
out = append(out, fmt.Sprintf("on the network %s with %s — kept by this machine's setting, so they still reach it by name once taken",
|
|
n, strings.Join(members, ", ")))
|
|
continue
|
|
}
|
|
out = append(out, fmt.Sprintf("on the network %s with %s, which may reach it by name and will not once it moves to the module's own network"+
|
|
" (`settings set %s --node <node>` with {%q: {<container>: [%q]}} keeps it)",
|
|
n, strings.Join(members, ", "), h.Module, catalogue.NetworksSetting, n))
|
|
}
|
|
for _, n := range keeps {
|
|
if _, found := f.networks[n]; !found {
|
|
out = append(out, fmt.Sprintf("keeps the network %s by this machine's setting, which the found container is not on", n))
|
|
}
|
|
}
|
|
if len(f.ports) > 0 || len(f.declaredPorts) > 0 {
|
|
out = append(out, fmt.Sprintf("publishes %s; the module declares %s",
|
|
orNone(strings.Join(f.ports, " ")), orNone(strings.Join(f.declaredPorts, " "))))
|
|
// How far each published port reaches now, as the machine reported it: the listener the
|
|
// runtime publishes for this container. The found firewall's and the guard's rules are
|
|
// not read; what they let through is said as what was reported reachable.
|
|
var reach []string
|
|
for _, r := range reported.Reachable {
|
|
if r.By == h.Target && r.Published {
|
|
reach = append(reach, fmt.Sprintf("%s:%d (%s, container port %d)", r.Address, r.Port, r.Protocol, r.ContainerPort))
|
|
}
|
|
}
|
|
switch {
|
|
case len(reach) > 0:
|
|
line := "reachable now at " + strings.Join(reach, ", ")
|
|
if reported.Firewall != "" && reported.Firewall != "none" {
|
|
line += ", behind the found firewall (" + reported.Firewall + "), whose rules are not read"
|
|
}
|
|
out = append(out, line)
|
|
case len(f.ports) > 0 && len(reported.Reachable) > 0:
|
|
out = append(out, "not reported reachable on the machine")
|
|
}
|
|
}
|
|
if len(f.mounts) > 0 || len(f.declaredVolumes) > 0 {
|
|
out = append(out, fmt.Sprintf("mounts %s; the module declares %s",
|
|
orNone(strings.Join(f.mounts, " ")), orNone(strings.Join(f.declaredVolumes, " "))))
|
|
}
|
|
if f.differs {
|
|
out = append(out, "the declared content differs from the file found (- lost, + new):")
|
|
for _, d := range f.difference {
|
|
out = append(out, " "+d)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// day is a timestamp as a person reads it in a preview: its date.
|
|
func day(stamp string) string {
|
|
if len(stamp) >= 10 {
|
|
return stamp[:10]
|
|
}
|
|
return stamp
|
|
}
|
|
|
|
// reportFreshFor is how old a node's account of itself may be for the flip to act on it. A
|
|
// variable so a test can age a report without waiting.
|
|
var reportFreshFor = 15 * time.Minute
|
|
|
|
// converge previews, and with yes makes, the flip of an adopted node to converged: every module it
|
|
// runs is taken, the filter module is assigned to load the mesh's derived filter in place of the
|
|
// guard, and the found firewall is retired — disabled, never flushed — by the host.
|
|
//
|
|
// Refused while an assigned module still holds a found container: each service is taken on its
|
|
// own, when its data has moved, never by the flip. And refused on a preview that would be stale:
|
|
// what is reachable is the node's last account, so that account must be of what it was last sent.
|
|
//
|
|
// **The flip acts on the preview the operator saw** and on nothing else. The preview ends with a
|
|
// short digest of what it said — every reachable thing and its fate, the modules the flip takes and
|
|
// the filter — and yes must name that digest: if anything the preview would say has changed since,
|
|
// the flip is refused rather than done on a preview nobody read. And it is refused on an account
|
|
// older than reportFreshFor: what was reachable then is not evidence of what is reachable now.
|
|
func converge(ctx context.Context, open *stores, node string, yes bool, digest string,
|
|
filter string) (string, error) {
|
|
inv := open.inventory
|
|
if filter == "" {
|
|
filter = DefaultFilter
|
|
}
|
|
if yes {
|
|
// Held from the checks to the send, so no push composed before the flip is sent after it
|
|
// and returns the node to adopted.
|
|
held, release, err := holdNodes(ctx, open, []string{node})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer release()
|
|
ctx = held
|
|
}
|
|
record, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if !record.Adopted {
|
|
return "", fmt.Errorf("%s is converged already; there is nothing to flip", node)
|
|
}
|
|
reports, err := inv.LastReports(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
current := false
|
|
for _, r := range reports {
|
|
if r.Node == node {
|
|
current = r.Current
|
|
}
|
|
}
|
|
reported, err := inv.AdoptionOf(ctx, node)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if !current || reported.At.IsZero() {
|
|
return "", fmt.Errorf("%s has not reported on the declaration it was last sent, so what it "+
|
|
"says is reachable may not be the machine as it is: run `push %s --wait 2m` and "+
|
|
"converge once it has applied", node, node)
|
|
}
|
|
|
|
assignedWhenPreviewed, err := inv.Assigned(ctx, node)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
plan, settings, err := planFor(ctx, open, node)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
runs := map[string]bool{}
|
|
for _, m := range plan.Modules {
|
|
runs[m.Module] = true
|
|
}
|
|
var holding []string
|
|
for _, h := range reported.Held {
|
|
if h.Kind == "container" && runs[h.Module] {
|
|
holding = append(holding, fmt.Sprintf(" %s holds the found container %s — take %s %s "+
|
|
"once its data has moved", h.Module, h.Target, node, h.Module))
|
|
}
|
|
}
|
|
if len(holding) > 0 {
|
|
sort.Strings(holding)
|
|
return "", fmt.Errorf("%s still holds what it found, and a service is taken on its own, "+
|
|
"never by the flip:\n%s", node, strings.Join(holding, "\n"))
|
|
}
|
|
// And refused while a peer of the tunnel this hub took over has not enrolled (novox/hq ADR
|
|
// 0105): the flip loads the derived filter and retires the found firewall, and a machine the
|
|
// mesh has no record of is not one the filter admits — it would go dark.
|
|
if _, hubName, adopted, err := inv.AdoptedTunnel(ctx); err != nil {
|
|
return "", err
|
|
} else if adopted && hubName == node {
|
|
carried, err := inv.CarriedPeers(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
var waiting []string
|
|
for _, c := range carried {
|
|
if c.EnrolledAs == "" {
|
|
waiting = append(waiting, fmt.Sprintf(" %s at %s", overlay.CarriedName(c.PublicKey), c.Address))
|
|
}
|
|
}
|
|
if len(waiting) > 0 {
|
|
return "", fmt.Errorf("%s carries peers of the tunnel it took over that have not enrolled, and "+
|
|
"converging would cut them off — enrol each first (`overlay show` says which are enrolled):\n%s",
|
|
node, strings.Join(waiting, "\n"))
|
|
}
|
|
}
|
|
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
filterModule, known := shelf[filter]
|
|
if !known {
|
|
return "", fmt.Errorf("%w: %s — converging assigns it to load the mesh's filter; "+
|
|
"name another with --filter", inventory.ErrNoSuchModule, filter)
|
|
}
|
|
if filterModule.Filtering == nil {
|
|
return "", fmt.Errorf("%s loads no filter of the mesh's; name a module that does with --filter",
|
|
filter)
|
|
}
|
|
|
|
gens, err := generators(ctx, open)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
rules, err := plan.Rules(with)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
taken, err := inv.Taken(ctx, node)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
|
|
outward: plan.PublicDomain != "", outwardLinks: with.OutwardLinks}
|
|
preview, saw := previewOf(node, reported, derived, plan, taken, filter, runs[filter])
|
|
preview += "\n\n preview " + saw
|
|
if !yes {
|
|
return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes %s` to do "+
|
|
"it.", node, saw), nil
|
|
}
|
|
// An account naming nothing reachable is not an account of a machine: every machine answers
|
|
// on ssh, and the host's collectors failing — `ss` refusing, or the container runtime not
|
|
// answering, which drops every published port at once — leaves exactly this. Flipping on it
|
|
// would close ports the preview never named.
|
|
if yes && countReachable(reported) == 0 {
|
|
return preview, fmt.Errorf("%s says nothing is reachable on it, which no machine that is "+
|
|
"up ever is: its account looks partial — whatever reads what is listening, or what "+
|
|
"the container runtime publishes, did not answer. Fix that on the machine and run "+
|
|
"`push %s --wait 2m`, then preview again", node, node)
|
|
}
|
|
if age := time.Since(reported.At); age > reportFreshFor {
|
|
return preview, fmt.Errorf("%s last said what is reachable on it %s ago, and the flip acts "+
|
|
"only on an account newer than %s: wait for its next report, or run `push %s --wait 2m`, "+
|
|
"then preview again", node, age.Round(time.Second), reportFreshFor, node)
|
|
}
|
|
if digest == "" {
|
|
return preview, fmt.Errorf("converging %s acts on the preview you saw: name its digest, "+
|
|
"`converge %s --yes %s`, once you have read it", node, node, saw)
|
|
}
|
|
if digest != saw {
|
|
return preview, fmt.Errorf("what converging %s would do has changed since preview %s "+
|
|
"(it is now %s): read the preview above, and run `converge %s --yes %s` if it is "+
|
|
"what you want", node, digest, saw, node, saw)
|
|
}
|
|
|
|
// The flip. The filter first, and only kept if the node still resolves with it: a node that
|
|
// cannot be worked out would be sent nothing, and would sit with its guard and no filter.
|
|
assigned, err := inv.Assigned(ctx, node)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
// And nothing assigned since the preview was composed: the flip takes every module the node
|
|
// runs, and one assigned in between would be taken without ever having been previewed.
|
|
if !slices.Equal(assigned, assignedWhenPreviewed) {
|
|
return preview, fmt.Errorf("what %s runs changed while this was converging (it is now %s): "+
|
|
"the flip takes every module on the node, so read the preview again", node,
|
|
strings.Join(assigned, ", "))
|
|
}
|
|
if !slices.Contains(assigned, filter) {
|
|
if _, err := inv.Assign(ctx, node, filter); err != nil {
|
|
return "", err
|
|
}
|
|
if _, _, err := planFor(ctx, open, node); err != nil {
|
|
_ = inv.Unassign(ctx, node, filter)
|
|
return "", fmt.Errorf("%s cannot run %s, so it was not converged: %w", node, filter, err)
|
|
}
|
|
}
|
|
took, err := inv.Converge(ctx, node)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
said := preview + fmt.Sprintf("\n\n%s is converged", node)
|
|
if len(took) > 0 {
|
|
said += "; took " + strings.Join(took, ", ")
|
|
}
|
|
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
|
return said + "\n and it could not be sent: run `push " + node + "`", err
|
|
}
|
|
return said + "\n sent: the host loads the mesh's filter and disables the firewall it found", nil
|
|
}
|
|
|
|
// previewOf is what converging a node will change, before it changes it, and a short digest of
|
|
// what it said: every reachable thing and its fate, the modules the flip takes and the filter. The
|
|
// digest is what the flip is asked to act on, so it changes whenever any of those would.
|
|
func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
|
|
plan catalogue.Resolution, taken []string, filter string, filterAssigned bool) (string, string) {
|
|
var said []string
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "converging %s\n", node)
|
|
fmt.Fprintf(&b, "\n reachable on the machine now, as it reported at %s:\n",
|
|
reported.At.Local().Format(time.DateTime))
|
|
for _, r := range reported.Reachable {
|
|
if loopback(r.Address) {
|
|
continue
|
|
}
|
|
what := fmt.Sprintf("%s/%d", r.Protocol, r.Port)
|
|
if r.By != "" {
|
|
what += " " + r.By
|
|
}
|
|
if r.Published {
|
|
what += fmt.Sprintf(" (published, container port %d)", r.ContainerPort)
|
|
}
|
|
fate := derived.fate(r)
|
|
fmt.Fprintf(&b, " %-44s %s\n", what, fate)
|
|
said = append(said, fmt.Sprintf("reach %s %s %s", r.Address, what, fate))
|
|
}
|
|
if countReachable(reported) == 0 {
|
|
// Said as what it is: no machine that is up is reachable on nothing, so this is an
|
|
// account that did not come back, not a machine with nothing on it.
|
|
b.WriteString(" nothing reported — this account looks partial, and the flip is " +
|
|
"refused on it\n")
|
|
said = append(said, "reach nothing reported")
|
|
}
|
|
// What the machine routes for others is not a listener and not a published port, so nothing
|
|
// above can show it; the derived filter's forward chain drops it all the same.
|
|
b.WriteString(" not previewed: traffic the machine routes that is not a published port " +
|
|
"(a tunnel, NAT in the found firewall) — the derived filter drops it unless a module " +
|
|
"declares it\n")
|
|
// Which links the filter constrains, said rather than left to the sentence above (novox/hq ADR
|
|
// 0140). Everything arriving anywhere else is this machine's own guest and keeps working — which
|
|
// is what a reader most wants to know, because the previous shape of this filter cut a machine's
|
|
// guests off at the flip without saying so, and that is how this was found.
|
|
if len(derived.outwardLinks) > 0 {
|
|
b.WriteString(fmt.Sprintf(" it filters what arrives on: %s, and on the private network "+
|
|
"— everything its own guests send keeps working\n",
|
|
strings.Join(derived.outwardLinks, ", ")))
|
|
} else {
|
|
b.WriteString(" it has reported no link facing outside, so no filter can be composed " +
|
|
"for it — the flip is refused until it reports one\n")
|
|
}
|
|
|
|
isTaken := map[string]bool{}
|
|
for _, m := range taken {
|
|
isTaken[m] = true
|
|
}
|
|
var takes []string
|
|
for _, m := range plan.Modules {
|
|
if !isTaken[m.Module] {
|
|
takes = append(takes, m.Module)
|
|
}
|
|
}
|
|
if !filterAssigned && !isTaken[filter] {
|
|
takes = append(takes, filter)
|
|
}
|
|
sort.Strings(takes)
|
|
b.WriteString("\n the flip takes:\n")
|
|
if len(takes) == 0 {
|
|
b.WriteString(" nothing — every module is taken already\n")
|
|
}
|
|
for _, m := range takes {
|
|
fmt.Fprintf(&b, " %s\n", m)
|
|
said = append(said, "take "+m)
|
|
// Every kind it holds — a directory, a service, an archive, a process, a user as well as a
|
|
// file (novox/hq ADR 0103) — each said, and each part of what the flip is asked to act on.
|
|
for _, h := range reported.Held {
|
|
if h.Module != m {
|
|
continue
|
|
}
|
|
fmt.Fprintf(&b, " replacing the found %s", heldLine(h))
|
|
if h.Kept != "" {
|
|
fmt.Fprintf(&b, ", original kept at %s", h.Kept)
|
|
}
|
|
b.WriteString("\n")
|
|
said = append(said, "replace "+m+" "+heldLine(h)+" "+h.Kept)
|
|
}
|
|
}
|
|
if !filterAssigned {
|
|
fmt.Fprintf(&b, "\n and assigns %s, which loads the mesh's filter in place of its guard\n", filter)
|
|
}
|
|
fw := reported.Firewall
|
|
if fw == "" || fw == "none" {
|
|
b.WriteString(" no firewall was found on the machine; the mesh's filter is its first\n")
|
|
} else {
|
|
fmt.Fprintf(&b, " the found firewall (%s) is disabled, never flushed: its configuration stays on disk\n", fw)
|
|
}
|
|
said = append(said, fmt.Sprintf("filter %s assigned=%t firewall=%s", filter, filterAssigned, fw))
|
|
// Sorted: the same account, reported in another order, is the same preview.
|
|
sort.Strings(said)
|
|
sum := sha256.Sum256([]byte(strings.Join(said, "\n")))
|
|
return strings.TrimRight(b.String(), "\n"), hex.EncodeToString(sum[:])[:12]
|
|
}
|
|
|
|
// derivedFilter is what the filter the flip loads is rendered from, as AsNftables renders it.
|
|
type derivedFilter struct {
|
|
rules []catalogue.Rule
|
|
foundation []int
|
|
// mesh is every address on the private network; outward says the machine faces outside.
|
|
mesh []string
|
|
outward bool
|
|
// outwardLinks is the links this machine reported as facing outside it (novox/hq ADR 0140).
|
|
// The filter constrains what arrives on them; everything arriving elsewhere is this machine's
|
|
// own guest and is not filtered.
|
|
outwardLinks []string
|
|
}
|
|
|
|
// closesOutside is what a narrowing from everywhere to the private network is called: it closes.
|
|
const closesOutside = "WILL CLOSE to everything outside the private network"
|
|
|
|
// fate is what the derived filter does to one reachable thing: which module declares it and from
|
|
// where, or that it will close — wholly, or to everything outside the private network. Rendered
|
|
// exactly as AsNftables admits it, ssh included.
|
|
func (d derivedFilter) fate(r inventory.Reach) string {
|
|
// Bound to an address on the private network, it was never reachable from outside it, so
|
|
// admitting it from the mesh narrows nothing.
|
|
onMesh := slices.Contains(d.mesh, strings.Trim(r.Address, "[]"))
|
|
if r.Protocol == "tcp" && r.Port == catalogue.SSHPort {
|
|
// From everywhere only when the machine faces outward or the mesh has no addresses to
|
|
// narrow it to; otherwise from the private network only.
|
|
if d.outward || len(d.mesh) == 0 || onMesh {
|
|
return "stays open — ssh is never closed"
|
|
}
|
|
return closesOutside + " — ssh stays open from the mesh, never closed there"
|
|
}
|
|
for _, port := range d.foundation {
|
|
if r.Protocol == "tcp" && r.Port == port {
|
|
return "stays open — the mesh's own, from anywhere"
|
|
}
|
|
}
|
|
// This machine's own guests ask it for an address and for names, and those two arrive here
|
|
// (novox/hq ADR 0140). Admitted by the link they arrive on, so a listener bound anywhere but an
|
|
// outward link keeps answering them.
|
|
if (r.Protocol == "udp" && (r.Port == 53 || r.Port == 67)) || (r.Protocol == "tcp" && r.Port == 53) {
|
|
return "stays open — this machine's own guests asking it for an address and for names"
|
|
}
|
|
for _, rule := range d.rules {
|
|
if rule.Port != r.Port || rule.Protocol != r.Protocol {
|
|
continue
|
|
}
|
|
by := strings.Join(rule.Because, ", ")
|
|
switch rule.From {
|
|
case catalogue.FromMachine:
|
|
return fmt.Sprintf("WILL CLOSE to the network — declared by %s for this machine only", by)
|
|
case catalogue.FromMesh:
|
|
if len(d.mesh) == 0 {
|
|
return fmt.Sprintf("WILL CLOSE — declared by %s from the mesh, and this node "+
|
|
"knows no mesh addresses", by)
|
|
}
|
|
if !onMesh {
|
|
return fmt.Sprintf("%s — declared by %s from the mesh only", closesOutside, by)
|
|
}
|
|
}
|
|
return fmt.Sprintf("declared by %s (from %s)", by, rule.From)
|
|
}
|
|
return "WILL CLOSE — no module assigned here declares it"
|
|
}
|
|
|
|
// countReachable is how much of a node's account of itself names something off the machine.
|
|
// Loopback is left out for the same reason the preview leaves it out: nothing outside reaches it,
|
|
// so a report of loopback alone says nothing about what the filter would close.
|
|
func countReachable(reported inventory.Adoption) int {
|
|
n := 0
|
|
for _, r := range reported.Reachable {
|
|
if !loopback(r.Address) {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// heldLine is one thing a node holds as found, as take and the converge preview both say it.
|
|
func heldLine(h inventory.Held) string {
|
|
return fmt.Sprintf("%s %s (%s)", h.Kind, h.Target, h.ID)
|
|
}
|
|
|
|
// loopback is an address nothing off the machine reaches.
|
|
func loopback(address string) bool {
|
|
a := strings.Trim(address, "[]")
|
|
return strings.HasPrefix(a, "127.") || a == "::1" || a == "localhost"
|
|
}
|
|
|
|
// adopt returns a converged node to adopted: the mesh's filter is unloaded, the guard restored,
|
|
// the found firewall enabled again and the openings converged through it once more. What was
|
|
// taken stays taken.
|
|
func adopt(ctx context.Context, open *stores, node string) (string, error) {
|
|
inv := open.inventory
|
|
record, err := inv.NodeByName(ctx, node)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if record.Adopted {
|
|
return "", fmt.Errorf("%s is adopted already", node)
|
|
}
|
|
held, release, err := holdNodes(ctx, open, []string{node})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer release()
|
|
ctx = held
|
|
if err := inv.SetAdopted(ctx, node, true); err != nil {
|
|
return "", err
|
|
}
|
|
said := fmt.Sprintf("%s is adopted; what was taken on it stays taken", node)
|
|
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
|
return said + "\n and it could not be sent: run `push " + node + "`", err
|
|
}
|
|
return said + "\n sent: the host unloads the mesh's filter and enables the firewall it found", nil
|
|
}
|
|
|
|
// takeCommand, convergeCommand and adoptCommand are the command line's adapters to the acts above.
|
|
func takeCommand(ctx context.Context, args []string) error {
|
|
set := flag.NewFlagSet("take", flag.ContinueOnError)
|
|
yes := set.Bool("yes", false, "cut over as previewed, naming the digest the preview printed after it; "+
|
|
"without it the comparison is printed and nothing is taken")
|
|
downgrade := set.Bool("downgrade", false, "take it although the module's image is older than the one running")
|
|
var replace, mint stringList
|
|
set.Var(&replace, "replace", "a found file's path whose content the module may replace although it differs (repeatable; * for every one)")
|
|
set.Var(&mint, "mint", "a secret the service shall take the mesh's minted value for, although it already has its own (repeatable)")
|
|
positionals, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(positionals) < 2 || len(positionals) > 3 || (len(positionals) == 3 && !*yes) {
|
|
return errors.New("take <node> <module> [--yes <digest>] [--downgrade] [--replace <path>]... [--mint <secret>]...")
|
|
}
|
|
opts := takeOptions{Yes: *yes, Downgrade: *downgrade, Replace: map[string]bool{}, Mint: map[string]bool{}}
|
|
if len(positionals) == 3 {
|
|
opts.Digest = positionals[2]
|
|
}
|
|
for _, r := range replace {
|
|
opts.Replace[r] = true
|
|
}
|
|
for _, m := range mint {
|
|
opts.Mint[m] = true
|
|
}
|
|
return runAct(ctx, func(open *stores) (string, error) { return take(ctx, open, positionals[0], positionals[1], opts) })
|
|
}
|
|
|
|
// stringList is a repeatable flag.
|
|
type stringList []string
|
|
|
|
func (l *stringList) String() string { return strings.Join(*l, ",") }
|
|
func (l *stringList) Set(v string) error { *l = append(*l, v); return nil }
|
|
|
|
func convergeCommand(ctx context.Context, args []string) error {
|
|
set := flag.NewFlagSet("converge", flag.ContinueOnError)
|
|
yes := set.String("yes", "", "do it, naming the digest the preview printed; without it, only "+
|
|
"the preview")
|
|
filter := set.String("filter", DefaultFilter, "the module that loads the mesh's filter")
|
|
positionals, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(positionals) != 1 {
|
|
return errors.New("converge <node> [--yes <digest>] [--filter nftables]")
|
|
}
|
|
return runAct(ctx, func(open *stores) (string, error) {
|
|
return converge(ctx, open, positionals[0], *yes != "", *yes, *filter)
|
|
})
|
|
}
|
|
|
|
func adoptCommand(ctx context.Context, args []string) error {
|
|
if len(args) != 1 {
|
|
return errors.New("adopt <node>")
|
|
}
|
|
return runAct(ctx, func(open *stores) (string, error) { return adopt(ctx, open, args[0]) })
|
|
}
|
|
|
|
func runAct(ctx context.Context, act func(*stores) (string, error)) error {
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
said, err := act(open)
|
|
if said != "" {
|
|
fmt.Println(said)
|
|
}
|
|
if err != nil && said != "" {
|
|
fmt.Println()
|
|
}
|
|
return err
|
|
}
|