Stop nothing the mesh cannot replace, give the tunnel back on failure, and take it over after enrolment
Review of the ADR 0105 build (hq ADR 0105). The takeover stopped the found unit and then found out whether the mesh's interface would do; a start that failed left the machine with no tunnel at all. Now nothing is stopped until the declared interface listens on the found port at the found address and the key file it names holds the found key — the refusal names the remedy — and a mesh interface that fails to start after the takeover has the found unit started again, with the account saying so. The account has three states (not taken, taken, down) and is given on every takeover, failure included. An interface raised by hand is looked at again for a moment and then refused naming `wg-quick down`. A found unit started again by hand beside the mesh's is said, not stopped: on the hub it cannot hold the port, and on a spoke two interfaces with one key would fight. `mesh-host overlay take --tunnel <iface>` is the path for a node that enrolled before the mesh knew to take a tunnel over: the found key becomes its overlay key — identity, sealing and serving keys untouched, so nothing sealed to the node is remade — and the mesh is told with a rekey signed by the identity key, over the key left, the key taken and the tunnel. Told first, written second, so a run again puts right whichever half did not happen.
This commit is contained in:
+229
-37
@@ -34,26 +34,53 @@ type TakenTunnel struct {
|
||||
Port int
|
||||
Range string
|
||||
Peers int
|
||||
// Taken is whether the found interface is down and disabled and the mesh's up in its place.
|
||||
Taken bool
|
||||
// State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one
|
||||
// down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's
|
||||
// not up: the peers reach nothing). Note is what this apply did about it.
|
||||
State string
|
||||
Note string
|
||||
Kept string
|
||||
}
|
||||
|
||||
// The states, as the link says them.
|
||||
const (
|
||||
NotTaken = "not-taken"
|
||||
Taken = "taken"
|
||||
TunnelDown = "down"
|
||||
)
|
||||
|
||||
// takeoverRecheck is how often, and takeoverRechecks how many times, a found interface still up
|
||||
// after its unit stopped is looked at again before the takeover is refused: `wg-quick down` by a
|
||||
// person takes a moment. Variables so a test need not wait.
|
||||
var (
|
||||
takeoverRecheck = 2 * time.Second
|
||||
takeoverRechecks = 3
|
||||
)
|
||||
|
||||
// takeOverID is the held record's id for the found configuration: the service's own with a suffix,
|
||||
// so it is declared for as long as the service is and never mistaken for the service itself.
|
||||
func takeOverID(svc *declaration.Service) string { return svc.ID + ".takes-over" }
|
||||
|
||||
// takeOver keeps the found tunnel's configuration and stops its unit, ahead of the service that
|
||||
// replaces it. Returned is the hold's outcome, and what was found for the report.
|
||||
//
|
||||
// **Nothing is stopped until the mesh's interface is known to be able to replace it** (the record's
|
||||
// option 2 is exactly this going wrong): the declared configuration must listen on the found port
|
||||
// at the found address, and the key file it points at must hold the found key. Only then is the
|
||||
// found unit stopped — and `stopped` says whether this apply did, so a mesh interface that then
|
||||
// fails to start can have the found unit started again.
|
||||
func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, d *declaration.Declaration,
|
||||
known *store.State, run Runner, keep Keep, now time.Time) (Outcome, TakenTunnel, error) {
|
||||
known *store.State, run Runner, keep Keep, now time.Time) (out Outcome, facts TakenTunnel, stopped bool, err error) {
|
||||
t := svc.TakesOver
|
||||
id := takeOverID(svc)
|
||||
module, _ := d.Adoption.UntakenModuleOf(svc.ID)
|
||||
if module == "" {
|
||||
module = "the private network"
|
||||
}
|
||||
facts := TakenTunnel{Interface: t.Interface}
|
||||
facts = TakenTunnel{Interface: t.Interface, State: NotTaken}
|
||||
|
||||
// 0. What the found configuration says, before anything: the checks below are against it.
|
||||
found, ferr := readFoundTunnel(t.Config)
|
||||
|
||||
// 1. The configuration, kept like any held file. A synthetic file resource stands for it, so
|
||||
// the same code keeps its original, digests it and notices it changing.
|
||||
@@ -62,65 +89,97 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service,
|
||||
out, held, err := hold(ctx, sys, file, module, was, already,
|
||||
"the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now)
|
||||
if err != nil {
|
||||
return begin(file), facts, fmt.Errorf("keeping the found tunnel's configuration: %w", err)
|
||||
return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err)
|
||||
}
|
||||
known.RecordHeld(held)
|
||||
facts.Kept = held.Kept
|
||||
// What the file says, for the report: read from the machine, or from the kept original when
|
||||
// the machine's copy is gone. The private key stays in the file; nothing here keeps it.
|
||||
// What the file says, for the report: from the machine, or from the kept original when the
|
||||
// machine's copy is gone. The private key stays in the file; nothing here keeps it.
|
||||
unread := ""
|
||||
raw, err := os.ReadFile(t.Config)
|
||||
if err != nil && held.Kept != "" {
|
||||
raw, err = os.ReadFile(held.Kept)
|
||||
if ferr != nil && held.Kept != "" {
|
||||
found, ferr = readFoundTunnel(held.Kept)
|
||||
}
|
||||
if err == nil {
|
||||
if found, perr := tunnel.Parse(raw); perr == nil {
|
||||
facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers)
|
||||
} else {
|
||||
unread = perr.Error()
|
||||
}
|
||||
if ferr == nil {
|
||||
facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers)
|
||||
} else {
|
||||
unread = err.Error()
|
||||
unread = ferr.Error()
|
||||
}
|
||||
|
||||
// 2. The found unit: stopped if it runs, disabled if it starts at boot. A unit that is not
|
||||
// there is not an error — the interface may have been raised another way, which the check
|
||||
// below catches — and neither is one already down.
|
||||
// 2. Where things stand: the found unit, and the mesh's.
|
||||
foundState, unitErr := sys.ServiceState(ctx, run, t.Unit)
|
||||
meshState, _ := sys.ServiceState(ctx, run, svc.Unit)
|
||||
if foundState == "running" && meshState == "running" {
|
||||
// Both up. On the hub this cannot last — the found unit cannot bind the port the mesh's
|
||||
// holds — and on a spoke two interfaces with one key flap between them. Not stopped again
|
||||
// by the mesh: what is found on an adopted node is reported, and the first takeover was
|
||||
// the one act (the PR note says why). Said, so a person sees it.
|
||||
facts.Note = t.Unit + " is running again beside the mesh's interface; not stopped by the mesh — " +
|
||||
"`systemctl stop " + t.Unit + "` on the machine"
|
||||
}
|
||||
|
||||
// 3. Before the found unit is stopped: can the mesh's interface replace it? Its declared
|
||||
// configuration must listen on the found port at the found address, and the key file it
|
||||
// points at must hold the found key, or the peers would be dropped the moment it came up.
|
||||
if foundState == "running" && meshState != "running" {
|
||||
if ferr != nil {
|
||||
return out, facts, false, fmt.Errorf("the found tunnel's configuration at %s cannot be read as a "+
|
||||
"tunnel's (%v), so nothing says what the mesh's interface must match; %s is left running",
|
||||
t.Config, ferr, t.Unit)
|
||||
}
|
||||
if err := replaces(d, svc, found); err != nil {
|
||||
return out, facts, false, fmt.Errorf("%w; %s is left running", err, t.Unit)
|
||||
}
|
||||
}
|
||||
|
||||
// 4. The found unit: stopped if it runs and the mesh's does not, disabled if it starts at
|
||||
// boot. A unit that is not there is not an error — the interface may have been raised
|
||||
// another way, which the check below catches — and neither is one already down.
|
||||
var did []string
|
||||
state, err := sys.ServiceState(ctx, run, t.Unit)
|
||||
switch {
|
||||
case err != nil:
|
||||
case unitErr != nil:
|
||||
did = append(did, t.Unit+" is not a unit here")
|
||||
case state == "running":
|
||||
case foundState == "running" && meshState != "running":
|
||||
if err := sys.SetServiceState(ctx, run, t.Unit, "stopped"); err != nil {
|
||||
return out, facts, fmt.Errorf("stopping the found %s: %w", t.Unit, err)
|
||||
return out, facts, false, fmt.Errorf("stopping the found %s: %w", t.Unit, err)
|
||||
}
|
||||
after, err := sys.ServiceState(ctx, run, t.Unit)
|
||||
if err != nil {
|
||||
return out, facts, err
|
||||
return out, facts, true, err
|
||||
}
|
||||
if after != "stopped" {
|
||||
return out, facts, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after)
|
||||
return out, facts, true, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after)
|
||||
}
|
||||
stopped = true
|
||||
did = append(did, "stopped "+t.Unit)
|
||||
}
|
||||
if err == nil {
|
||||
if unitErr == nil {
|
||||
if boot, err := sys.ServiceBoot(ctx, run, t.Unit); err == nil && boot == "enabled" {
|
||||
if err := sys.SetServiceBoot(ctx, run, t.Unit, "disabled"); err != nil {
|
||||
return out, facts, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err)
|
||||
return out, facts, stopped, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err)
|
||||
}
|
||||
did = append(did, "disabled it at boot")
|
||||
}
|
||||
}
|
||||
|
||||
// 3. The interface is gone. If it is still up, something other than its unit raised it, and
|
||||
// starting the mesh's on the same port and address would fail or, worse, half work.
|
||||
if up, err := run(ctx, "wg", "show", "interfaces"); err == nil {
|
||||
for _, iface := range strings.Fields(up) {
|
||||
if iface == t.Interface {
|
||||
return out, facts, fmt.Errorf("%s is still up after its unit %s was stopped: something other "+
|
||||
"than that unit raises it, and the mesh's interface cannot take its port and address "+
|
||||
"while it does. Nothing was flushed", t.Interface, t.Unit)
|
||||
// 5. The interface is gone. If it is still up, something other than its unit raised it —
|
||||
// the predecessor brings its up by hand — and the mesh's interface cannot take its port
|
||||
// and address while it is. Looked at again for a moment, since a person taking it down
|
||||
// takes a moment; then refused, naming what to do.
|
||||
if meshState != "running" {
|
||||
for try := 0; ; try++ {
|
||||
if !interfaceUp(ctx, run, t.Interface) {
|
||||
break
|
||||
}
|
||||
if try >= takeoverRechecks {
|
||||
return out, facts, stopped, fmt.Errorf("%s is still up although its unit %s is not running: it was "+
|
||||
"raised by hand, not by its unit, and the mesh's interface cannot take its port and "+
|
||||
"address while it is. On the machine: `wg-quick down %s` — the next reconcile takes it "+
|
||||
"over. Nothing was flushed", t.Interface, t.Unit, t.Interface)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return out, facts, stopped, ctx.Err()
|
||||
case <-time.After(takeoverRecheck):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -140,7 +199,140 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service,
|
||||
// peers was carried, which reads as a tunnel that was not one.
|
||||
out.Detail += "; what it says could not be read as a tunnel's: " + unread
|
||||
}
|
||||
return out, facts, nil
|
||||
return out, facts, stopped, nil
|
||||
}
|
||||
|
||||
// readFoundTunnel is the found configuration as a tunnel.
|
||||
func readFoundTunnel(path string) (tunnel.Found, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return tunnel.Found{}, err
|
||||
}
|
||||
return tunnel.Parse(raw)
|
||||
}
|
||||
|
||||
// interfaceUp is whether a WireGuard interface is up on the machine.
|
||||
func interfaceUp(ctx context.Context, run Runner, iface string) bool {
|
||||
up, err := run(ctx, "wg", "show", "interfaces")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, name := range strings.Fields(up) {
|
||||
if name == iface {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// replaces holds the mesh's declared interface configuration against the found tunnel it is to
|
||||
// replace: same port, same address, and a key file holding the found key. The configuration is
|
||||
// the file the service restarts on; its `PostUp = wg set %i private-key <path>` names the key.
|
||||
func replaces(d *declaration.Declaration, svc *declaration.Service, found tunnel.Found) error {
|
||||
var conf *declaration.File
|
||||
for _, r := range d.Resources {
|
||||
f, ok := r.(*declaration.File)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, id := range svc.RestartOn {
|
||||
if f.ID == id {
|
||||
conf = f
|
||||
}
|
||||
}
|
||||
}
|
||||
if conf == nil {
|
||||
return fmt.Errorf("%s takes over %s and restarts on no declared file, so the interface it would "+
|
||||
"raise cannot be checked against the found one", svc.Unit, found.Interface)
|
||||
}
|
||||
port, address, keyPath := "", "", ""
|
||||
for _, line := range strings.Split(conf.Content, "\n") {
|
||||
key, value, ok := strings.Cut(strings.TrimSpace(line), "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value)
|
||||
switch key {
|
||||
case "listenport":
|
||||
port = value
|
||||
case "address":
|
||||
address = strings.TrimSpace(strings.Split(value, ",")[0])
|
||||
case "postup":
|
||||
if _, after, ok := strings.Cut(value, "private-key "); ok {
|
||||
keyPath = strings.Fields(after)[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
var wrong []string
|
||||
if port != fmt.Sprint(found.Port) {
|
||||
wrong = append(wrong, fmt.Sprintf("it listens on port %q and the tunnel on %d", port, found.Port))
|
||||
}
|
||||
if host(address) != host(found.Address) {
|
||||
wrong = append(wrong, fmt.Sprintf("its address is %q and the tunnel's %s", address, found.Address))
|
||||
}
|
||||
switch raw, err := os.ReadFile(keyPath); {
|
||||
case keyPath == "":
|
||||
wrong = append(wrong, "it names no key file")
|
||||
case err != nil:
|
||||
wrong = append(wrong, fmt.Sprintf("its key file %s cannot be read (%v)", keyPath, err))
|
||||
default:
|
||||
public, perr := tunnel.PublicKeyOf(strings.TrimSpace(string(raw)))
|
||||
if perr != nil || public != found.PublicKey {
|
||||
wrong = append(wrong, fmt.Sprintf("the key at %s is not the tunnel's — `mesh-host overlay take "+
|
||||
"--tunnel %s` on this machine takes it, then push again", keyPath, found.Interface))
|
||||
}
|
||||
}
|
||||
if len(wrong) > 0 {
|
||||
return fmt.Errorf("the mesh's interface would not replace the tunnel on %s: %s — the peers would be "+
|
||||
"dropped the moment it came up. Re-place the hub on the tunnel's address and port and push again",
|
||||
found.Interface, strings.Join(wrong, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// host is an address without its prefix length.
|
||||
func host(address string) string {
|
||||
if i := strings.Index(address, "/"); i >= 0 {
|
||||
return address[:i]
|
||||
}
|
||||
return address
|
||||
}
|
||||
|
||||
// tunnelState is where the tunnel stands, read from the machine: the found unit or interface up
|
||||
// and the mesh's not is not taken; the mesh's up and the found one down is taken; neither up is
|
||||
// down — the peers reach nothing.
|
||||
func tunnelState(ctx context.Context, sys system.System, foundUnit, meshUnit string, run Runner) string {
|
||||
foundState, _ := sys.ServiceState(ctx, run, foundUnit)
|
||||
meshState, _ := sys.ServiceState(ctx, run, meshUnit)
|
||||
foundUp := foundState == "running" || interfaceUp(ctx, run, strings.TrimPrefix(foundUnit, "wg-quick@"))
|
||||
switch {
|
||||
case meshState == "running" && !foundUp:
|
||||
return Taken
|
||||
case meshState == "running":
|
||||
// Both up: not a takeover that holds, and said as not taken so nobody reads it as one.
|
||||
return NotTaken
|
||||
case foundUp:
|
||||
return NotTaken
|
||||
default:
|
||||
return TunnelDown
|
||||
}
|
||||
}
|
||||
|
||||
// restoreFound starts the found unit again after the mesh's interface failed to replace it, so the
|
||||
// machine has the tunnel it had rather than none, and says so in the account.
|
||||
func restoreFound(ctx context.Context, sys system.System, unit string, run Runner, facts *TakenTunnel) {
|
||||
if err := sys.SetServiceState(ctx, run, unit, "running"); err != nil {
|
||||
facts.State = TunnelDown
|
||||
facts.Note += "; " + unit + " could not be started again (" + err.Error() + ") — on the machine: systemctl start " + unit
|
||||
return
|
||||
}
|
||||
if state, err := sys.ServiceState(ctx, run, unit); err != nil || state != "running" {
|
||||
facts.State = TunnelDown
|
||||
facts.Note += "; " + unit + " was started again and is not running — on the machine: systemctl start " + unit
|
||||
return
|
||||
}
|
||||
facts.State = NotTaken
|
||||
facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had"
|
||||
}
|
||||
|
||||
// takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since
|
||||
|
||||
Reference in New Issue
Block a user