package apply import ( "context" "errors" "fmt" "os" "path/filepath" "strings" "time" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" "github.com/novox/mesh-host/internal/tunnel" ) // The private network takes over the tunnel it found (novox/hq ADR 0105). // // The controller says so on the interface's service: `takes-over` names the found interface, the // unit that raised it and its configuration file. Before the mesh's unit is started, the host keeps // that file like any held file — the original recorded before anything else happens to it — and // stops and disables the found unit. Never a flush: `wg set … peer … remove` is never run, the // file is never written, and the found interface goes down the way its own unit takes it down. // Then the mesh's interface comes up, with the found key the node took at enrolment, on the found // port, with the found peers in its list — and a peer of the tunnel cannot tell it changed hands. // // Every apply, not once: a found unit somebody starts again would take the port back from the // mesh's interface, so it is stopped again and said so. That is the one place an adopted node // undoes something done by hand, and it is because the tunnel is the mesh's now. // // **Until the take is proven, and then the found configuration is retired** (novox/hq ADR 0119). // Keeping it on disk was the caution the take needed: if the mesh's interface does not come up, // the found unit is started again and the peers never notice. That caution is spent once the // tunnel is taken — the found unit down and disabled, the mesh's interface up — and a peer has // handshaken with the mesh's interface. From then on a configuration nothing maintains, one // command away from raising a second way onto the network, is not a rollback path but a door // nobody watches. So it is removed from where its unit reads it; its original, kept before // anything happened to it (ADR 0100), stays kept; and the hold on it ends. A take never proven // keeps it, and says so — a broken take is visible, not silently retired. // TakenTunnel is what an apply says about a tunnel it took over, for the node's report. type TakenTunnel struct { Interface string Port int Range string Peers int // State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one // down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's // not up: the peers reach nothing). Note is what this apply did about it — and, for a taken // tunnel, whether the take is proven and its found configuration retired (novox/hq ADR 0119). // Kept is where the found configuration's original is, retired or not. State string Note string Kept string } // The states, as the link says them. const ( NotTaken = "not-taken" Taken = "taken" TunnelDown = "down" ) // takeoverRecheck is how often, and takeoverRechecks how many times, a found interface still up // after its unit stopped is looked at again before the takeover is refused: `wg-quick down` by a // person takes a moment. Variables so a test need not wait. var ( takeoverRecheck = 2 * time.Second takeoverRechecks = 3 ) // takeOverID is the held record's id for the found configuration: the service's own with a suffix, // so it is declared for as long as the service is and never mistaken for the service itself. func takeOverID(svc *declaration.Service) string { return svc.ID + ".takes-over" } // takeOver keeps the found tunnel's configuration and stops its unit, ahead of the service that // replaces it. Returned is the hold's outcome, and what was found for the report. // // **Nothing is stopped until the mesh's interface is known to be able to replace it** (the record's // option 2 is exactly this going wrong): the declared configuration must listen on the found port // at the found address, and the key file it points at must hold the found key. Only then is the // found unit stopped — and `stopped` says whether this apply did, so a mesh interface that then // fails to start can have the found unit started again. func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, d *declaration.Declaration, known *store.State, run Runner, keep Keep, now time.Time) (out Outcome, facts TakenTunnel, stopped bool, err error) { t := svc.TakesOver id := takeOverID(svc) module, _ := d.Adoption.UntakenModuleOf(svc.ID) if module == "" { module = "the private network" } facts = TakenTunnel{Interface: t.Interface, State: NotTaken} // 0. What the found configuration says, before anything: the checks below are against it. found, ferr := readFoundTunnel(t.Config) // 1. The configuration, kept like any held file. A synthetic file resource stands for it, so // the same code keeps its original, digests it and notices it changing. // // **Unless it was retired** (novox/hq ADR 0119): the take was proven and the mesh removed // it, so there is nothing to hold and nothing missing — only where its original is, which // the retirement recorded. A hold still standing is let go: that is what retiring it meant. // // **One that comes back is held again on its FIRST original** — the one kept before anything // happened to it (ADR 0100), never whatever was put back — and retired again by the first // apply that finds the take still proven, keeping what came back only if it differs from // what is already kept. Put back by hand while the private network is assigned, it is not a // rollback: that means unassigning the private network first, and the note says so. file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config} var held store.Held retired, wasRetired := known.RetiredAt(t.Config) cameBack := wasRetired && present(t.Config) switch { case wasRetired && !cameBack: known.Release(id) held = store.Held{Kept: retired.Kept} out = begin(file) out.Action = "unchanged" facts.Note = "the found configuration " + t.Config + " was retired once the take was proven; " + "its original is kept at " + retired.Kept + " and the mesh never brings it back" default: was, already := known.HeldAt(id) why := "the configuration of the tunnel " + t.Interface + ", taken over by " + svc.Unit if cameBack && !already { digest := retired.Digest if digest == "" { if raw, err := os.ReadFile(retired.Kept); err == nil { digest = digestOf(string(raw)) } } was = store.Held{ID: id, Module: module, Kind: string(declaration.TypeFile), Target: t.Config, Since: now, Why: why, Kept: retired.Kept, Digest: digest} already = true } out, held, err = hold(ctx, sys, file, module, was, already, why, run, keep, now) if err != nil { return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err) } known.RecordHeld(held) if cameBack { facts.Note = "the found configuration " + t.Config + " came back after it was retired; while the " + "private network is assigned the mesh retires it again, so rolling back to the found tunnel " + "means unassigning the private network first" } } facts.Kept = held.Kept // What the file says, for the report: from the machine, or from the kept original when the // machine's copy is gone. The private key stays in the file; nothing here keeps it. unread := "" if ferr != nil && held.Kept != "" { found, ferr = readFoundTunnel(held.Kept) } if ferr == nil { facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers) } else { unread = ferr.Error() } // 2. Where things stand: the found unit, and the mesh's. foundState, unitErr := sys.ServiceState(ctx, run, t.Unit) meshState, _ := sys.ServiceState(ctx, run, svc.Unit) if foundState == "running" && meshState == "running" { // Both up. On the hub this cannot last — the found unit cannot bind the port the mesh's // holds — and on a spoke two interfaces with one key flap between them. Not stopped again // by the mesh: what is found on an adopted node is reported, and the first takeover was // the one act (the PR note says why). Said, so a person sees it. facts.Note = t.Unit + " is running again beside the mesh's interface; not stopped by the mesh — " + "`systemctl stop " + t.Unit + "` on the machine" } // 3. Before the found unit is stopped: can the mesh's interface replace it? Its declared // configuration must listen on the found port at the found address, and the key file it // points at must hold the found key, or the peers would be dropped the moment it came up. if foundState == "running" && meshState != "running" { if ferr != nil { return out, facts, false, fmt.Errorf("the found tunnel's configuration at %s cannot be read as a "+ "tunnel's (%v), so nothing says what the mesh's interface must match; %s is left running", t.Config, ferr, t.Unit) } if err := replaces(d, svc, found); err != nil { return out, facts, false, fmt.Errorf("%w; %s is left running", err, t.Unit) } } // 4. The found unit: stopped if it runs and the mesh's does not, disabled if it starts at // boot. A unit that is not there is not an error — the interface may have been raised // another way, which the check below catches — and neither is one already down. var did []string switch { case unitErr != nil: did = append(did, t.Unit+" is not a unit here") case foundState == "running" && meshState != "running": if err := sys.SetServiceState(ctx, run, t.Unit, "stopped"); err != nil { return out, facts, false, fmt.Errorf("stopping the found %s: %w", t.Unit, err) } after, err := sys.ServiceState(ctx, run, t.Unit) if err != nil { return out, facts, true, err } if after != "stopped" { return out, facts, true, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after) } stopped = true did = append(did, "stopped "+t.Unit) } if unitErr == nil { if boot, err := sys.ServiceBoot(ctx, run, t.Unit); err == nil && boot == "enabled" { if err := sys.SetServiceBoot(ctx, run, t.Unit, "disabled"); err != nil { return out, facts, stopped, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err) } did = append(did, "disabled it at boot") } } // 5. The interface is gone. If it is still up, something other than its unit raised it — // the predecessor brings its up by hand — and the mesh's interface cannot take its port // and address while it is. Looked at again for a moment, since a person taking it down // takes a moment; then refused, naming what to do. if meshState != "running" { for try := 0; ; try++ { if !interfaceUp(ctx, run, t.Interface) { break } if try >= takeoverRechecks { return out, facts, stopped, fmt.Errorf("%s is still up although its unit %s is not running: it was "+ "raised by hand, not by its unit, and the mesh's interface cannot take its port and "+ "address while it is. On the machine: `wg-quick down %s` — the next reconcile takes it "+ "over. Nothing was flushed", t.Interface, t.Unit, t.Interface) } select { case <-ctx.Done(): return out, facts, stopped, ctx.Err() case <-time.After(takeoverRecheck): } } } out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found" switch { case cameBack: out.Detail = "the tunnel " + t.Interface + "'s configuration, back after it was retired; held until " + "it is retired again" case wasRetired: out.Detail = "the tunnel " + t.Interface + "'s configuration, retired once the take was proven" } if held.Kept != "" { out.Detail += " (original at " + held.Kept + ")" } if len(did) > 0 { out.Detail += "; " + strings.Join(did, ", ") + " — never flushed" } if held.Changed != "" { out.Detail += "; " + held.Changed + " by something other than the mesh since it was found" } if unread != "" { // Said, not swallowed: the report would otherwise say a tunnel with no port and no // peers was carried, which reads as a tunnel that was not one. out.Detail += "; what it says could not be read as a tunnel's: " + unread } return out, facts, stopped, nil } // readFoundTunnel is the found configuration as a tunnel. func readFoundTunnel(path string) (tunnel.Found, error) { raw, err := os.ReadFile(path) if err != nil { return tunnel.Found{}, err } return tunnel.Parse(raw) } // interfaceUp is whether a WireGuard interface is up on the machine. func interfaceUp(ctx context.Context, run Runner, iface string) bool { up, err := run(ctx, "wg", "show", "interfaces") if err != nil { return false } for _, name := range strings.Fields(up) { if name == iface { return true } } return false } // replaces holds the mesh's declared interface configuration against the found tunnel it is to // replace: same port, same address, and a key file holding the found key. The configuration is // the file the service restarts on; its `PostUp = wg set %i private-key ` names the key. func replaces(d *declaration.Declaration, svc *declaration.Service, found tunnel.Found) error { var conf *declaration.File for _, r := range d.Resources { f, ok := r.(*declaration.File) if !ok { continue } for _, id := range svc.RestartOn { if f.ID == id { conf = f } } } if conf == nil { return fmt.Errorf("%s takes over %s and restarts on no declared file, so the interface it would "+ "raise cannot be checked against the found one", svc.Unit, found.Interface) } port, address, keyPath := "", "", "" for _, line := range strings.Split(conf.Content, "\n") { key, value, ok := strings.Cut(strings.TrimSpace(line), "=") if !ok { continue } key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value) switch key { case "listenport": port = value case "address": address = strings.TrimSpace(strings.Split(value, ",")[0]) case "postup": if _, after, ok := strings.Cut(value, "private-key "); ok { keyPath = strings.Fields(after)[0] } } } var wrong []string if port != fmt.Sprint(found.Port) { wrong = append(wrong, fmt.Sprintf("it listens on port %q and the tunnel on %d", port, found.Port)) } if host(address) != host(found.Address) { wrong = append(wrong, fmt.Sprintf("its address is %q and the tunnel's %s", address, found.Address)) } switch raw, err := os.ReadFile(keyPath); { case keyPath == "": wrong = append(wrong, "it names no key file") case err != nil: wrong = append(wrong, fmt.Sprintf("its key file %s cannot be read (%v)", keyPath, err)) default: public, perr := tunnel.PublicKeyOf(strings.TrimSpace(string(raw))) if perr != nil || public != found.PublicKey { wrong = append(wrong, fmt.Sprintf("the key at %s is not the tunnel's — `mesh-host overlay take "+ "--tunnel %s` on this machine takes it, then push again", keyPath, found.Interface)) } } if len(wrong) > 0 { return fmt.Errorf("the mesh's interface would not replace the tunnel on %s: %s — the peers would be "+ "dropped the moment it came up. Re-place the hub on the tunnel's address and port and push again", found.Interface, strings.Join(wrong, "; ")) } return nil } // host is an address without its prefix length. func host(address string) string { if i := strings.Index(address, "/"); i >= 0 { return address[:i] } return address } // tunnelState is where the tunnel stands, read from the machine: the found unit or interface up // and the mesh's not is not taken; the mesh's up and the found one down is taken; neither up is // down — the peers reach nothing. func tunnelState(ctx context.Context, sys system.System, foundUnit, meshUnit string, run Runner) string { foundState, _ := sys.ServiceState(ctx, run, foundUnit) meshState, _ := sys.ServiceState(ctx, run, meshUnit) foundUp := foundState == "running" || interfaceUp(ctx, run, strings.TrimPrefix(foundUnit, "wg-quick@")) switch { case meshState == "running" && !foundUp: return Taken case meshState == "running": // Both up: not a takeover that holds, and said as not taken so nobody reads it as one. return NotTaken case foundUp: return NotTaken default: return TunnelDown } } // restoreFound starts the found unit again after the mesh's interface failed to replace it, so the // machine has the tunnel it had rather than none, and says so in the account. func restoreFound(ctx context.Context, sys system.System, unit string, run Runner, facts *TakenTunnel) { if err := sys.SetServiceState(ctx, run, unit, "running"); err != nil { facts.State = TunnelDown facts.Note += "; " + unit + " could not be started again (" + err.Error() + ") — on the machine: systemctl start " + unit return } if state, err := sys.ServiceState(ctx, run, unit); err != nil || state != "running" { facts.State = TunnelDown facts.Note += "; " + unit + " was started again and is not running — on the machine: systemctl start " + unit return } facts.State = NotTaken facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had" } // wireguardDir is where a found tunnel's configuration may be retired from: wg-quick's own, and // nowhere else. A variable so a test can hand in a directory. var wireguardDir = tunnel.ConfigDir // retireFound removes the found tunnel's configuration from where its unit reads it, once the take // is proven, and ends the hold on it (novox/hq ADR 0119). Asked after the mesh's service applied // and the tunnel reads as taken; retired says whether this apply retired it, and out is then what // replaces the take's outcome for the configuration. // // **Proven is taken and a handshake.** Taken alone — the found unit down and disabled, the mesh's // interface up — says the mesh's interface exists, not that any peer reaches it: an interface up // with the wrong key is taken and carries nothing. A peer that has completed a handshake with it // has checked its key, so that is the proof, asked of the kernel through `wg`. Any handshake counts, // however old: a change to the mesh's configuration restarts its unit, which recreates the // interface and resets its counters, so a time that is there at all was made by this interface. // Anything short of one — no peer yet, every time zero, `wg` missing or failing — keeps the file, // and the account says which: a take that never proves itself is visible rather than silently // retired. // // **Only what the take names, and only wg-quick's own file.** Nothing is removed unless the path // is exactly `/.conf`, is not a path the mesh itself writes, and is // a file rather than a link: removing a link would leave the key-bearing file it points at where it // is, a retirement in name only, so that one is said and left to a person. // // **The original must still be kept.** It is the record of what the predecessor was and a // person's only way back (ADR 0100); a kept copy that has gone missing is said, and the file is // not removed, since removing it then would lose the only copy. What is on disk now, if it differs // from the first original and from what was kept at the last retirement, is kept too before it // goes — by content, so the first original is never overwritten and a file that keeps coming back // the same keeps nothing more. // // The found unit is left disabled; without its configuration it cannot raise the interface, so // every later apply's check of it finds nothing to do. Nothing here ever writes the file back. func retireFound(ctx context.Context, svc *declaration.Service, d *declaration.Declaration, known *store.State, run Runner, keep Keep, facts *TakenTunnel, now time.Time) (out Outcome, retired bool) { t := svc.TakesOver id := takeOverID(svc) if facts.State != Taken { return out, false } held, isHeld := known.HeldAt(id) if !isHeld { // Retired already (takeOver let any hold go and said so), or never held: nothing to do. return out, false } say := func(note string) { if facts.Note != "" { facts.Note += "; " } facts.Note += note } notRetired := func(why string) (Outcome, bool) { say("the found configuration " + t.Config + " is not retired: " + why) return Outcome{}, false } mesh := strings.TrimPrefix(svc.Unit, "wg-quick@") peers, err := tunnel.Handshaken(ctx, tunnel.Runner(run), mesh) if err != nil { say("taken, not yet proven: " + err.Error() + "; the found configuration " + t.Config + " is kept") return out, false } if peers == 0 { say("taken, not yet proven: no peer has handshaken on " + mesh + "; the found configuration " + t.Config + " is kept") return out, false } proven := fmt.Sprintf("proven: %d peer(s) handshaken on %s", peers, mesh) say(proven) // What may be removed at all. if want := filepath.Join(wireguardDir, t.Interface+".conf"); t.Config != want { return notRetired("only " + want + ", the found interface's own wg-quick configuration, is ever " + "retired by the mesh, and the take names " + t.Config) } if known.Recorded(string(declaration.TypeFile), t.Config) || declaresFile(d, t.Config) { return notRetired("it is a path the mesh itself writes") } if info, err := os.Lstat(t.Config); err == nil && info.Mode()&os.ModeSymlink != 0 { target, _ := os.Readlink(t.Config) return notRetired("it is a link to " + target + "; removing the link would leave the key-bearing file " + "it points at, so it must be retired by hand — both are kept") } // The kept original, read back — not just named in a record. if held.Kept == "" { return notRetired("no original of it was kept, so removing it would leave no record of what the " + "predecessor was") } original, err := os.ReadFile(held.Kept) if err != nil || (held.Digest != "" && digestOf(string(original)) != held.Digest) { why := "is missing" if err == nil { why = "no longer holds what was found" } else if !errors.Is(err, os.ErrNotExist) { why = "cannot be read (" + err.Error() + ")" } return notRetired("its kept original " + held.Kept + " " + why + ", so removing it would lose the only copy") } before, cameBack := known.RetiredAt(t.Config) record := store.Retired{ID: id, Path: t.Config, Kept: held.Kept, Digest: digestOf(string(original)), At: now} if cameBack { // The first original stays the record's, and so does what the last retirement kept. record.Extra, record.ExtraDigest, record.Again = before.Extra, before.ExtraDigest, before.Again+1 } newCopy := "" gone := !present(t.Config) if !gone { current, err := os.ReadFile(t.Config) if err != nil { return notRetired("it cannot be read (" + err.Error() + ")") } if sum := digestOf(string(current)); sum != record.Digest && sum != record.ExtraDigest { if keep == nil { return notRetired("it holds something other than its kept original and this host has nowhere " + "to keep it") } where, err := keep(t.Config, current, 0o600) if err != nil { return notRetired("keeping what it holds now failed (" + err.Error() + ")") } record.Extra, record.ExtraDigest, newCopy = where, sum, where } if err := os.Remove(t.Config); err != nil && !errors.Is(err, os.ErrNotExist) { return notRetired("removing it failed (" + err.Error() + ")") } if present(t.Config) { return notRetired("it is still there after it was removed") } } known.RecordRetired(record) known.Release(id) facts.Kept = held.Kept copied := "" if newCopy != "" { copied = "; what it held, which differed from the original, is kept at " + newCopy } out = Outcome{ID: id, Type: string(declaration.TypeFile), Target: t.Config, Action: "removed"} switch { case cameBack: say("the found configuration came back and was retired again — its original still kept at " + held.Kept + copied + "; rolling back to the found tunnel means unassigning the private network first") out.Detail = "the found configuration came back and was retired again; original kept at " + held.Kept + copied default: say("the found configuration " + t.Config + " is retired — its original kept at " + held.Kept + copied + ", " + t.Unit + " left disabled, and the mesh never brings it back") out.Detail = "retired: the take of " + t.Interface + " is " + proven + "; original kept at " + held.Kept + copied } if gone { // Already gone — removed by something other than the mesh, or by an apply whose record was // never saved. Nothing removed here; the hold ends all the same. out.Action = "unchanged" out.Detail = "retired: the take of " + t.Interface + " is " + proven + " and " + t.Config + " was already gone; original kept at " + held.Kept } return out, true } // declaresFile is whether a declaration writes a file at a path. func declaresFile(d *declaration.Declaration, path string) bool { for _, r := range d.Resources { if f, ok := r.(*declaration.File); ok && filepath.Clean(f.Path) == filepath.Clean(path) { return true } } return false } // takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since // a machine has one private network. func takesOver(d *declaration.Declaration) *declaration.Service { for _, r := range d.Resources { if svc, ok := r.(*declaration.Service); ok && svc.TakesOver != nil { return svc } } return nil } // errNotAdopted is a takeover on a declaration that does not say the node is adopted, which the // parser refuses already; kept as a second line of defence at the point of acting. var errNotAdopted = errors.New("a tunnel is taken over on an adopted node only")