Take over the found tunnel: its key, its port, its peers; stop it, never flush (hq ADR 0105) #24
+86
-1
@@ -56,6 +56,8 @@ const usage = `mesh-host — the node host
|
|||||||
apply FILE make this machine match a declaration from a file
|
apply FILE make this machine match a declaration from a file
|
||||||
reconcile make this machine match the declaration this host carries
|
reconcile make this machine match the declaration this host carries
|
||||||
bundle show what this host carries
|
bundle show what this host carries
|
||||||
|
overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this
|
||||||
|
node's overlay key and the mesh is told, signed; --tunnel <iface> when several are up
|
||||||
owned what this host has applied and still owns
|
owned what this host has applied and still owns
|
||||||
version
|
version
|
||||||
|
|
||||||
@@ -147,6 +149,13 @@ func parseArgs(args []string) (string, options, error) {
|
|||||||
opts.file = positionals[0]
|
opts.file = positionals[0]
|
||||||
return command, opts, nil
|
return command, opts, nil
|
||||||
}
|
}
|
||||||
|
if command == "overlay" {
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return "", opts, errors.New("overlay take [--tunnel <iface>]")
|
||||||
|
}
|
||||||
|
opts.file = positionals[0]
|
||||||
|
return command, opts, nil
|
||||||
|
}
|
||||||
// Anything left over was neither the command nor a flag. Refused rather than ignored: a
|
// Anything left over was neither the command nor a flag. Refused rather than ignored: a
|
||||||
// mistyped argument that changes nothing and reports success is worse than an error.
|
// mistyped argument that changes nothing and reports success is worse than an error.
|
||||||
if len(positionals) > 0 {
|
if len(positionals) > 0 {
|
||||||
@@ -234,6 +243,8 @@ func run(ctx context.Context, command string, opts options) error {
|
|||||||
|
|
||||||
case "enrol", "enroll":
|
case "enrol", "enroll":
|
||||||
return enrol(ctx, opts)
|
return enrol(ctx, opts)
|
||||||
|
case "overlay":
|
||||||
|
return overlayCommand(ctx, opts)
|
||||||
|
|
||||||
case "run":
|
case "run":
|
||||||
return runLink(ctx, opts)
|
return runLink(ctx, opts)
|
||||||
@@ -608,6 +619,80 @@ func enrol(ctx context.Context, opts options) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// overlayCommand is `mesh-host overlay take`: this node takes the tunnel found on its machine over
|
||||||
|
// after it enrolled (novox/hq ADR 0105). For a node that enrolled before the mesh knew to take a
|
||||||
|
// tunnel over — re-enrolling would rotate its identity, sealing and serving keys, and with them
|
||||||
|
// every credential the mesh sealed to it.
|
||||||
|
func overlayCommand(ctx context.Context, opts options) error {
|
||||||
|
if opts.file != "take" {
|
||||||
|
return errors.New("overlay take [--tunnel <iface>] — the one thing `overlay` does here")
|
||||||
|
}
|
||||||
|
identityPath := identity.Path(opts.state)
|
||||||
|
mine, err := identity.Load(identityPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
found, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w. Nothing was changed", err)
|
||||||
|
}
|
||||||
|
taken, rekey, err := rekeyOnto(mine, found)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("taking over %s: this node's overlay key becomes the tunnel's, %s\n", found, taken.Overlay.Public)
|
||||||
|
fmt.Printf(" identity, sealing and serving keys are untouched\n")
|
||||||
|
|
||||||
|
// Told first, then written: a mesh told and a machine not yet written is put right by running
|
||||||
|
// this again (the mesh refuses the stale second rekey and changes nothing; the files are
|
||||||
|
// rewritten the same). A machine written and a mesh not told would raise the mesh's interface
|
||||||
|
// on a key the mesh does not know at the next restart.
|
||||||
|
if err := link.Publish(ctx, link.Membership{
|
||||||
|
Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint,
|
||||||
|
Password: mine.Membership.Password, Signer: mine.Membership.Signer,
|
||||||
|
}, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil {
|
||||||
|
return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err)
|
||||||
|
}
|
||||||
|
fmt.Printf(" told the mesh signed rekey sent; `node show %s` on the controller says whether it took\n", mine.Node)
|
||||||
|
|
||||||
|
if err := os.WriteFile(identity.OverlayKeyPath(opts.state),
|
||||||
|
[]byte(taken.Overlay.Private+"\n"), 0o600); err != nil {
|
||||||
|
return fmt.Errorf("the mesh was told and this node's overlay key could not be written: %w — run this again", err)
|
||||||
|
}
|
||||||
|
if err := identity.Save(identityPath, taken); err != nil {
|
||||||
|
return fmt.Errorf("the mesh was told and this node's identity could not be saved: %w — run this again", err)
|
||||||
|
}
|
||||||
|
fmt.Printf(" written %s and the identity; the mesh's interface reads the key when the next push restarts it\n",
|
||||||
|
identity.OverlayKeyPath(opts.state))
|
||||||
|
fmt.Printf(" next on the controller: `overlay place %s --hub --endpoint <host>:%d …`, `plan %s --json`, then push\n",
|
||||||
|
mine.Node, found.Port, mine.Node)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rekeyOnto is the identity with the found tunnel's key as its overlay key, and the signed rekey
|
||||||
|
// that tells the mesh. Pure, so it can be held to: node, sealing and serving keys are the same
|
||||||
|
// bytes in and out; only the overlay key moves. Run again after a take, the previous key it names
|
||||||
|
// is the one before the take, so the mesh can tell a repeat from a replay.
|
||||||
|
func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, link.Rekey, error) {
|
||||||
|
overlay, err := identity.OverlayKeyFrom(found.PrivateKey())
|
||||||
|
if err != nil {
|
||||||
|
return identity.Identity{}, link.Rekey{}, err
|
||||||
|
}
|
||||||
|
previous := mine.Overlay.Public
|
||||||
|
if previous == overlay.Public && mine.OverlayBefore != "" {
|
||||||
|
previous = mine.OverlayBefore
|
||||||
|
}
|
||||||
|
taken := mine
|
||||||
|
taken.Overlay = overlay
|
||||||
|
if previous != overlay.Public {
|
||||||
|
taken.OverlayBefore = previous
|
||||||
|
}
|
||||||
|
presented := carried(found)
|
||||||
|
rekey := link.Rekey{Previous: previous, OverlayKey: overlay.Public, Tunnel: presented}
|
||||||
|
rekey.Proof = mine.Sign(link.RekeyProof(mine.Node, previous, overlay.Public, presented))
|
||||||
|
return taken, rekey, nil
|
||||||
|
}
|
||||||
|
|
||||||
// carried is a found tunnel as it is presented to the mesh: everything but its private key.
|
// carried is a found tunnel as it is presented to the mesh: everything but its private key.
|
||||||
func carried(t tunnel.Found) *link.Tunnel {
|
func carried(t tunnel.Found) *link.Tunnel {
|
||||||
out := &link.Tunnel{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port,
|
out := &link.Tunnel{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port,
|
||||||
@@ -934,7 +1019,7 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
|
|||||||
// And the tunnel the private network took over, as this apply found it (novox/hq ADR 0105).
|
// And the tunnel the private network took over, as this apply found it (novox/hq ADR 0105).
|
||||||
if t := outcome.Tunnel; t != nil {
|
if t := outcome.Tunnel; t != nil {
|
||||||
report.Tunnel = &link.CarriedTunnel{Interface: t.Interface, Port: t.Port, Range: t.Range,
|
report.Tunnel = &link.CarriedTunnel{Interface: t.Interface, Port: t.Port, Range: t.Range,
|
||||||
Peers: t.Peers, Taken: t.Taken, Kept: t.Kept}
|
Peers: t.Peers, State: t.State, Note: t.Note, Kept: t.Kept}
|
||||||
}
|
}
|
||||||
reached, err := reachable.Collect(ctx, apply.ExecRunner)
|
reached, err := reachable.Collect(ctx, apply.ExecRunner)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/ed25519"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/tunnel"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0105: `overlay take` moves this node's overlay key onto the found tunnel's and
|
||||||
|
// nothing else — identity, sealing and serving keys stay as they were — and tells the mesh with a
|
||||||
|
// proof signed by the identity key, over the previous key, the new one and the tunnel.
|
||||||
|
|
||||||
|
func anEnrolledNode(t *testing.T) identity.Identity {
|
||||||
|
t.Helper()
|
||||||
|
mine, err := identity.Generate("anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mine.Overlay, err = identity.GenerateOverlayKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mine.Membership = identity.Membership{Broker: "198.51.100.1:5671", Fingerprint: "sha256:aa",
|
||||||
|
Signer: make([]byte, ed25519.PublicKeySize), Password: "p"}
|
||||||
|
return mine
|
||||||
|
}
|
||||||
|
|
||||||
|
func aFoundTunnel(t *testing.T) tunnel.Found {
|
||||||
|
t.Helper()
|
||||||
|
private, err := identity.GenerateOverlayKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
found, err := tunnel.Parse([]byte("[Interface]\nPrivateKey = " + private.Private + "\nListenPort = 51900\n" +
|
||||||
|
"Address = 192.0.2.1/24\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
found.Interface, found.Unit, found.Config = "wg0", "wg-quick@wg0", "/etc/wireguard/wg0.conf"
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTakingATunnelMovesOnlyTheOverlayKeyAndSignsForIt(t *testing.T) {
|
||||||
|
mine := anEnrolledNode(t)
|
||||||
|
found := aFoundTunnel(t)
|
||||||
|
before := mine.Overlay.Public
|
||||||
|
|
||||||
|
taken, rekey, err := rekeyOnto(mine, found)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if taken.Overlay.Public != found.PublicKey || taken.Overlay.Private != found.PrivateKey() {
|
||||||
|
t.Fatal("the overlay key is not the tunnel's")
|
||||||
|
}
|
||||||
|
if string(taken.Public) != string(mine.Public) || string(taken.Private) != string(mine.Private) ||
|
||||||
|
taken.Node != mine.Node || taken.Membership.Password != mine.Membership.Password ||
|
||||||
|
taken.Membership.Broker != mine.Membership.Broker {
|
||||||
|
t.Fatal("something other than the overlay key moved")
|
||||||
|
}
|
||||||
|
if taken.OverlayBefore != before {
|
||||||
|
t.Errorf("the key before the take was not kept: %q", taken.OverlayBefore)
|
||||||
|
}
|
||||||
|
if rekey.Previous != before || rekey.OverlayKey != found.PublicKey || rekey.Tunnel == nil ||
|
||||||
|
rekey.Tunnel.PublicKey != found.PublicKey || len(rekey.Tunnel.Peers) != 1 {
|
||||||
|
t.Fatalf("the rekey does not say what moved: %+v", rekey)
|
||||||
|
}
|
||||||
|
if !ed25519.Verify(ed25519.PublicKey(mine.Public),
|
||||||
|
link.RekeyProof("anchor", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
|
||||||
|
t.Fatal("the rekey is not signed by this node's identity key over what it says")
|
||||||
|
}
|
||||||
|
if ed25519.Verify(ed25519.PublicKey(mine.Public),
|
||||||
|
link.RekeyProof("laptop", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
|
||||||
|
t.Fatal("the proof is not bound to the node")
|
||||||
|
}
|
||||||
|
for _, said := range []string{rekey.Previous, rekey.OverlayKey, rekey.Tunnel.Interface} {
|
||||||
|
if strings.Contains(said, found.PrivateKey()) {
|
||||||
|
t.Fatal("the private key travels")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run again after the take — the mesh not yet told, or told and refused — the previous key it
|
||||||
|
// names is still the one before the take, so the mesh can tell a repeat from a replay.
|
||||||
|
again, second, err := rekeyOnto(taken, found)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if second.Previous != before || again.OverlayBefore != before || again.Overlay.Public != found.PublicKey {
|
||||||
|
t.Fatalf("a take run again does not name the key before the first: %+v", second)
|
||||||
|
}
|
||||||
|
}
|
||||||
+25
-4
@@ -339,22 +339,32 @@ func ApplyKeeping(
|
|||||||
// it (novox/hq ADR 0105): its configuration kept, its unit stopped and disabled, never
|
// it (novox/hq ADR 0105): its configuration kept, its unit stopped and disabled, never
|
||||||
// flushed. A failure here fails the service too — the mesh's interface is not started on a
|
// flushed. A failure here fails the service too — the mesh's interface is not started on a
|
||||||
// port the found one still holds.
|
// port the found one still holds.
|
||||||
|
stoppedFound := false
|
||||||
if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil {
|
if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil {
|
||||||
var outcome Outcome
|
var outcome Outcome
|
||||||
var facts TakenTunnel
|
var facts TakenTunnel
|
||||||
var err error
|
var err error
|
||||||
if d.Adoption == nil {
|
if d.Adoption == nil {
|
||||||
err = errNotAdopted
|
err = errNotAdopted
|
||||||
|
facts = TakenTunnel{Interface: svc.TakesOver.Interface, State: NotTaken}
|
||||||
} else {
|
} else {
|
||||||
outcome, facts, err = takeOver(ctx, sys, svc, d, &known, run, keep, time.Now().UTC())
|
outcome, facts, stoppedFound, err = takeOver(ctx, sys, svc, d, &known, run, keep, time.Now().UTC())
|
||||||
}
|
}
|
||||||
|
// Always an account, failure included: the last account standing must never be an
|
||||||
|
// older "taken" over a machine whose takeover has since gone wrong.
|
||||||
|
report.Tunnel = &facts
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
report.Tunnel.Note = err.Error()
|
||||||
|
if stoppedFound {
|
||||||
|
// The found unit is down and the mesh's not up: the one state where the
|
||||||
|
// peers reach nothing. Started again, and said.
|
||||||
|
restoreFound(ctx, sys, svc.TakesOver.Unit, run, report.Tunnel)
|
||||||
|
}
|
||||||
failures = append(failures, &Error{Resource: svc.Identity(), Err: err, Done: report})
|
failures = append(failures, &Error{Resource: svc.Identity(), Err: err, Done: report})
|
||||||
log(fmt.Sprintf(" failed %s (%s): %v", svc.Identity(), svc.Unit, err))
|
log(fmt.Sprintf(" failed %s (%s): %v", svc.Identity(), svc.Unit, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
report.Outcomes = append(report.Outcomes, outcome)
|
report.Outcomes = append(report.Outcomes, outcome)
|
||||||
report.Tunnel = &facts
|
|
||||||
log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail))
|
log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -377,6 +387,17 @@ func ApplyKeeping(
|
|||||||
failed := &Error{Resource: resource.Identity(), Err: err, Done: report}
|
failed := &Error{Resource: resource.Identity(), Err: err, Done: report}
|
||||||
failures = append(failures, failed)
|
failures = append(failures, failed)
|
||||||
log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), outcome.Target, err))
|
log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), outcome.Target, err))
|
||||||
|
if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil {
|
||||||
|
// The mesh's interface did not come up after the found one was stopped: no
|
||||||
|
// tunnel at all. The found unit is started again — the machine goes back to
|
||||||
|
// what it had — and the account says so (novox/hq ADR 0105).
|
||||||
|
report.Tunnel.Note = "the mesh's interface did not come up: " + err.Error()
|
||||||
|
if stoppedFound {
|
||||||
|
restoreFound(ctx, sys, svc.TakesOver.Unit, run, report.Tunnel)
|
||||||
|
} else {
|
||||||
|
report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// **A failed action stops what follows. Nothing else does.**
|
// **A failed action stops what follows. Nothing else does.**
|
||||||
//
|
//
|
||||||
@@ -426,8 +447,8 @@ func ApplyKeeping(
|
|||||||
}
|
}
|
||||||
if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil {
|
if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil {
|
||||||
// The found interface is down and the mesh's is up in its place: the tunnel changed
|
// The found interface is down and the mesh's is up in its place: the tunnel changed
|
||||||
// hands (novox/hq ADR 0105).
|
// hands (novox/hq ADR 0105). Read from the machine, not assumed.
|
||||||
report.Tunnel.Taken = true
|
report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run)
|
||||||
}
|
}
|
||||||
report.Outcomes = append(report.Outcomes, outcome)
|
report.Outcomes = append(report.Outcomes, outcome)
|
||||||
if outcome.Action != "unchanged" {
|
if outcome.Action != "unchanged" {
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ type machine struct {
|
|||||||
|
|
||||||
type fakeUnit struct {
|
type fakeUnit struct {
|
||||||
active, enabled string
|
active, enabled string
|
||||||
|
// wontStart is a unit that accepts `start` and stays inactive — one that starts and dies.
|
||||||
|
wontStart bool
|
||||||
// fragment is where systemd loads the unit from; empty means /etc/systemd/system, where an
|
// fragment is where systemd loads the unit from; empty means /etc/systemd/system, where an
|
||||||
// administrator installs one.
|
// administrator installs one.
|
||||||
fragment string
|
fragment string
|
||||||
@@ -65,7 +67,9 @@ func (m *machine) systemctl(args []string) (string, error) {
|
|||||||
}
|
}
|
||||||
return u.enabled + "\n", nil
|
return u.enabled + "\n", nil
|
||||||
case "start":
|
case "start":
|
||||||
u.active = "active"
|
if !u.wontStart {
|
||||||
|
u.active = "active"
|
||||||
|
}
|
||||||
case "stop":
|
case "stop":
|
||||||
u.active = "inactive"
|
u.active = "inactive"
|
||||||
case "enable":
|
case "enable":
|
||||||
|
|||||||
+229
-37
@@ -34,26 +34,53 @@ type TakenTunnel struct {
|
|||||||
Port int
|
Port int
|
||||||
Range string
|
Range string
|
||||||
Peers int
|
Peers int
|
||||||
// Taken is whether the found interface is down and disabled and the mesh's up in its place.
|
// State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one
|
||||||
Taken bool
|
// down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's
|
||||||
|
// not up: the peers reach nothing). Note is what this apply did about it.
|
||||||
|
State string
|
||||||
|
Note string
|
||||||
Kept string
|
Kept string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The states, as the link says them.
|
||||||
|
const (
|
||||||
|
NotTaken = "not-taken"
|
||||||
|
Taken = "taken"
|
||||||
|
TunnelDown = "down"
|
||||||
|
)
|
||||||
|
|
||||||
|
// takeoverRecheck is how often, and takeoverRechecks how many times, a found interface still up
|
||||||
|
// after its unit stopped is looked at again before the takeover is refused: `wg-quick down` by a
|
||||||
|
// person takes a moment. Variables so a test need not wait.
|
||||||
|
var (
|
||||||
|
takeoverRecheck = 2 * time.Second
|
||||||
|
takeoverRechecks = 3
|
||||||
|
)
|
||||||
|
|
||||||
// takeOverID is the held record's id for the found configuration: the service's own with a suffix,
|
// takeOverID is the held record's id for the found configuration: the service's own with a suffix,
|
||||||
// so it is declared for as long as the service is and never mistaken for the service itself.
|
// so it is declared for as long as the service is and never mistaken for the service itself.
|
||||||
func takeOverID(svc *declaration.Service) string { return svc.ID + ".takes-over" }
|
func takeOverID(svc *declaration.Service) string { return svc.ID + ".takes-over" }
|
||||||
|
|
||||||
// takeOver keeps the found tunnel's configuration and stops its unit, ahead of the service that
|
// takeOver keeps the found tunnel's configuration and stops its unit, ahead of the service that
|
||||||
// replaces it. Returned is the hold's outcome, and what was found for the report.
|
// replaces it. Returned is the hold's outcome, and what was found for the report.
|
||||||
|
//
|
||||||
|
// **Nothing is stopped until the mesh's interface is known to be able to replace it** (the record's
|
||||||
|
// option 2 is exactly this going wrong): the declared configuration must listen on the found port
|
||||||
|
// at the found address, and the key file it points at must hold the found key. Only then is the
|
||||||
|
// found unit stopped — and `stopped` says whether this apply did, so a mesh interface that then
|
||||||
|
// fails to start can have the found unit started again.
|
||||||
func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, d *declaration.Declaration,
|
func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, d *declaration.Declaration,
|
||||||
known *store.State, run Runner, keep Keep, now time.Time) (Outcome, TakenTunnel, error) {
|
known *store.State, run Runner, keep Keep, now time.Time) (out Outcome, facts TakenTunnel, stopped bool, err error) {
|
||||||
t := svc.TakesOver
|
t := svc.TakesOver
|
||||||
id := takeOverID(svc)
|
id := takeOverID(svc)
|
||||||
module, _ := d.Adoption.UntakenModuleOf(svc.ID)
|
module, _ := d.Adoption.UntakenModuleOf(svc.ID)
|
||||||
if module == "" {
|
if module == "" {
|
||||||
module = "the private network"
|
module = "the private network"
|
||||||
}
|
}
|
||||||
facts := TakenTunnel{Interface: t.Interface}
|
facts = TakenTunnel{Interface: t.Interface, State: NotTaken}
|
||||||
|
|
||||||
|
// 0. What the found configuration says, before anything: the checks below are against it.
|
||||||
|
found, ferr := readFoundTunnel(t.Config)
|
||||||
|
|
||||||
// 1. The configuration, kept like any held file. A synthetic file resource stands for it, so
|
// 1. The configuration, kept like any held file. A synthetic file resource stands for it, so
|
||||||
// the same code keeps its original, digests it and notices it changing.
|
// the same code keeps its original, digests it and notices it changing.
|
||||||
@@ -62,65 +89,97 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service,
|
|||||||
out, held, err := hold(ctx, sys, file, module, was, already,
|
out, held, err := hold(ctx, sys, file, module, was, already,
|
||||||
"the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now)
|
"the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return begin(file), facts, fmt.Errorf("keeping the found tunnel's configuration: %w", err)
|
return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err)
|
||||||
}
|
}
|
||||||
known.RecordHeld(held)
|
known.RecordHeld(held)
|
||||||
facts.Kept = held.Kept
|
facts.Kept = held.Kept
|
||||||
// What the file says, for the report: read from the machine, or from the kept original when
|
// What the file says, for the report: from the machine, or from the kept original when the
|
||||||
// the machine's copy is gone. The private key stays in the file; nothing here keeps it.
|
// machine's copy is gone. The private key stays in the file; nothing here keeps it.
|
||||||
unread := ""
|
unread := ""
|
||||||
raw, err := os.ReadFile(t.Config)
|
if ferr != nil && held.Kept != "" {
|
||||||
if err != nil && held.Kept != "" {
|
found, ferr = readFoundTunnel(held.Kept)
|
||||||
raw, err = os.ReadFile(held.Kept)
|
|
||||||
}
|
}
|
||||||
if err == nil {
|
if ferr == nil {
|
||||||
if found, perr := tunnel.Parse(raw); perr == nil {
|
facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers)
|
||||||
facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers)
|
|
||||||
} else {
|
|
||||||
unread = perr.Error()
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
unread = err.Error()
|
unread = ferr.Error()
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. The found unit: stopped if it runs, disabled if it starts at boot. A unit that is not
|
// 2. Where things stand: the found unit, and the mesh's.
|
||||||
// there is not an error — the interface may have been raised another way, which the check
|
foundState, unitErr := sys.ServiceState(ctx, run, t.Unit)
|
||||||
// below catches — and neither is one already down.
|
meshState, _ := sys.ServiceState(ctx, run, svc.Unit)
|
||||||
|
if foundState == "running" && meshState == "running" {
|
||||||
|
// Both up. On the hub this cannot last — the found unit cannot bind the port the mesh's
|
||||||
|
// holds — and on a spoke two interfaces with one key flap between them. Not stopped again
|
||||||
|
// by the mesh: what is found on an adopted node is reported, and the first takeover was
|
||||||
|
// the one act (the PR note says why). Said, so a person sees it.
|
||||||
|
facts.Note = t.Unit + " is running again beside the mesh's interface; not stopped by the mesh — " +
|
||||||
|
"`systemctl stop " + t.Unit + "` on the machine"
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Before the found unit is stopped: can the mesh's interface replace it? Its declared
|
||||||
|
// configuration must listen on the found port at the found address, and the key file it
|
||||||
|
// points at must hold the found key, or the peers would be dropped the moment it came up.
|
||||||
|
if foundState == "running" && meshState != "running" {
|
||||||
|
if ferr != nil {
|
||||||
|
return out, facts, false, fmt.Errorf("the found tunnel's configuration at %s cannot be read as a "+
|
||||||
|
"tunnel's (%v), so nothing says what the mesh's interface must match; %s is left running",
|
||||||
|
t.Config, ferr, t.Unit)
|
||||||
|
}
|
||||||
|
if err := replaces(d, svc, found); err != nil {
|
||||||
|
return out, facts, false, fmt.Errorf("%w; %s is left running", err, t.Unit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. The found unit: stopped if it runs and the mesh's does not, disabled if it starts at
|
||||||
|
// boot. A unit that is not there is not an error — the interface may have been raised
|
||||||
|
// another way, which the check below catches — and neither is one already down.
|
||||||
var did []string
|
var did []string
|
||||||
state, err := sys.ServiceState(ctx, run, t.Unit)
|
|
||||||
switch {
|
switch {
|
||||||
case err != nil:
|
case unitErr != nil:
|
||||||
did = append(did, t.Unit+" is not a unit here")
|
did = append(did, t.Unit+" is not a unit here")
|
||||||
case state == "running":
|
case foundState == "running" && meshState != "running":
|
||||||
if err := sys.SetServiceState(ctx, run, t.Unit, "stopped"); err != nil {
|
if err := sys.SetServiceState(ctx, run, t.Unit, "stopped"); err != nil {
|
||||||
return out, facts, fmt.Errorf("stopping the found %s: %w", t.Unit, err)
|
return out, facts, false, fmt.Errorf("stopping the found %s: %w", t.Unit, err)
|
||||||
}
|
}
|
||||||
after, err := sys.ServiceState(ctx, run, t.Unit)
|
after, err := sys.ServiceState(ctx, run, t.Unit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return out, facts, err
|
return out, facts, true, err
|
||||||
}
|
}
|
||||||
if after != "stopped" {
|
if after != "stopped" {
|
||||||
return out, facts, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after)
|
return out, facts, true, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after)
|
||||||
}
|
}
|
||||||
|
stopped = true
|
||||||
did = append(did, "stopped "+t.Unit)
|
did = append(did, "stopped "+t.Unit)
|
||||||
}
|
}
|
||||||
if err == nil {
|
if unitErr == nil {
|
||||||
if boot, err := sys.ServiceBoot(ctx, run, t.Unit); err == nil && boot == "enabled" {
|
if boot, err := sys.ServiceBoot(ctx, run, t.Unit); err == nil && boot == "enabled" {
|
||||||
if err := sys.SetServiceBoot(ctx, run, t.Unit, "disabled"); err != nil {
|
if err := sys.SetServiceBoot(ctx, run, t.Unit, "disabled"); err != nil {
|
||||||
return out, facts, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err)
|
return out, facts, stopped, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err)
|
||||||
}
|
}
|
||||||
did = append(did, "disabled it at boot")
|
did = append(did, "disabled it at boot")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. The interface is gone. If it is still up, something other than its unit raised it, and
|
// 5. The interface is gone. If it is still up, something other than its unit raised it —
|
||||||
// starting the mesh's on the same port and address would fail or, worse, half work.
|
// the predecessor brings its up by hand — and the mesh's interface cannot take its port
|
||||||
if up, err := run(ctx, "wg", "show", "interfaces"); err == nil {
|
// and address while it is. Looked at again for a moment, since a person taking it down
|
||||||
for _, iface := range strings.Fields(up) {
|
// takes a moment; then refused, naming what to do.
|
||||||
if iface == t.Interface {
|
if meshState != "running" {
|
||||||
return out, facts, fmt.Errorf("%s is still up after its unit %s was stopped: something other "+
|
for try := 0; ; try++ {
|
||||||
"than that unit raises it, and the mesh's interface cannot take its port and address "+
|
if !interfaceUp(ctx, run, t.Interface) {
|
||||||
"while it does. Nothing was flushed", t.Interface, t.Unit)
|
break
|
||||||
|
}
|
||||||
|
if try >= takeoverRechecks {
|
||||||
|
return out, facts, stopped, fmt.Errorf("%s is still up although its unit %s is not running: it was "+
|
||||||
|
"raised by hand, not by its unit, and the mesh's interface cannot take its port and "+
|
||||||
|
"address while it is. On the machine: `wg-quick down %s` — the next reconcile takes it "+
|
||||||
|
"over. Nothing was flushed", t.Interface, t.Unit, t.Interface)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return out, facts, stopped, ctx.Err()
|
||||||
|
case <-time.After(takeoverRecheck):
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -140,7 +199,140 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service,
|
|||||||
// peers was carried, which reads as a tunnel that was not one.
|
// peers was carried, which reads as a tunnel that was not one.
|
||||||
out.Detail += "; what it says could not be read as a tunnel's: " + unread
|
out.Detail += "; what it says could not be read as a tunnel's: " + unread
|
||||||
}
|
}
|
||||||
return out, facts, nil
|
return out, facts, stopped, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readFoundTunnel is the found configuration as a tunnel.
|
||||||
|
func readFoundTunnel(path string) (tunnel.Found, error) {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return tunnel.Found{}, err
|
||||||
|
}
|
||||||
|
return tunnel.Parse(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// interfaceUp is whether a WireGuard interface is up on the machine.
|
||||||
|
func interfaceUp(ctx context.Context, run Runner, iface string) bool {
|
||||||
|
up, err := run(ctx, "wg", "show", "interfaces")
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, name := range strings.Fields(up) {
|
||||||
|
if name == iface {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// replaces holds the mesh's declared interface configuration against the found tunnel it is to
|
||||||
|
// replace: same port, same address, and a key file holding the found key. The configuration is
|
||||||
|
// the file the service restarts on; its `PostUp = wg set %i private-key <path>` names the key.
|
||||||
|
func replaces(d *declaration.Declaration, svc *declaration.Service, found tunnel.Found) error {
|
||||||
|
var conf *declaration.File
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
f, ok := r.(*declaration.File)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, id := range svc.RestartOn {
|
||||||
|
if f.ID == id {
|
||||||
|
conf = f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if conf == nil {
|
||||||
|
return fmt.Errorf("%s takes over %s and restarts on no declared file, so the interface it would "+
|
||||||
|
"raise cannot be checked against the found one", svc.Unit, found.Interface)
|
||||||
|
}
|
||||||
|
port, address, keyPath := "", "", ""
|
||||||
|
for _, line := range strings.Split(conf.Content, "\n") {
|
||||||
|
key, value, ok := strings.Cut(strings.TrimSpace(line), "=")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value)
|
||||||
|
switch key {
|
||||||
|
case "listenport":
|
||||||
|
port = value
|
||||||
|
case "address":
|
||||||
|
address = strings.TrimSpace(strings.Split(value, ",")[0])
|
||||||
|
case "postup":
|
||||||
|
if _, after, ok := strings.Cut(value, "private-key "); ok {
|
||||||
|
keyPath = strings.Fields(after)[0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var wrong []string
|
||||||
|
if port != fmt.Sprint(found.Port) {
|
||||||
|
wrong = append(wrong, fmt.Sprintf("it listens on port %q and the tunnel on %d", port, found.Port))
|
||||||
|
}
|
||||||
|
if host(address) != host(found.Address) {
|
||||||
|
wrong = append(wrong, fmt.Sprintf("its address is %q and the tunnel's %s", address, found.Address))
|
||||||
|
}
|
||||||
|
switch raw, err := os.ReadFile(keyPath); {
|
||||||
|
case keyPath == "":
|
||||||
|
wrong = append(wrong, "it names no key file")
|
||||||
|
case err != nil:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("its key file %s cannot be read (%v)", keyPath, err))
|
||||||
|
default:
|
||||||
|
public, perr := tunnel.PublicKeyOf(strings.TrimSpace(string(raw)))
|
||||||
|
if perr != nil || public != found.PublicKey {
|
||||||
|
wrong = append(wrong, fmt.Sprintf("the key at %s is not the tunnel's — `mesh-host overlay take "+
|
||||||
|
"--tunnel %s` on this machine takes it, then push again", keyPath, found.Interface))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(wrong) > 0 {
|
||||||
|
return fmt.Errorf("the mesh's interface would not replace the tunnel on %s: %s — the peers would be "+
|
||||||
|
"dropped the moment it came up. Re-place the hub on the tunnel's address and port and push again",
|
||||||
|
found.Interface, strings.Join(wrong, "; "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// host is an address without its prefix length.
|
||||||
|
func host(address string) string {
|
||||||
|
if i := strings.Index(address, "/"); i >= 0 {
|
||||||
|
return address[:i]
|
||||||
|
}
|
||||||
|
return address
|
||||||
|
}
|
||||||
|
|
||||||
|
// tunnelState is where the tunnel stands, read from the machine: the found unit or interface up
|
||||||
|
// and the mesh's not is not taken; the mesh's up and the found one down is taken; neither up is
|
||||||
|
// down — the peers reach nothing.
|
||||||
|
func tunnelState(ctx context.Context, sys system.System, foundUnit, meshUnit string, run Runner) string {
|
||||||
|
foundState, _ := sys.ServiceState(ctx, run, foundUnit)
|
||||||
|
meshState, _ := sys.ServiceState(ctx, run, meshUnit)
|
||||||
|
foundUp := foundState == "running" || interfaceUp(ctx, run, strings.TrimPrefix(foundUnit, "wg-quick@"))
|
||||||
|
switch {
|
||||||
|
case meshState == "running" && !foundUp:
|
||||||
|
return Taken
|
||||||
|
case meshState == "running":
|
||||||
|
// Both up: not a takeover that holds, and said as not taken so nobody reads it as one.
|
||||||
|
return NotTaken
|
||||||
|
case foundUp:
|
||||||
|
return NotTaken
|
||||||
|
default:
|
||||||
|
return TunnelDown
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// restoreFound starts the found unit again after the mesh's interface failed to replace it, so the
|
||||||
|
// machine has the tunnel it had rather than none, and says so in the account.
|
||||||
|
func restoreFound(ctx context.Context, sys system.System, unit string, run Runner, facts *TakenTunnel) {
|
||||||
|
if err := sys.SetServiceState(ctx, run, unit, "running"); err != nil {
|
||||||
|
facts.State = TunnelDown
|
||||||
|
facts.Note += "; " + unit + " could not be started again (" + err.Error() + ") — on the machine: systemctl start " + unit
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if state, err := sys.ServiceState(ctx, run, unit); err != nil || state != "running" {
|
||||||
|
facts.State = TunnelDown
|
||||||
|
facts.Note += "; " + unit + " was started again and is not running — on the machine: systemctl start " + unit
|
||||||
|
return
|
||||||
|
}
|
||||||
|
facts.State = NotTaken
|
||||||
|
facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had"
|
||||||
}
|
}
|
||||||
|
|
||||||
// takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since
|
// takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since
|
||||||
|
|||||||
+124
-33
@@ -14,53 +14,63 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// novox/hq ADR 0105: the host raises the mesh's interface with the found key and peers, stops the
|
// novox/hq ADR 0105: the host raises the mesh's interface with the found key and peers, stops the
|
||||||
// found interface without flushing it, and keeps its configuration.
|
// found interface without flushing it, and keeps its configuration — and stops nothing until the
|
||||||
|
// mesh's interface is known to be able to replace it.
|
||||||
|
|
||||||
// foundConf is the predecessor's configuration, with a real key made once per run: the key is
|
// foundKey is the predecessor's private key, a real one made once per run: the key is what the
|
||||||
// what the takeover must never print or copy, so it had better be one.
|
// takeover must never print or copy, so it had better be one.
|
||||||
var foundConf = func() string {
|
var foundKey = func() string {
|
||||||
k, err := ecdh.X25519().GenerateKey(rand.Reader)
|
k, err := ecdh.X25519().GenerateKey(rand.Reader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
return "[Interface]\nPrivateKey = " + base64.StdEncoding.EncodeToString(k.Bytes()) + "\n" +
|
return base64.StdEncoding.EncodeToString(k.Bytes())
|
||||||
"ListenPort = 51900\nAddress = 192.0.2.1/24\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n" +
|
|
||||||
"\n[Peer]\nPublicKey = PEER-B=\nAllowedIPs = 192.0.2.3/32\n"
|
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
var foundConf = "[Interface]\nPrivateKey = " + foundKey + "\n" +
|
||||||
|
"ListenPort = 51900\nAddress = 192.0.2.1/24\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n" +
|
||||||
|
"\n[Peer]\nPublicKey = PEER-B=\nAllowedIPs = 192.0.2.3/32\n"
|
||||||
|
|
||||||
// aTakeover is the private network's declaration for an adopted hub whose interface takes over
|
// aTakeover is the private network's declaration for an adopted hub whose interface takes over
|
||||||
// the found tunnel: the mesh's configuration — with the found key set from the node's own key file
|
// the found tunnel: the mesh's configuration — on the found port and address, its key set from the
|
||||||
// and the found peers in its list — and the interface's service naming what it replaces.
|
// node's own key file, the found peers in its list — and the interface's service naming what it
|
||||||
func aTakeover(t *testing.T, config, mesh string) *declaration.Declaration {
|
// replaces. Port and address are parameters so a test can declare a wrong one.
|
||||||
|
func aTakeover(t *testing.T, config, mesh, keyFile, port, address string) *declaration.Declaration {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
return adopted(t,
|
return adopted(t,
|
||||||
`{"taken":[],"untaken":{"mesh-wireguard":["mesh-wireguard.overlay-config","mesh-wireguard.overlay-up"]}}`,
|
`{"taken":[],"untaken":{"mesh-wireguard":["mesh-wireguard.overlay-config","mesh-wireguard.overlay-up"]}}`,
|
||||||
`{"id":"mesh-wireguard.overlay-config","type":"file","path":"`+mesh+`","mode":"0600",
|
`{"id":"mesh-wireguard.overlay-config","type":"file","path":"`+mesh+`","mode":"0600",
|
||||||
"content":"[Interface]\nAddress = 192.0.2.1/32\nListenPort = 51900\nPostUp = wg set %i private-key /var/lib/mesh-host/overlay.key\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"},
|
"content":"[Interface]\nAddress = `+address+`/32\nListenPort = `+port+`\nPostUp = wg set %i private-key `+keyFile+`\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"},
|
||||||
{"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0","state":"running","boot":"enabled",
|
{"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0","state":"running","boot":"enabled",
|
||||||
"restart-on":["mesh-wireguard.overlay-config"],
|
"restart-on":["mesh-wireguard.overlay-config"],
|
||||||
"takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"`+config+`"}}`)
|
"takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"`+config+`"}}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
// aHubInUse is a machine with the predecessor's tunnel up and the mesh's not yet.
|
// aHubInUse is a machine with the predecessor's tunnel up and the mesh's not yet: the found
|
||||||
func aHubInUse(t *testing.T) (dir, config, mesh string, m *machine) {
|
// configuration on disk, and the node's key file holding the found key, as enrolment left it.
|
||||||
|
func aHubInUse(t *testing.T) (dir, config, mesh, keyFile string, m *machine) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dir = t.TempDir()
|
dir = t.TempDir()
|
||||||
config = filepath.Join(dir, "wg0.conf")
|
config = filepath.Join(dir, "wg0.conf")
|
||||||
mesh = filepath.Join(dir, "mesh0.conf")
|
mesh = filepath.Join(dir, "mesh0.conf")
|
||||||
|
keyFile = filepath.Join(dir, "overlay.key")
|
||||||
if err := os.WriteFile(config, []byte(foundConf), 0o600); err != nil {
|
if err := os.WriteFile(config, []byte(foundConf), 0o600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
if err := os.WriteFile(keyFile, []byte(foundKey+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
m = &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{
|
m = &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{
|
||||||
"wg-quick@wg0": {active: "active", enabled: "enabled"},
|
"wg-quick@wg0": {active: "active", enabled: "enabled"},
|
||||||
"wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"},
|
"wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"},
|
||||||
}}
|
}}
|
||||||
return dir, config, mesh, m
|
takeoverRecheck = 0
|
||||||
|
return dir, config, mesh, keyFile, m
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T) {
|
func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T) {
|
||||||
dir, config, mesh, m := aHubInUse(t)
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
report, state := applyAdopted(t, aTakeover(t, config, mesh), store.State{}, m, dir)
|
report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)
|
||||||
|
|
||||||
// The found interface: its unit stopped and disabled, and nothing else done to it.
|
// The found interface: its unit stopped and disabled, and nothing else done to it.
|
||||||
if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" {
|
if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" {
|
||||||
@@ -93,16 +103,18 @@ func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T
|
|||||||
t.Fatalf("the mesh's configuration does not carry the found peer, or carries a key:\n%s", got)
|
t.Fatalf("the mesh's configuration does not carry the found peer, or carries a key:\n%s", got)
|
||||||
}
|
}
|
||||||
// And the report says so, with what was found — port, range, peers — and never the key.
|
// And the report says so, with what was found — port, range, peers — and never the key.
|
||||||
if report.Tunnel == nil || !report.Tunnel.Taken || report.Tunnel.Port != 51900 ||
|
if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Port != 51900 ||
|
||||||
report.Tunnel.Range != "192.0.2.0/24" || report.Tunnel.Peers != 2 || report.Tunnel.Kept != held.Kept {
|
report.Tunnel.Range != "192.0.2.0/24" || report.Tunnel.Peers != 2 || report.Tunnel.Kept != held.Kept {
|
||||||
t.Fatalf("the report does not say what was carried: %+v", report.Tunnel)
|
t.Fatalf("the report does not say what was carried: %+v", report.Tunnel)
|
||||||
}
|
}
|
||||||
private := strings.TrimSpace(strings.SplitN(strings.SplitN(foundConf, "PrivateKey = ", 2)[1], "\n", 2)[0])
|
|
||||||
for _, o := range report.Outcomes {
|
for _, o := range report.Outcomes {
|
||||||
if strings.Contains(o.Detail, private) {
|
if strings.Contains(o.Detail, foundKey) {
|
||||||
t.Errorf("the found key was printed in an outcome: %+v", o)
|
t.Errorf("the found key was printed in an outcome: %+v", o)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if strings.Contains(report.Tunnel.Note, foundKey) {
|
||||||
|
t.Error("the found key was printed in the account")
|
||||||
|
}
|
||||||
if o := outcomeOf(report, "mesh-wireguard.overlay-up.takes-over"); o.Action != "held" ||
|
if o := outcomeOf(report, "mesh-wireguard.overlay-up.takes-over"); o.Action != "held" ||
|
||||||
!strings.Contains(o.Detail, "stopped wg-quick@wg0") || !strings.Contains(o.Detail, "never flushed") {
|
!strings.Contains(o.Detail, "stopped wg-quick@wg0") || !strings.Contains(o.Detail, "never flushed") {
|
||||||
t.Errorf("the takeover was not reported as a hold that stopped the found unit: %+v", o)
|
t.Errorf("the takeover was not reported as a hold that stopped the found unit: %+v", o)
|
||||||
@@ -112,9 +124,67 @@ func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestATakeoverIsSteadyAndTheFoundUnitStaysDown(t *testing.T) {
|
func TestNothingIsStoppedUntilTheMeshsInterfaceCanReplaceTheFoundOne(t *testing.T) {
|
||||||
dir, config, mesh, m := aHubInUse(t)
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
d := aTakeover(t, config, mesh)
|
otherKey := filepath.Join(dir, "other.key")
|
||||||
|
k, _ := ecdh.X25519().GenerateKey(rand.Reader)
|
||||||
|
if err := os.WriteFile(otherKey, []byte(base64.StdEncoding.EncodeToString(k.Bytes())+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cases := map[string]*declaration.Declaration{
|
||||||
|
"another port": aTakeover(t, config, mesh, keyFile, "51821", "192.0.2.1"),
|
||||||
|
"another address": aTakeover(t, config, mesh, keyFile, "51900", "10.42.0.1"),
|
||||||
|
"another key": aTakeover(t, config, mesh, otherKey, "51900", "192.0.2.1"),
|
||||||
|
"no key file": aTakeover(t, config, mesh, filepath.Join(dir, "missing.key"), "51900", "192.0.2.1"),
|
||||||
|
}
|
||||||
|
for name, d := range cases {
|
||||||
|
m.asked = nil
|
||||||
|
report, state, err := ApplyKeeping(t.Context(), archHost(t), d, store.State{},
|
||||||
|
store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "would not replace the tunnel") {
|
||||||
|
t.Fatalf("%s: the takeover was not refused: %v", name, err)
|
||||||
|
}
|
||||||
|
if name == "another key" && !strings.Contains(err.Error(), "overlay take") {
|
||||||
|
t.Errorf("%s: the refusal does not name the remedy: %v", name, err)
|
||||||
|
}
|
||||||
|
if m.units["wg-quick@wg0"].active != "active" || m.did("systemctl stop wg-quick@wg0") {
|
||||||
|
t.Fatalf("%s: the found unit was stopped although the mesh's interface could not replace it", name)
|
||||||
|
}
|
||||||
|
if m.units["wg-quick@mesh0"].active == "active" {
|
||||||
|
t.Fatalf("%s: the mesh's interface was started on top of the found one", name)
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "would not replace") {
|
||||||
|
t.Fatalf("%s: the account does not say the tunnel is not taken and why: %+v", name, report.Tunnel)
|
||||||
|
}
|
||||||
|
if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held {
|
||||||
|
t.Errorf("%s: the found configuration was not kept before the refusal", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMeshInterfaceThatFailsToStartGivesTheFoundOneBack(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
m.units["wg-quick@mesh0"].wontStart = true
|
||||||
|
report, _, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"),
|
||||||
|
store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a mesh interface that did not come up was reported as applied")
|
||||||
|
}
|
||||||
|
if !m.did("systemctl stop wg-quick@wg0") || !m.did("systemctl start wg-quick@wg0") {
|
||||||
|
t.Fatalf("the found unit was not stopped and then started again: %v", m.asked)
|
||||||
|
}
|
||||||
|
if m.units["wg-quick@wg0"].active != "active" {
|
||||||
|
t.Fatal("the machine was left with no tunnel at all")
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != NotTaken ||
|
||||||
|
!strings.Contains(report.Tunnel.Note, "did not come up") || !strings.Contains(report.Tunnel.Note, "started again") {
|
||||||
|
t.Fatalf("the account does not say the mesh's interface failed and the found one was given back: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATakeoverIsSteadyAndAFoundUnitUpAgainIsSaidNotStopped(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
|
||||||
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
m.asked = nil
|
m.asked = nil
|
||||||
|
|
||||||
@@ -128,28 +198,49 @@ func TestATakeoverIsSteadyAndTheFoundUnitStaysDown(t *testing.T) {
|
|||||||
if m.did("systemctl stop wg-quick@wg0") {
|
if m.did("systemctl stop wg-quick@wg0") {
|
||||||
t.Error("a found unit already down was stopped again")
|
t.Error("a found unit already down was stopped again")
|
||||||
}
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != Taken {
|
||||||
|
t.Errorf("a steady takeover does not read as taken: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
|
||||||
// Somebody starts the found unit again: it would take the port back, so it is stopped again
|
// Somebody starts the found unit again beside the mesh's interface. Not stopped by the mesh —
|
||||||
// — the one thing on an adopted node the mesh undoes, because the tunnel is the mesh's now.
|
// on the hub it cannot hold the port, on a spoke stopping it would be a fight — but said.
|
||||||
m.units["wg-quick@wg0"].active = "active"
|
m.units["wg-quick@wg0"].active = "active"
|
||||||
m.asked = nil
|
m.asked = nil
|
||||||
_, _ = applyAdopted(t, d, again, m, dir)
|
report, _ = applyAdopted(t, d, again, m, dir)
|
||||||
if m.units["wg-quick@wg0"].active != "inactive" || !m.did("systemctl stop wg-quick@wg0") {
|
if m.did("systemctl stop wg-quick@wg0") {
|
||||||
t.Error("a found unit started again was left holding the mesh's port")
|
t.Error("a found unit started again by hand was stopped by the mesh")
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "running again beside") {
|
||||||
|
t.Errorf("the account does not say the found unit is up again: %+v", report.Tunnel)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAFoundInterfaceStillUpAfterItsUnitStoppedRefusesTheTakeover(t *testing.T) {
|
func TestAFoundInterfaceRaisedByHandIsRefusedNamingTheRemedy(t *testing.T) {
|
||||||
dir, config, mesh, m := aHubInUse(t)
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
// The unit is not running, yet the interface is up: the predecessor raised it by hand.
|
||||||
|
m.units["wg-quick@wg0"].active = "inactive"
|
||||||
m.wgUp = "wg0 mesh0\n"
|
m.wgUp = "wg0 mesh0\n"
|
||||||
_, state, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh), store.State{},
|
report, state, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"),
|
||||||
store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
|
store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
|
||||||
if err == nil || !strings.Contains(err.Error(), "still up") || !strings.Contains(err.Error(), "Nothing was flushed") {
|
if err == nil || !strings.Contains(err.Error(), "wg-quick down wg0") || !strings.Contains(err.Error(), "Nothing was flushed") {
|
||||||
t.Fatalf("an interface something else raises was taken over anyway: %v", err)
|
t.Fatalf("an interface raised by hand was not refused naming the remedy: %v", err)
|
||||||
}
|
}
|
||||||
if m.units["wg-quick@mesh0"].active == "active" {
|
if m.units["wg-quick@mesh0"].active == "active" {
|
||||||
t.Error("the mesh's interface was started on a port the found one still holds")
|
t.Error("the mesh's interface was started on a port the found one still holds")
|
||||||
}
|
}
|
||||||
|
// Looked at more than once before giving up: a person taking it down takes a moment.
|
||||||
|
shows := 0
|
||||||
|
for _, a := range m.asked {
|
||||||
|
if a == "wg show interfaces" {
|
||||||
|
shows++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if shows < takeoverRechecks+1 {
|
||||||
|
t.Errorf("the interface was looked at %d time(s) before the refusal; a person needs a moment", shows)
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != NotTaken {
|
||||||
|
t.Errorf("the account does not say the tunnel is not taken: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held {
|
if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held {
|
||||||
t.Error("the found configuration was not kept before the refusal")
|
t.Error("the found configuration was not kept before the refusal")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,8 +49,13 @@ type Identity struct {
|
|||||||
Membership Membership `json:"membership"`
|
Membership Membership `json:"membership"`
|
||||||
|
|
||||||
// Overlay is this node's key on the private network. Generated here, like the identity above,
|
// Overlay is this node's key on the private network. Generated here, like the identity above,
|
||||||
// and for the same reason: the mesh computes a graph it cannot impersonate.
|
// and for the same reason: the mesh computes a graph it cannot impersonate — or, on an adopted
|
||||||
|
// node that took a found tunnel over, that tunnel's key (novox/hq ADR 0105).
|
||||||
Overlay OverlayKey `json:"overlay"`
|
Overlay OverlayKey `json:"overlay"`
|
||||||
|
// OverlayBefore is the public half of the overlay key this node held before it took a found
|
||||||
|
// tunnel's, so a take run again names the key the mesh still records. Empty on a node that
|
||||||
|
// never took one.
|
||||||
|
OverlayBefore string `json:"overlay_before,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Membership is how this node reaches the mesh it belongs to, and who it believes.
|
// Membership is how this node reaches the mesh it belongs to, and who it believes.
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
package link
|
package link
|
||||||
|
|
||||||
import "time"
|
import (
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
// The wire formats shared with the control plane, which defines them separately because this
|
// The wire formats shared with the control plane, which defines them separately because this
|
||||||
// binary requires nothing present and does not import it. A test on each side asserts the field
|
// binary requires nothing present and does not import it. A test on each side asserts the field
|
||||||
@@ -105,18 +109,64 @@ type Report struct {
|
|||||||
// 0105): which interface, its port, range and peer count, whether the found interface is down
|
// 0105): which interface, its port, range and peer count, whether the found interface is down
|
||||||
// and the mesh's up in its place, and where the found configuration's original was kept.
|
// and the mesh's up in its place, and where the found configuration's original was kept.
|
||||||
Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
|
Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
|
||||||
|
|
||||||
|
// Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq
|
||||||
|
// ADR 0105). Not an account of the machine: a report carrying one says nothing else.
|
||||||
|
Rekey *Rekey `json:"rekey,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CarriedTunnel is this node's account of the tunnel it took over.
|
// CarriedTunnel is this node's account of the tunnel it took over. State is one of the Carried
|
||||||
|
// states below; Note is what the host did about it, when it did something.
|
||||||
type CarriedTunnel struct {
|
type CarriedTunnel struct {
|
||||||
Interface string `json:"interface"`
|
Interface string `json:"interface"`
|
||||||
Port int `json:"port"`
|
Port int `json:"port"`
|
||||||
Range string `json:"range"`
|
Range string `json:"range"`
|
||||||
Peers int `json:"peers"`
|
Peers int `json:"peers"`
|
||||||
Taken bool `json:"taken"`
|
State string `json:"state"`
|
||||||
|
Note string `json:"note,omitempty"`
|
||||||
Kept string `json:"kept,omitempty"`
|
Kept string `json:"kept,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The states a carried tunnel can be in: the found interface still up and the mesh's not; the
|
||||||
|
// found one down and the mesh's up with its key; or the found one down and the mesh's not up — the
|
||||||
|
// one state where the peers reach nothing, said as its own word so nothing reads it as either of
|
||||||
|
// the others.
|
||||||
|
const (
|
||||||
|
CarriedNotTaken = "not-taken"
|
||||||
|
CarriedTaken = "taken"
|
||||||
|
CarriedDown = "down"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Rekey is this node saying it took a found tunnel's key as its overlay key after enrolling
|
||||||
|
// (novox/hq ADR 0105): the path for a node that enrolled before the mesh knew to take a tunnel
|
||||||
|
// over, since re-enrolling would rotate every key it holds. Signed with the identity key over
|
||||||
|
// RekeyProof, so a report forged on a stolen broker account cannot move this node's overlay key.
|
||||||
|
type Rekey struct {
|
||||||
|
// Previous is the overlay key this node held until now, as the mesh records it; the mesh
|
||||||
|
// refuses a rekey naming another, which is how a replayed one is refused.
|
||||||
|
Previous string `json:"previous"`
|
||||||
|
OverlayKey string `json:"overlay_key"`
|
||||||
|
Tunnel *Tunnel `json:"tunnel"`
|
||||||
|
Proof []byte `json:"proof"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RekeyProof is what a node signs when it rekeys — the node, the key it leaves, the key it takes
|
||||||
|
// and the tunnel it took it from — so a proof cannot be moved to another node or another tunnel.
|
||||||
|
// Byte for byte the mesh's own (mesh-controller internal/link RekeyProof).
|
||||||
|
func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte {
|
||||||
|
var t Tunnel
|
||||||
|
if tunnel != nil {
|
||||||
|
t = *tunnel
|
||||||
|
}
|
||||||
|
peers := make([]string, 0, len(t.Peers))
|
||||||
|
for _, p := range t.Peers {
|
||||||
|
peers = append(peers, p.PublicKey+"@"+p.Address)
|
||||||
|
}
|
||||||
|
return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" +
|
||||||
|
t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" +
|
||||||
|
t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ","))
|
||||||
|
}
|
||||||
|
|
||||||
// Held is one file or container found on an adopted node and kept as it was.
|
// Held is one file or container found on an adopted node and kept as it was.
|
||||||
type Held struct {
|
type Held struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
|
|||||||
@@ -377,6 +377,39 @@ func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) R
|
|||||||
}
|
}
|
||||||
|
|
||||||
// publishReport tells the mesh what this node did, and says whether the broker took it.
|
// publishReport tells the mesh what this node did, and says whether the broker took it.
|
||||||
|
// Publish sends one report on this node's own connection and returns: the one-shot path for a
|
||||||
|
// report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same
|
||||||
|
// account, the same pinned certificate and the same exchange as the running host's reports.
|
||||||
|
func Publish(ctx context.Context, m Membership, report Report, timeout time.Duration) error {
|
||||||
|
config, err := PinnedConfig(m.Fingerprint)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
dsn := fmt.Sprintf("amqps://%s:%s@%s/",
|
||||||
|
url.QueryEscape(m.Node), url.QueryEscape(m.Password), m.Broker)
|
||||||
|
conn, err := amqp.DialConfig(dsn, amqp.Config{
|
||||||
|
TLSClientConfig: config,
|
||||||
|
Dial: amqp.DefaultDial(timeout),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrWrongCertificate) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return fmt.Errorf("cannot reach the broker at %s: %w", m.Broker, err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
channel, err := conn.Channel()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer channel.Close()
|
||||||
|
var said string
|
||||||
|
if !publishReport(ctx, channel, m, report, func(s string) { said = s }, timeout) {
|
||||||
|
return errors.New(said)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report,
|
func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report,
|
||||||
say Announce, timeout time.Duration) bool {
|
say Announce, timeout time.Duration) bool {
|
||||||
report.Node = m.Node
|
report.Node = m.Node
|
||||||
|
|||||||
Reference in New Issue
Block a user