diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 6bdb549..aced7bd 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -364,6 +364,7 @@ func ApplyKeeping( // flushed. A failure here fails the service too — the mesh's interface is not started on a // port the found one still holds. stoppedFound := false + tookAt := -1 if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil { var outcome Outcome var facts TakenTunnel @@ -388,8 +389,11 @@ func ApplyKeeping( log(fmt.Sprintf(" failed %s (%s): %v", svc.Identity(), svc.Unit, err)) continue } + tookAt = len(report.Outcomes) report.Outcomes = append(report.Outcomes, outcome) - log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) + if outcome.Action == "held" { + log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) + } } was, _ := known.Find(resource.Identity()) @@ -510,6 +514,16 @@ func ApplyKeeping( // The found interface is down and the mesh's is up in its place: the tunnel changed // hands (novox/hq ADR 0105). Read from the machine, not assumed. report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run) + // And once a peer has handshaken with it, the take is proven and the found + // configuration is retired — here, after the mesh's service applied, so in the apply + // of the take itself only if a peer is already through; otherwise a later apply + // retires it (novox/hq ADR 0119). What it did replaces what the take said of the file. + if o, did := retireFound(ctx, svc, d, &known, run, keep, report.Tunnel, time.Now().UTC()); did { + if tookAt >= 0 { + report.Outcomes[tookAt] = o + } + log(fmt.Sprintf(" %s %s (%s): %s", o.Action, o.ID, o.Target, o.Detail)) + } } report.Outcomes = append(report.Outcomes, outcome) if outcome.Action != "unchanged" { diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 850f254..5f67996 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -21,6 +21,10 @@ type machine struct { asked []string // wgUp is what `wg show interfaces` answers: the tunnels up on the machine. wgUp string + // handshakes is what `wg show latest-handshakes` answers, and handshakesFail the + // error it fails with instead — a machine with no `wg`, say (novox/hq ADR 0119). + handshakes string + handshakesFail error // units are service units by name, as systemd would report them; volumes are the runtime's // named volumes. @@ -101,6 +105,9 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e return m.systemctl(args) } if name == "wg" { + if len(args) > 0 && args[len(args)-1] == "latest-handshakes" { + return m.handshakes, m.handshakesFail + } return m.wgUp, nil } if name == "getent" { diff --git a/internal/apply/plan.go b/internal/apply/plan.go index 1937617..475a34c 100644 --- a/internal/apply/plan.go +++ b/internal/apply/plan.go @@ -56,6 +56,12 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step { for _, r := range d.Resources { declared[r.Identity()] = true } + // The found tunnel's configuration is held under an id of its own, declared for as long as the + // service taking it over is — as ApplyKeeping counts it, or a plan would forget a hold the + // apply keeps (novox/hq ADR 0105). + if svc := takesOver(d); svc != nil { + declared[takeOverID(svc)] = true + } rec := known.Firewall ufw := rec != nil && rec.Kind == string(firewall.UFW) @@ -136,7 +142,12 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step { } steps = append(steps, orphans...) for _, r := range rest { - steps = append(steps, planned(r, d, known)) + step := planned(r, d, known) + if svc, ok := r.(*declaration.Service); ok && svc.TakesOver != nil && d.Adoption != nil && step.Verb != "hold" { + // The take comes before the service that replaces the tunnel, as it does in the apply. + steps = append(steps, plannedTake(svc, known)) + } + steps = append(steps, step) } // Only a declaration from the mesh converges a node; a bundle or a file never retires the @@ -239,6 +250,31 @@ func planned(r declaration.Resource, d *declaration.Declaration, known store.Sta return step } +// plannedTake is what the take of a found tunnel would do to its configuration (novox/hq ADR 0105, +// ADR 0119): kept as found while the take is not proven, and retired — removed from where its unit +// reads it, its original staying kept — by the first apply that finds the mesh's interface up in +// its place with a peer handshaken. Whether that is this apply is read from the machine, which a +// plan does not do, so it says when rather than whether. One the mesh retired already is said as +// retired: nothing brings it back. +func plannedTake(svc *declaration.Service, known store.State) Step { + t := svc.TakesOver + step := Step{Verb: "hold", Type: string(declaration.TypeFile), ID: takeOverID(svc), Target: t.Config} + if r, ok := known.RetiredAt(t.Config); ok { + step.Verb = "check" + step.Why = "retired once the take of " + t.Interface + " was proven; its original stays at " + r.Kept + + " and the mesh never brings it back" + return step + } + step.Why = "the configuration of the tunnel " + t.Interface + ", kept as found while " + svc.Unit + + " takes it over (" + t.Unit + " stopped and disabled, never flushed); retired — removed from " + + t.Config + ", its original staying kept — once the take is proven by a peer handshaking on " + + strings.TrimPrefix(svc.Unit, "wg-quick@") + if h, ok := known.HeldAt(takeOverID(svc)); ok && h.Kept != "" { + step.Why += "; the original is at " + h.Kept + } + return step +} + // readsChanged is which of the files a container was created reading the apply will hand it // changed — the same comparison applyContainer makes (novox/hq 04-ISSUES/103), settled from the // declaration and the record alone. diff --git a/internal/apply/takeover.go b/internal/apply/takeover.go index 359412c..0669746 100644 --- a/internal/apply/takeover.go +++ b/internal/apply/takeover.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "strings" "time" @@ -27,6 +28,16 @@ import ( // Every apply, not once: a found unit somebody starts again would take the port back from the // mesh's interface, so it is stopped again and said so. That is the one place an adopted node // undoes something done by hand, and it is because the tunnel is the mesh's now. +// +// **Until the take is proven, and then the found configuration is retired** (novox/hq ADR 0119). +// Keeping it on disk was the caution the take needed: if the mesh's interface does not come up, +// the found unit is started again and the peers never notice. That caution is spent once the +// tunnel is taken — the found unit down and disabled, the mesh's interface up — and a peer has +// handshaken with the mesh's interface. From then on a configuration nothing maintains, one +// command away from raising a second way onto the network, is not a rollback path but a door +// nobody watches. So it is removed from where its unit reads it; its original, kept before +// anything happened to it (ADR 0100), stays kept; and the hold on it ends. A take never proven +// keeps it, and says so — a broken take is visible, not silently retired. // TakenTunnel is what an apply says about a tunnel it took over, for the node's report. type TakenTunnel struct { @@ -36,7 +47,9 @@ type TakenTunnel struct { Peers int // State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one // down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's - // not up: the peers reach nothing). Note is what this apply did about it. + // not up: the peers reach nothing). Note is what this apply did about it — and, for a taken + // tunnel, whether the take is proven and its found configuration retired (novox/hq ADR 0119). + // Kept is where the found configuration's original is, retired or not. State string Note string Kept string @@ -84,14 +97,53 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, // 1. The configuration, kept like any held file. A synthetic file resource stands for it, so // the same code keeps its original, digests it and notices it changing. + // + // **Unless it was retired** (novox/hq ADR 0119): the take was proven and the mesh removed + // it, so there is nothing to hold and nothing missing — only where its original is, which + // the retirement recorded. A hold still standing is let go: that is what retiring it meant. + // + // **One that comes back is held again on its FIRST original** — the one kept before anything + // happened to it (ADR 0100), never whatever was put back — and retired again by the first + // apply that finds the take still proven, keeping what came back only if it differs from + // what is already kept. Put back by hand while the private network is assigned, it is not a + // rollback: that means unassigning the private network first, and the note says so. file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config} - was, already := known.HeldAt(id) - out, held, err := hold(ctx, sys, file, module, was, already, - "the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now) - if err != nil { - return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err) + var held store.Held + retired, wasRetired := known.RetiredAt(t.Config) + cameBack := wasRetired && present(t.Config) + switch { + case wasRetired && !cameBack: + known.Release(id) + held = store.Held{Kept: retired.Kept} + out = begin(file) + out.Action = "unchanged" + facts.Note = "the found configuration " + t.Config + " was retired once the take was proven; " + + "its original is kept at " + retired.Kept + " and the mesh never brings it back" + default: + was, already := known.HeldAt(id) + why := "the configuration of the tunnel " + t.Interface + ", taken over by " + svc.Unit + if cameBack && !already { + digest := retired.Digest + if digest == "" { + if raw, err := os.ReadFile(retired.Kept); err == nil { + digest = digestOf(string(raw)) + } + } + was = store.Held{ID: id, Module: module, Kind: string(declaration.TypeFile), Target: t.Config, + Since: now, Why: why, Kept: retired.Kept, Digest: digest} + already = true + } + out, held, err = hold(ctx, sys, file, module, was, already, why, run, keep, now) + if err != nil { + return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err) + } + known.RecordHeld(held) + if cameBack { + facts.Note = "the found configuration " + t.Config + " came back after it was retired; while the " + + "private network is assigned the mesh retires it again, so rolling back to the found tunnel " + + "means unassigning the private network first" + } } - known.RecordHeld(held) facts.Kept = held.Kept // What the file says, for the report: from the machine, or from the kept original when the // machine's copy is gone. The private key stays in the file; nothing here keeps it. @@ -185,6 +237,13 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, } out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found" + switch { + case cameBack: + out.Detail = "the tunnel " + t.Interface + "'s configuration, back after it was retired; held until " + + "it is retired again" + case wasRetired: + out.Detail = "the tunnel " + t.Interface + "'s configuration, retired once the take was proven" + } if held.Kept != "" { out.Detail += " (original at " + held.Kept + ")" } @@ -335,6 +394,175 @@ func restoreFound(ctx context.Context, sys system.System, unit string, run Runne facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had" } +// wireguardDir is where a found tunnel's configuration may be retired from: wg-quick's own, and +// nowhere else. A variable so a test can hand in a directory. +var wireguardDir = tunnel.ConfigDir + +// retireFound removes the found tunnel's configuration from where its unit reads it, once the take +// is proven, and ends the hold on it (novox/hq ADR 0119). Asked after the mesh's service applied +// and the tunnel reads as taken; retired says whether this apply retired it, and out is then what +// replaces the take's outcome for the configuration. +// +// **Proven is taken and a handshake.** Taken alone — the found unit down and disabled, the mesh's +// interface up — says the mesh's interface exists, not that any peer reaches it: an interface up +// with the wrong key is taken and carries nothing. A peer that has completed a handshake with it +// has checked its key, so that is the proof, asked of the kernel through `wg`. Any handshake counts, +// however old: a change to the mesh's configuration restarts its unit, which recreates the +// interface and resets its counters, so a time that is there at all was made by this interface. +// Anything short of one — no peer yet, every time zero, `wg` missing or failing — keeps the file, +// and the account says which: a take that never proves itself is visible rather than silently +// retired. +// +// **Only what the take names, and only wg-quick's own file.** Nothing is removed unless the path +// is exactly `/.conf`, is not a path the mesh itself writes, and is +// a file rather than a link: removing a link would leave the key-bearing file it points at where it +// is, a retirement in name only, so that one is said and left to a person. +// +// **The original must still be kept.** It is the record of what the predecessor was and a +// person's only way back (ADR 0100); a kept copy that has gone missing is said, and the file is +// not removed, since removing it then would lose the only copy. What is on disk now, if it differs +// from the first original and from what was kept at the last retirement, is kept too before it +// goes — by content, so the first original is never overwritten and a file that keeps coming back +// the same keeps nothing more. +// +// The found unit is left disabled; without its configuration it cannot raise the interface, so +// every later apply's check of it finds nothing to do. Nothing here ever writes the file back. +func retireFound(ctx context.Context, svc *declaration.Service, d *declaration.Declaration, known *store.State, + run Runner, keep Keep, facts *TakenTunnel, now time.Time) (out Outcome, retired bool) { + t := svc.TakesOver + id := takeOverID(svc) + if facts.State != Taken { + return out, false + } + held, isHeld := known.HeldAt(id) + if !isHeld { + // Retired already (takeOver let any hold go and said so), or never held: nothing to do. + return out, false + } + say := func(note string) { + if facts.Note != "" { + facts.Note += "; " + } + facts.Note += note + } + notRetired := func(why string) (Outcome, bool) { + say("the found configuration " + t.Config + " is not retired: " + why) + return Outcome{}, false + } + mesh := strings.TrimPrefix(svc.Unit, "wg-quick@") + peers, err := tunnel.Handshaken(ctx, tunnel.Runner(run), mesh) + if err != nil { + say("taken, not yet proven: " + err.Error() + "; the found configuration " + t.Config + " is kept") + return out, false + } + if peers == 0 { + say("taken, not yet proven: no peer has handshaken on " + mesh + "; the found configuration " + + t.Config + " is kept") + return out, false + } + proven := fmt.Sprintf("proven: %d peer(s) handshaken on %s", peers, mesh) + say(proven) + + // What may be removed at all. + if want := filepath.Join(wireguardDir, t.Interface+".conf"); t.Config != want { + return notRetired("only " + want + ", the found interface's own wg-quick configuration, is ever " + + "retired by the mesh, and the take names " + t.Config) + } + if known.Recorded(string(declaration.TypeFile), t.Config) || declaresFile(d, t.Config) { + return notRetired("it is a path the mesh itself writes") + } + if info, err := os.Lstat(t.Config); err == nil && info.Mode()&os.ModeSymlink != 0 { + target, _ := os.Readlink(t.Config) + return notRetired("it is a link to " + target + "; removing the link would leave the key-bearing file " + + "it points at, so it must be retired by hand — both are kept") + } + + // The kept original, read back — not just named in a record. + if held.Kept == "" { + return notRetired("no original of it was kept, so removing it would leave no record of what the " + + "predecessor was") + } + original, err := os.ReadFile(held.Kept) + if err != nil || (held.Digest != "" && digestOf(string(original)) != held.Digest) { + why := "is missing" + if err == nil { + why = "no longer holds what was found" + } else if !errors.Is(err, os.ErrNotExist) { + why = "cannot be read (" + err.Error() + ")" + } + return notRetired("its kept original " + held.Kept + " " + why + ", so removing it would lose the only copy") + } + + before, cameBack := known.RetiredAt(t.Config) + record := store.Retired{ID: id, Path: t.Config, Kept: held.Kept, Digest: digestOf(string(original)), At: now} + if cameBack { + // The first original stays the record's, and so does what the last retirement kept. + record.Extra, record.ExtraDigest, record.Again = before.Extra, before.ExtraDigest, before.Again+1 + } + newCopy := "" + gone := !present(t.Config) + if !gone { + current, err := os.ReadFile(t.Config) + if err != nil { + return notRetired("it cannot be read (" + err.Error() + ")") + } + if sum := digestOf(string(current)); sum != record.Digest && sum != record.ExtraDigest { + if keep == nil { + return notRetired("it holds something other than its kept original and this host has nowhere " + + "to keep it") + } + where, err := keep(t.Config, current, 0o600) + if err != nil { + return notRetired("keeping what it holds now failed (" + err.Error() + ")") + } + record.Extra, record.ExtraDigest, newCopy = where, sum, where + } + if err := os.Remove(t.Config); err != nil && !errors.Is(err, os.ErrNotExist) { + return notRetired("removing it failed (" + err.Error() + ")") + } + if present(t.Config) { + return notRetired("it is still there after it was removed") + } + } + + known.RecordRetired(record) + known.Release(id) + facts.Kept = held.Kept + copied := "" + if newCopy != "" { + copied = "; what it held, which differed from the original, is kept at " + newCopy + } + out = Outcome{ID: id, Type: string(declaration.TypeFile), Target: t.Config, Action: "removed"} + switch { + case cameBack: + say("the found configuration came back and was retired again — its original still kept at " + + held.Kept + copied + "; rolling back to the found tunnel means unassigning the private network first") + out.Detail = "the found configuration came back and was retired again; original kept at " + held.Kept + copied + default: + say("the found configuration " + t.Config + " is retired — its original kept at " + held.Kept + copied + + ", " + t.Unit + " left disabled, and the mesh never brings it back") + out.Detail = "retired: the take of " + t.Interface + " is " + proven + "; original kept at " + held.Kept + copied + } + if gone { + // Already gone — removed by something other than the mesh, or by an apply whose record was + // never saved. Nothing removed here; the hold ends all the same. + out.Action = "unchanged" + out.Detail = "retired: the take of " + t.Interface + " is " + proven + " and " + t.Config + + " was already gone; original kept at " + held.Kept + } + return out, true +} + +// declaresFile is whether a declaration writes a file at a path. +func declaresFile(d *declaration.Declaration, path string) bool { + for _, r := range d.Resources { + if f, ok := r.(*declaration.File); ok && filepath.Clean(f.Path) == filepath.Clean(path) { + return true + } + } + return false +} + // takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since // a machine has one private network. func takesOver(d *declaration.Declaration) *declaration.Service { diff --git a/internal/apply/takeover_test.go b/internal/apply/takeover_test.go index 18fde16..8b6c24e 100644 --- a/internal/apply/takeover_test.go +++ b/internal/apply/takeover_test.go @@ -4,6 +4,7 @@ import ( "crypto/ecdh" "crypto/rand" "encoding/base64" + "errors" "os" "path/filepath" "strings" @@ -65,6 +66,10 @@ func aHubInUse(t *testing.T) (dir, config, mesh, keyFile string, m *machine) { "wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"}, }} takeoverRecheck = 0 + // The found configuration lives in this test's own wireguard directory (novox/hq ADR 0119). + was := wireguardDir + wireguardDir = dir + t.Cleanup(func() { wireguardDir = was }) return dir, config, mesh, keyFile, m } @@ -77,7 +82,10 @@ func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T t.Fatalf("the found unit was not stopped and disabled: %+v", u) } for _, asked := range m.asked { - if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") { + // Only ever asked about: which interfaces are up, and whether a peer has handshaken with + // the mesh's own (novox/hq ADR 0119). + if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") && + asked != "wg show mesh0 latest-handshakes" { t.Errorf("the found interface was touched with %q; it is stopped, never flushed", asked) } if strings.HasPrefix(asked, "wg-quick") || strings.Contains(asked, "peer remove") { @@ -254,3 +262,454 @@ func TestATakeoverIsRefusedOnAConvergedDeclaration(t *testing.T) { t.Fatalf("a takeover on a converged node was accepted: %v", err) } } + +// novox/hq ADR 0119: once the take is proven — taken, and a peer handshaken on the mesh's +// interface — the found configuration is removed from where its unit reads it, its original stays +// kept and the hold on it ends. Never before, and never brought back. + +const takesOverID = "mesh-wireguard.overlay-up.takes-over" + +// handshaken is `wg show mesh0 latest-handshakes` with one of the two peers through. +const handshaken = "PEER-A=\t1790000000\nPEER-B=\t0\n" + +func TestAProvenTakeRetiresTheFoundConfiguration(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes = handshaken + report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir) + + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Fatalf("a proven take left the found configuration where its unit reads it: %v", err) + } + retired, ok := state.RetiredAt(config) + if !ok || retired.Kept == "" || retired.ID != takesOverID { + t.Fatalf("the retirement was not recorded: %+v", state.Retired) + } + if kept, _ := os.ReadFile(retired.Kept); string(kept) != foundConf { + t.Fatalf("the kept original did not survive the retirement: %q", kept) + } + if _, held := state.HeldAt(takesOverID); held { + t.Error("the hold on the found configuration did not end with its retirement") + } + if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" { + t.Errorf("the found unit is not left down and disabled: %+v", u) + } + if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Kept != retired.Kept || + !strings.Contains(report.Tunnel.Note, "proven: 1 peer(s) handshaken on mesh0") || + !strings.Contains(report.Tunnel.Note, "is retired") { + t.Fatalf("the account does not say the take is proven and the configuration retired: %+v", report.Tunnel) + } + if o := outcomeOf(report, takesOverID); o.Action != "removed" || !strings.Contains(o.Detail, "retired") { + t.Errorf("the retirement is not what the apply says it did to the file: %+v", o) + } + // And the account still carries what was found, read from the kept original. + if report.Tunnel.Port != 51900 || report.Tunnel.Peers != 2 { + t.Errorf("the account lost what the tunnel was: %+v", report.Tunnel) + } +} + +func TestATakeNotProvenKeepsTheFoundConfigurationAndSaysSo(t *testing.T) { + cases := map[string]struct { + handshakes string + fail error + says string + }{ + "no peer at all": {"", nil, "no peer has handshaken on mesh0"}, + "every handshake at zero": {"PEER-A=\t0\nPEER-B=\t0\n", nil, "no peer has handshaken on mesh0"}, + "wg is not there": {"", errors.New(`exec: "wg": executable file not found in $PATH`), "executable file not found"}, + "the answer is nonsense": {"unable to access interface\n", nil, "not a peer and a time"}, + } + for name, c := range cases { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes, m.handshakesFail = c.handshakes, c.fail + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + report, state := applyAdopted(t, d, store.State{}, m, dir) + + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatalf("%s: a take not proven lost the found configuration", name) + } + if _, held := state.HeldAt(takesOverID); !held { + t.Errorf("%s: the hold ended although the take is not proven", name) + } + if _, retired := state.RetiredAt(config); retired { + t.Errorf("%s: recorded as retired", name) + } + if report.Tunnel == nil || report.Tunnel.State != Taken || + !strings.Contains(report.Tunnel.Note, "taken, not yet proven") || + !strings.Contains(report.Tunnel.Note, c.says) || !strings.Contains(report.Tunnel.Note, "is kept") { + t.Errorf("%s: the account does not say the take is not proven and why: %+v", name, report.Tunnel) + } + + // A later apply that finds a peer through retires it: the take itself need not be the one. + m.handshakes, m.handshakesFail = handshaken, nil + _, state = applyAdopted(t, d, state, m, dir) + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Errorf("%s: the apply after the take was proven kept the found configuration", name) + } + if _, retired := state.RetiredAt(config); !retired { + t.Errorf("%s: the later retirement was not recorded", name) + } + } +} + +func TestAFoundConfigurationWhoseKeptOriginalIsMissingIsNotRetired(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + held, _ := state.HeldAt(takesOverID) + if err := os.Remove(held.Kept); err != nil { + t.Fatal(err) + } + + m.handshakes = handshaken + report, state := applyAdopted(t, d, state, m, dir) + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatal("the found configuration was removed with no kept original left of it") + } + if _, still := state.HeldAt(takesOverID); !still { + t.Error("the hold ended although nothing was retired") + } + if _, retired := state.RetiredAt(config); retired { + t.Error("recorded as retired") + } + if report.Tunnel == nil || !strings.Contains(report.Tunnel.Note, "is not retired") || + !strings.Contains(report.Tunnel.Note, held.Kept+" is missing") { + t.Errorf("the account does not say the kept original is missing: %+v", report.Tunnel) + } +} + +func TestAFoundConfigurationRewrittenSinceItWasFoundIsKeptAgainBeforeItGoes(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + rewritten := foundConf + "\n[Peer]\nPublicKey = PEER-C=\nAllowedIPs = 192.0.2.4/32\n" + if err := os.WriteFile(config, []byte(rewritten), 0o600); err != nil { + t.Fatal(err) + } + + m.handshakes = handshaken + _, state = applyAdopted(t, d, state, m, dir) + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Fatal("a proven take kept a rewritten configuration") + } + retired, _ := state.RetiredAt(config) + if first, _ := os.ReadFile(retired.Kept); string(first) != foundConf { + t.Errorf("the first original was overwritten: %q", first) + } + kept, _ := filepath.Glob(filepath.Join(dir, "kept", "*-wg0.conf")) + var found bool + for _, k := range kept { + if got, _ := os.ReadFile(k); string(got) == rewritten { + found = true + } + } + if !found { + t.Errorf("what the file held when it was retired was not kept: %v", kept) + } +} + +func TestARetiredTakeIsSteadyAndItsFoundUnitFindsNothingToDo(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes = handshaken + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + retired, _ := state.RetiredAt(config) + + // wg-quick@wg0 with no configuration: inactive, and disabled — the check of it every apply + // makes must find nothing to do and fail on nothing. + m.asked = nil + report, again := applyAdopted(t, d, state, m, dir) + if report.Changed() { + t.Errorf("an apply after the retirement moved the machine: %+v", report.Outcomes) + } + if m.did("systemctl stop wg-quick@wg0") || m.did("systemctl start wg-quick@wg0") { + t.Errorf("the retired tunnel's unit was acted on: %v", m.asked) + } + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Error("the found configuration came back") + } + if _, held := again.HeldAt(takesOverID); held { + t.Error("a retired configuration is held again") + } + if r, ok := again.RetiredAt(config); !ok || r != retired { + t.Errorf("the retirement was not kept as it was: %+v", again.Retired) + } + if o := outcomeOf(report, takesOverID); o.Action != "unchanged" || !strings.Contains(o.Detail, "retired") { + t.Errorf("the retired configuration is not said as retired: %+v", o) + } + if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Kept != retired.Kept || + !strings.Contains(report.Tunnel.Note, "retired") || report.Tunnel.Port != 51900 { + t.Errorf("the account of a retired take does not say so: %+v", report.Tunnel) + } + + // Enabled at boot again by a person: disabled again, as any take does, and still no error. + m.units["wg-quick@wg0"].enabled = "enabled" + _, _ = applyAdopted(t, d, again, m, dir) + if m.units["wg-quick@wg0"].enabled != "disabled" { + t.Error("the found unit enabled again by hand was left to start at boot") + } +} + +func TestUndeclaringThePrivateNetworkAfterRetirementBringsNothingBack(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes = handshaken + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + + // The private network unassigned: only something else is declared. + other := adopted(t, `{"taken":[],"untaken":{}}`, + `{"id":"other.file","type":"file","path":"`+filepath.Join(dir, "other.conf")+`","content":"x\n"}`) + m.asked = nil + _, after := applyAdopted(t, other, state, m, dir) + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Fatal("undeclaring the private network brought the found configuration back") + } + if m.did("systemctl start wg-quick@wg0") || m.did("systemctl enable wg-quick@wg0") { + t.Errorf("undeclaring the private network started the found tunnel: %v", m.asked) + } + if _, ok := after.RetiredAt(config); !ok { + t.Error("the retirement was forgotten with the private network") + } + + // Assigned again, it finds the configuration retired rather than missing, and raises the + // mesh's interface. + report, _ := applyAdopted(t, d, after, m, dir) + if report.Tunnel == nil || report.Tunnel.State != Taken { + t.Errorf("the private network assigned again did not take the tunnel: %+v", report.Tunnel) + } + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Error("assigning the private network again brought the found configuration back") + } +} + +func TestAPlanSaysTheFoundConfigurationIsRetiredWhenTheTakeIsProven(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + + plan := Plan(d, store.State{}, store.OriginDeclared) + report, state := applyAdopted(t, d, store.State{}, m, dir) + if got, want := strings.Join(ids(plan), " "), strings.Join(outcomeIDs(report), " "); got != want { + t.Errorf("the plan said %q and the apply did %q", got, want) + } + var take Step + for _, s := range plan { + if s.ID == takesOverID { + take = s + } + } + if take.Verb != "hold" || take.Target != config || !strings.Contains(take.Why, "retired — removed from "+config) || + !strings.Contains(take.Why, "handshaking on mesh0") { + t.Errorf("the plan does not say the found configuration is retired once proven: %+v", take) + } + // Held and still declared: never planned as forgotten. + if strings.Contains(verbs(Plan(d, state, store.OriginDeclared)), "forget "+takesOverID) { + t.Error("the plan forgets a hold the apply keeps") + } + + m.handshakes = handshaken + _, state = applyAdopted(t, d, state, m, dir) + for _, s := range Plan(d, state, store.OriginDeclared) { + if s.ID == takesOverID && (s.Verb != "check" || !strings.Contains(s.Why, "retired once the take")) { + t.Errorf("a retired configuration is planned as %+v", s) + } + } +} + +// keptCopies is every copy kept of the found configuration. +func keptCopies(t *testing.T, dir string) []string { + t.Helper() + kept, err := filepath.Glob(filepath.Join(dir, "kept", "*-wg0.conf")) + if err != nil { + t.Fatal(err) + } + return kept +} + +func TestAConfigurationPutBackIsRetiredAgainOnItsFirstOriginalAndSettles(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes = handshaken + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + first, _ := state.RetiredAt(config) + + // Put back by hand with the original, while no peer is through yet: held on the first + // original, and the account says what a rollback takes. + write(t, config, foundConf) + m.handshakes = "" + report, state := applyAdopted(t, d, state, m, dir) + if held, ok := state.HeldAt(takesOverID); !ok || held.Kept != first.Kept { + t.Fatalf("what came back is not held on the first original: %+v", held) + } + if report.Tunnel == nil || !strings.Contains(report.Tunnel.Note, "came back after it was retired") || + !strings.Contains(report.Tunnel.Note, "unassigning the private network first") { + t.Errorf("the account does not say a rollback means unassigning the private network: %+v", report.Tunnel) + } + + // Proven: retired again, nothing more kept, said once. + m.handshakes = handshaken + report, state = applyAdopted(t, d, state, m, dir) + again, _ := state.RetiredAt(config) + if _, err := os.Lstat(config); !os.IsNotExist(err) || again.Kept != first.Kept || again.Extra != "" { + t.Fatalf("put back as it was, it was not retired again on the first original: %+v", again) + } + if o := outcomeOf(report, takesOverID); o.Action != "removed" || + !strings.HasPrefix(o.Detail, "the found configuration came back and was retired again") || + strings.Contains(o.Detail, "differed") { + t.Errorf("the second retirement is not said as one: %+v", o) + } + if !strings.Contains(report.Tunnel.Note, "unassigning the private network first") { + t.Errorf("the account does not say what a rollback takes: %q", report.Tunnel.Note) + } + if n := len(keptCopies(t, dir)); n != 1 { + t.Errorf("%d copies kept of one content", n) + } + if report, _ := applyAdopted(t, d, state, m, dir); report.Changed() { + t.Errorf("a steady machine moved after the second retirement: %+v", report.Outcomes) + } + + // Put back with something else: that is kept beside the first original, which stays the record's. + other := strings.Replace(foundConf, "PEER-B=", "PEER-Z=", 1) + write(t, config, other) + report, state = applyAdopted(t, d, state, m, dir) + third, _ := state.RetiredAt(config) + if third.Kept != first.Kept || third.Extra == "" || third.Extra == first.Kept { + t.Fatalf("other content was not kept apart from the first original: %+v", third) + } + if got, _ := os.ReadFile(third.Extra); string(got) != other { + t.Errorf("the extra copy does not hold what was put back: %q", got) + } + if o := outcomeOf(report, takesOverID); !strings.Contains(o.Detail, third.Extra) || + !strings.Contains(report.Tunnel.Note, third.Extra) { + t.Errorf("where the extra copy is was not said: %+v / %q", o, report.Tunnel.Note) + } + + // And the same other content again: nothing more kept, the record as it was. + write(t, config, other) + report, state = applyAdopted(t, d, state, m, dir) + fourth, _ := state.RetiredAt(config) + if fourth.Kept != first.Kept || fourth.Extra != third.Extra || len(keptCopies(t, dir)) != 2 { + t.Errorf("the same content put back again grew the copies: %+v, %v", fourth, keptCopies(t, dir)) + } + if o := outcomeOf(report, takesOverID); strings.Contains(o.Detail, "differed") { + t.Errorf("a copy already kept was said as new: %+v", o) + } + if report, _ := applyAdopted(t, d, state, m, dir); report.Changed() { + t.Errorf("a steady machine moved: %+v", report.Outcomes) + } +} + +func TestOnlyWgQuicksOwnConfigurationIsRetired(t *testing.T) { + // Not under the wireguard directory. + dir, config, mesh, keyFile, m := aHubInUse(t) + wireguardDir = filepath.Join(dir, "elsewhere") + m.handshakes = handshaken + report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir) + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatal("a configuration outside wg-quick's directory was removed") + } + if _, held := state.HeldAt(takesOverID); !held || !strings.Contains(report.Tunnel.Note, "is not retired: only ") { + t.Errorf("the refusal is not said, or the hold ended: %+v", report.Tunnel) + } + + // A path the mesh itself writes. + dir, config, mesh, keyFile, m = aHubInUse(t) + m.handshakes = handshaken + known := store.State{} + known.Record(store.Applied{ID: "bundle.wg0", Type: "file", Target: config, Origin: store.OriginCarried}) + report, _ = applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), known, m, dir) + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatal("a path the mesh writes was retired") + } + if !strings.Contains(report.Tunnel.Note, "a path the mesh itself writes") { + t.Errorf("the refusal is not said: %+v", report.Tunnel) + } +} + +func TestAFoundConfigurationThatIsALinkIsKeptAndLeftToAPerson(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + target := filepath.Join(dir, "predecessor", "hub.conf") + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + t.Fatal(err) + } + write(t, target, foundConf) + if err := os.Remove(config); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, config); err != nil { + t.Fatal(err) + } + m.handshakes = handshaken + report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir) + if _, err := os.Lstat(config); err != nil { + t.Fatal("the link was removed, leaving the key-bearing file it points at") + } + if got, _ := os.ReadFile(target); string(got) != foundConf { + t.Fatal("the file the link points at was touched") + } + held, ok := state.HeldAt(takesOverID) + if !ok { + t.Fatal("the hold ended") + } + if kept, _ := os.ReadFile(held.Kept); string(kept) != foundConf { + t.Errorf("what was kept is not what the link points at: %q", kept) + } + if !strings.Contains(report.Tunnel.Note, "is a link to "+target) || !strings.Contains(report.Tunnel.Note, "by hand") { + t.Errorf("the account does not say the link must be retired by hand: %q", report.Tunnel.Note) + } +} + +func TestARetirementWhoseRecordWasNeverSavedIsRecordedByTheNextApply(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + held, _ := state.HeldAt(takesOverID) + // An apply removed the file and stopped before its state was saved. + if err := os.Remove(config); err != nil { + t.Fatal(err) + } + m.handshakes = handshaken + report, state := applyAdopted(t, d, state, m, dir) + if r, ok := state.RetiredAt(config); !ok || r.Kept != held.Kept { + t.Fatalf("the retirement was not recorded: %+v", state.Retired) + } + if _, still := state.HeldAt(takesOverID); still { + t.Error("the hold did not end") + } + if o := outcomeOf(report, takesOverID); o.Action != "unchanged" || !strings.Contains(o.Detail, "already gone") { + t.Errorf("a file already gone is not said as such: %+v", o) + } +} + +func TestARemovalThatFailsKeepsTheFileAndTheHold(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root removes from a directory it may not write to") + } + dir, _, mesh, keyFile, m := aHubInUse(t) + wg := filepath.Join(dir, "wireguard") + if err := os.MkdirAll(wg, 0o700); err != nil { + t.Fatal(err) + } + config := filepath.Join(wg, "wg0.conf") + write(t, config, foundConf) + wireguardDir = wg + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + + if err := os.Chmod(wg, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(wg, 0o700) }) + m.handshakes = handshaken + report, state := applyAdopted(t, d, state, m, dir) + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatal("the found configuration is gone although it could not be removed") + } + if _, held := state.HeldAt(takesOverID); !held { + t.Error("the hold ended although nothing was retired") + } + if _, retired := state.RetiredAt(config); retired { + t.Error("recorded as retired") + } + if !strings.Contains(report.Tunnel.Note, "removing it failed") || !strings.Contains(report.Tunnel.Note, "permission denied") { + t.Errorf("the failed removal is not said: %q", report.Tunnel.Note) + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index 5cc738f..d0c6e63 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -706,9 +706,10 @@ type Service struct { // TakesOver names the found tunnel this service replaces (novox/hq ADR 0105): before this unit // is started, the named unit is stopped and disabled — never flushed — and its configuration - // file is kept like any held file. Only on an adopted node, and only said by the controller, - // which knows the found tunnel's key is this node's own: without that, starting this unit on - // the found one's port would drop every peer's packets. + // file is kept like any held file — until the take is proven by a peer's handshake, and then + // retired, its original staying kept (novox/hq ADR 0119). Only on an adopted node, and only + // said by the controller, which knows the found tunnel's key is this node's own: without that, + // starting this unit on the found one's port would drop every peer's packets. TakesOver *TakeOver `json:"takes-over,omitempty"` } diff --git a/internal/store/store.go b/internal/store/store.go index 71485a0..7306725 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -170,6 +170,40 @@ type State struct { // the bundle carried in the binary is not applied again: what genesis applied was rewritten // for this machine, and the mesh has said more since (novox/hq issue 104). Genesis *Genesis `json:"genesis,omitempty"` + + // Retired is each found tunnel configuration the mesh removed from where its unit reads it, + // once the private network's take of that tunnel was proven (novox/hq ADR 0119). + // + // **Not a hold, and never released with one.** The hold on the found configuration ends at the + // retirement — what it held for has been replaced, and the node stops reporting it — so without + // this the next apply would find no hold and no file and read the take as one whose + // configuration vanished before it could be kept. It is kept whether or not the private + // network stays declared: undeclaring brings nothing back (ADR 0118), and a private network + // assigned again finds the tunnel's configuration retired rather than missing. + Retired []Retired `json:"retired,omitempty"` +} + +// Retired is a found configuration the mesh removed once what replaced it was proven (novox/hq +// ADR 0119): where it was, under which hold it had been kept, and where its original still is. +type Retired struct { + ID string `json:"id"` + Path string `json:"path"` + // Kept is the original as found (novox/hq ADR 0100) — the record of what the predecessor was, + // and a person's way back if one is ever wanted. The mesh never copies it back. It is the FIRST + // original, and stays so however often the file comes back: a retirement repeated never moves + // it. Digest is what it holds. + Kept string `json:"kept"` + Digest string `json:"digest,omitempty"` + // Extra is where what was at the path when it was last retired is kept, when that differed from + // the first original — rewritten since it was found, or put back with other content — and + // ExtraDigest what it holds. One copy per distinct content: a file that comes back as it was + // last retired keeps nothing more. + Extra string `json:"extra,omitempty"` + ExtraDigest string `json:"extra_digest,omitempty"` + At time.Time `json:"at"` + // Again is how many times the configuration came back after it was retired, and was retired + // again (novox/hq ADR 0119). + Again int `json:"again,omitempty"` } // Modes a node can be in (novox/hq ADR 0100). @@ -312,6 +346,27 @@ func (s *State) Release(id string) { } } +// RetiredAt returns the retirement of the found configuration at a path, if the mesh retired one. +func (s State) RetiredAt(path string) (Retired, bool) { + for _, r := range s.Retired { + if r.Path == path { + return r, true + } + } + return Retired{}, false +} + +// RecordRetired adds or replaces the retirement of the configuration at one path. +func (s *State) RecordRetired(r Retired) { + for i, existing := range s.Retired { + if existing.Path == r.Path { + s.Retired[i] = r + return + } + } + s.Retired = append(s.Retired, r) +} + // Find returns what was applied under an identity. func (s State) Find(id string) (Applied, bool) { for _, r := range s.Resources { diff --git a/internal/tunnel/tunnel.go b/internal/tunnel/tunnel.go index 9e1793d..5590304 100644 --- a/internal/tunnel/tunnel.go +++ b/internal/tunnel/tunnel.go @@ -3,7 +3,9 @@ // // On an adopted node that is the hub, the mesh's interface is raised with the found interface's // private key, on its port, with its address and range, and every peer it had. The found interface -// is stopped, never flushed; its configuration stays on disk. What this package does is the +// is stopped, never flushed; its configuration stays on disk until the take is proven — a peer +// has handshaken with the mesh's interface — and is then retired (novox/hq ADR 0119). What this +// package does is the // reading: which interface is there, what its file says, and what of that travels to the mesh — // everything but the private key, which becomes the node's own overlay key and is stored the way // that key is stored. @@ -151,6 +153,47 @@ func Find(ctx context.Context, run Runner, named string) (Found, error) { return found, nil } +// Handshaken is how many peers of an interface have completed a handshake with it: the proof that +// the interface carries the tunnel, rather than merely being up (novox/hq ADR 0119). +// +// Asked of the running interface, since a handshake is a fact about the kernel's tunnel that no +// file records. A question that cannot be asked — no `wg` on the machine, no such interface, a +// permission refused — is an error and never a zero: "no peer has handshaken" retires nothing +// either, but it is a different thing to tell a person. +func Handshaken(ctx context.Context, run Runner, iface string) (int, error) { + out, err := run(ctx, "wg", "show", iface, "latest-handshakes") + if err != nil { + return 0, fmt.Errorf("cannot ask %s which peers have handshaken: %w", iface, err) + } + return ParseHandshakes(out) +} + +// ParseHandshakes reads `wg show latest-handshakes`: one line per peer, its public key +// and the Unix time of its latest handshake, tab-separated — zero for a peer that never has. What +// is counted is the peers with a time. A line that is not a key and a time is refused rather than +// skipped: output this does not understand is not evidence of anything. +func ParseHandshakes(out string) (int, error) { + n := 0 + for i, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) != 2 { + return 0, fmt.Errorf("line %d of the handshakes is not a peer and a time: %q", i+1, line) + } + at, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil || at < 0 { + return 0, fmt.Errorf("line %d of the handshakes does not end in a time: %q", i+1, line) + } + if at > 0 { + n++ + } + } + return n, nil +} + func orNone(names []string) string { if len(names) == 0 { return "none" diff --git a/internal/tunnel/tunnel_test.go b/internal/tunnel/tunnel_test.go index 19d73b3..c35ca4b 100644 --- a/internal/tunnel/tunnel_test.go +++ b/internal/tunnel/tunnel_test.go @@ -194,3 +194,49 @@ func TestAFoundTunnelReadsItsMTU(t *testing.T) { t.Fatalf("a config with no MTU must leave it zero; got %d", f2.MTU) } } + +// novox/hq ADR 0119: a take is proven by a handshake on the mesh's interface, read from `wg show +// latest-handshakes` — as wg prints it, a key and a Unix time per peer, zero for never. +func TestAHandshakeIsAPeerWithATime(t *testing.T) { + cases := map[string]struct { + out string + want int + }{ + "two peers, one handshaken": {"PEER-A=\t1790000000\nPEER-B=\t0\n", 1}, + "every peer handshaken": {"PEER-A=\t1790000000\nPEER-B=\t1790000042\n", 2}, + "no peer ever": {"PEER-A=\t0\nPEER-B=\t0\n", 0}, + "an interface with no peer": {"", 0}, + "spaces, a trailing line": {"PEER-A= 1790000000\n\n", 1}, + } + for name, c := range cases { + got, err := ParseHandshakes(c.out) + if err != nil || got != c.want { + t.Errorf("%s: %d peer(s) handshaken (%v), want %d", name, got, err, c.want) + } + } + // Output that is not a key and a time is not evidence of anything, and not a zero either. + for _, nonsense := range []string{"PEER-A=\n", "PEER-A=\tyesterday\n", "PEER-A=\t-1\n", "a b c\n"} { + if _, err := ParseHandshakes(nonsense); err == nil { + t.Errorf("%q was read as handshakes", nonsense) + } + } +} + +func TestHandshakesThatCannotBeAskedAreAnErrorNotAZero(t *testing.T) { + var asked string + ok := func(_ context.Context, name string, args ...string) (string, error) { + asked = name + " " + strings.Join(args, " ") + return "PEER-A=\t1790000000\n", nil + } + if n, err := Handshaken(context.Background(), ok, "mesh0"); err != nil || n != 1 || + asked != "wg show mesh0 latest-handshakes" { + t.Fatalf("asked %q and read %d (%v)", asked, n, err) + } + missing := func(context.Context, string, ...string) (string, error) { + return "", errors.New(`exec: "wg": executable file not found in $PATH`) + } + if _, err := Handshaken(context.Background(), missing, "mesh0"); err == nil || + !strings.Contains(err.Error(), "mesh0") { + t.Fatalf("a machine with no wg was read as one with no handshake: %v", err) + } +}