diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 6bdb549..af94644 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -364,6 +364,7 @@ func ApplyKeeping( // flushed. A failure here fails the service too — the mesh's interface is not started on a // port the found one still holds. stoppedFound := false + tookAt := -1 if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil { var outcome Outcome var facts TakenTunnel @@ -388,8 +389,11 @@ func ApplyKeeping( log(fmt.Sprintf(" failed %s (%s): %v", svc.Identity(), svc.Unit, err)) continue } + tookAt = len(report.Outcomes) report.Outcomes = append(report.Outcomes, outcome) - log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) + if outcome.Action == "held" { + log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) + } } was, _ := known.Find(resource.Identity()) @@ -510,6 +514,16 @@ func ApplyKeeping( // The found interface is down and the mesh's is up in its place: the tunnel changed // hands (novox/hq ADR 0105). Read from the machine, not assumed. report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run) + // And once a peer has handshaken with it, the take is proven and the found + // configuration is retired — here, after the mesh's service applied, so in the apply + // of the take itself only if a peer is already through; otherwise a later apply + // retires it (novox/hq ADR 0119). What it did replaces what the take said of the file. + if o, did := retireFound(ctx, svc, &known, run, keep, report.Tunnel, time.Now().UTC()); did { + if tookAt >= 0 { + report.Outcomes[tookAt] = o + } + log(fmt.Sprintf(" %s %s (%s): %s", o.Action, o.ID, o.Target, o.Detail)) + } } report.Outcomes = append(report.Outcomes, outcome) if outcome.Action != "unchanged" { diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 850f254..5f67996 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -21,6 +21,10 @@ type machine struct { asked []string // wgUp is what `wg show interfaces` answers: the tunnels up on the machine. wgUp string + // handshakes is what `wg show latest-handshakes` answers, and handshakesFail the + // error it fails with instead — a machine with no `wg`, say (novox/hq ADR 0119). + handshakes string + handshakesFail error // units are service units by name, as systemd would report them; volumes are the runtime's // named volumes. @@ -101,6 +105,9 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e return m.systemctl(args) } if name == "wg" { + if len(args) > 0 && args[len(args)-1] == "latest-handshakes" { + return m.handshakes, m.handshakesFail + } return m.wgUp, nil } if name == "getent" { diff --git a/internal/apply/plan.go b/internal/apply/plan.go index 1937617..475a34c 100644 --- a/internal/apply/plan.go +++ b/internal/apply/plan.go @@ -56,6 +56,12 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step { for _, r := range d.Resources { declared[r.Identity()] = true } + // The found tunnel's configuration is held under an id of its own, declared for as long as the + // service taking it over is — as ApplyKeeping counts it, or a plan would forget a hold the + // apply keeps (novox/hq ADR 0105). + if svc := takesOver(d); svc != nil { + declared[takeOverID(svc)] = true + } rec := known.Firewall ufw := rec != nil && rec.Kind == string(firewall.UFW) @@ -136,7 +142,12 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step { } steps = append(steps, orphans...) for _, r := range rest { - steps = append(steps, planned(r, d, known)) + step := planned(r, d, known) + if svc, ok := r.(*declaration.Service); ok && svc.TakesOver != nil && d.Adoption != nil && step.Verb != "hold" { + // The take comes before the service that replaces the tunnel, as it does in the apply. + steps = append(steps, plannedTake(svc, known)) + } + steps = append(steps, step) } // Only a declaration from the mesh converges a node; a bundle or a file never retires the @@ -239,6 +250,31 @@ func planned(r declaration.Resource, d *declaration.Declaration, known store.Sta return step } +// plannedTake is what the take of a found tunnel would do to its configuration (novox/hq ADR 0105, +// ADR 0119): kept as found while the take is not proven, and retired — removed from where its unit +// reads it, its original staying kept — by the first apply that finds the mesh's interface up in +// its place with a peer handshaken. Whether that is this apply is read from the machine, which a +// plan does not do, so it says when rather than whether. One the mesh retired already is said as +// retired: nothing brings it back. +func plannedTake(svc *declaration.Service, known store.State) Step { + t := svc.TakesOver + step := Step{Verb: "hold", Type: string(declaration.TypeFile), ID: takeOverID(svc), Target: t.Config} + if r, ok := known.RetiredAt(t.Config); ok { + step.Verb = "check" + step.Why = "retired once the take of " + t.Interface + " was proven; its original stays at " + r.Kept + + " and the mesh never brings it back" + return step + } + step.Why = "the configuration of the tunnel " + t.Interface + ", kept as found while " + svc.Unit + + " takes it over (" + t.Unit + " stopped and disabled, never flushed); retired — removed from " + + t.Config + ", its original staying kept — once the take is proven by a peer handshaking on " + + strings.TrimPrefix(svc.Unit, "wg-quick@") + if h, ok := known.HeldAt(takeOverID(svc)); ok && h.Kept != "" { + step.Why += "; the original is at " + h.Kept + } + return step +} + // readsChanged is which of the files a container was created reading the apply will hand it // changed — the same comparison applyContainer makes (novox/hq 04-ISSUES/103), settled from the // declaration and the record alone. diff --git a/internal/apply/takeover.go b/internal/apply/takeover.go index 359412c..8bc7130 100644 --- a/internal/apply/takeover.go +++ b/internal/apply/takeover.go @@ -27,6 +27,16 @@ import ( // Every apply, not once: a found unit somebody starts again would take the port back from the // mesh's interface, so it is stopped again and said so. That is the one place an adopted node // undoes something done by hand, and it is because the tunnel is the mesh's now. +// +// **Until the take is proven, and then the found configuration is retired** (novox/hq ADR 0119). +// Keeping it on disk was the caution the take needed: if the mesh's interface does not come up, +// the found unit is started again and the peers never notice. That caution is spent once the +// tunnel is taken — the found unit down and disabled, the mesh's interface up — and a peer has +// handshaken with the mesh's interface. From then on a configuration nothing maintains, one +// command away from raising a second way onto the network, is not a rollback path but a door +// nobody watches. So it is removed from where its unit reads it; its original, kept before +// anything happened to it (ADR 0100), stays kept; and the hold on it ends. A take never proven +// keeps it, and says so — a broken take is visible, not silently retired. // TakenTunnel is what an apply says about a tunnel it took over, for the node's report. type TakenTunnel struct { @@ -36,7 +46,9 @@ type TakenTunnel struct { Peers int // State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one // down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's - // not up: the peers reach nothing). Note is what this apply did about it. + // not up: the peers reach nothing). Note is what this apply did about it — and, for a taken + // tunnel, whether the take is proven and its found configuration retired (novox/hq ADR 0119). + // Kept is where the found configuration's original is, retired or not. State string Note string Kept string @@ -84,14 +96,35 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, // 1. The configuration, kept like any held file. A synthetic file resource stands for it, so // the same code keeps its original, digests it and notices it changing. + // + // **Unless it was retired** (novox/hq ADR 0119): the take was proven and the mesh removed + // it, so there is nothing to hold and nothing missing — only where its original is, which + // the retirement recorded. A hold still standing is let go: that is what retiring it meant. + // One put back at its path by a person is on the machine again, with no hold, and is found + // and kept afresh — the same content kept once, as any original is — and retired again by + // the first apply that finds the take still proven. file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config} - was, already := known.HeldAt(id) - out, held, err := hold(ctx, sys, file, module, was, already, - "the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now) - if err != nil { - return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err) + var held store.Held + retired, wasRetired := known.RetiredAt(t.Config) + if wasRetired && !present(t.Config) { + known.Release(id) + held = store.Held{Kept: retired.Kept} + out = begin(file) + out.Action = "unchanged" + facts.Note = "the found configuration " + t.Config + " was retired once the take was proven; " + + "its original is kept at " + retired.Kept + " and the mesh never brings it back" + } else { + if wasRetired { + known.Unretire(t.Config) + } + was, already := known.HeldAt(id) + out, held, err = hold(ctx, sys, file, module, was, already, + "the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now) + if err != nil { + return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err) + } + known.RecordHeld(held) } - known.RecordHeld(held) facts.Kept = held.Kept // What the file says, for the report: from the machine, or from the kept original when the // machine's copy is gone. The private key stays in the file; nothing here keeps it. @@ -185,6 +218,9 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, } out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found" + if wasRetired && out.Action == "unchanged" { + out.Detail = "the tunnel " + t.Interface + "'s configuration, retired once the take was proven" + } if held.Kept != "" { out.Detail += " (original at " + held.Kept + ")" } @@ -335,6 +371,124 @@ func restoreFound(ctx context.Context, sys system.System, unit string, run Runne facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had" } +// retireFound removes the found tunnel's configuration from where its unit reads it, once the take +// is proven, and ends the hold on it (novox/hq ADR 0119). Asked after the mesh's service applied +// and the tunnel reads as taken; retired says whether this apply retired it, and out is then what +// replaces the take's outcome for the configuration. +// +// **Proven is taken and a handshake.** Taken alone — the found unit down and disabled, the mesh's +// interface up — says the mesh's interface exists, not that any peer reaches it: an interface up +// with the wrong key is taken and carries nothing. A peer that has completed a handshake with it +// has checked its key, so that is the proof, asked of the kernel through `wg`. Anything short of +// one — no peer yet, every time zero, `wg` missing or failing — keeps the file, and the account +// says which: a take that never proves itself is visible rather than silently retired. +// +// **The original must still be kept.** It is the record of what the predecessor was and a +// person's only way back (ADR 0100); a kept copy that has gone missing is said, and the file is +// not removed, since removing it then would lose the only copy. What is on disk now, if something +// other than the mesh rewrote it since it was found, is kept too before it goes — by content, so +// the first original is never overwritten. +// +// The found unit is left disabled; without its configuration it cannot raise the interface, so +// every later apply's check of it finds nothing to do. Nothing here ever writes the file back. +func retireFound(ctx context.Context, svc *declaration.Service, known *store.State, run Runner, keep Keep, + facts *TakenTunnel, now time.Time) (out Outcome, retired bool) { + t := svc.TakesOver + id := takeOverID(svc) + if facts.State != Taken { + return out, false + } + held, isHeld := known.HeldAt(id) + if !isHeld { + // Retired already (takeOver let any hold go and said so), or never held: nothing to do. + return out, false + } + say := func(note string) { + if facts.Note != "" { + facts.Note += "; " + } + facts.Note += note + } + mesh := strings.TrimPrefix(svc.Unit, "wg-quick@") + peers, err := tunnel.Handshaken(ctx, tunnel.Runner(run), mesh) + if err != nil { + say("taken, not yet proven: " + err.Error() + "; the found configuration " + t.Config + " is kept") + return out, false + } + if peers == 0 { + say("taken, not yet proven: no peer has handshaken on " + mesh + "; the found configuration " + + t.Config + " is kept") + return out, false + } + proven := fmt.Sprintf("proven: %d peer(s) handshaken on %s", peers, mesh) + + // The kept original, read back — not just named in a record. + if held.Kept == "" { + say(proven + ", and the found configuration " + t.Config + " is not retired: no original of it " + + "was kept, so removing it would leave no record of what the predecessor was") + return out, false + } + original, err := os.ReadFile(held.Kept) + if err != nil || (held.Digest != "" && digestOf(string(original)) != held.Digest) { + why := "is missing" + if err == nil { + why = "no longer holds what was found" + } else if !errors.Is(err, os.ErrNotExist) { + why = "cannot be read (" + err.Error() + ")" + } + say(proven + ", and the found configuration " + t.Config + " is not retired: its kept original " + + held.Kept + " " + why + ", so removing it would lose the only copy") + return out, false + } + + gone := !present(t.Config) + if !gone { + current, err := os.ReadFile(t.Config) + if err != nil { + say(proven + ", and the found configuration " + t.Config + " is not retired: it cannot be read (" + + err.Error() + ")") + return out, false + } + if held.Digest != "" && digestOf(string(current)) != held.Digest { + if keep == nil { + say(proven + ", and the found configuration " + t.Config + " is not retired: it was rewritten " + + "since it was found and this host has nowhere to keep what it holds now") + return out, false + } + if _, err := keep(t.Config, current, 0o600); err != nil { + say(proven + ", and the found configuration " + t.Config + " is not retired: keeping what it " + + "holds now failed (" + err.Error() + ")") + return out, false + } + } + if err := os.Remove(t.Config); err != nil && !errors.Is(err, os.ErrNotExist) { + say(proven + ", and the found configuration " + t.Config + " could not be removed (" + err.Error() + ")") + return out, false + } + if present(t.Config) { + say(proven + ", and the found configuration " + t.Config + " is still there after it was removed") + return out, false + } + } + + known.RecordRetired(store.Retired{ID: id, Path: t.Config, Kept: held.Kept, At: now}) + known.Release(id) + facts.Kept = held.Kept + say(proven + "; the found configuration " + t.Config + " is retired — its original kept at " + + held.Kept + ", " + t.Unit + " left disabled, and the mesh never brings it back") + + out = Outcome{ID: id, Type: string(declaration.TypeFile), Target: t.Config, Action: "removed", + Detail: "retired: the take of " + t.Interface + " is " + proven + "; original kept at " + held.Kept} + if gone { + // Already gone — removed by something other than the mesh, or by an apply whose record was + // never saved. Nothing removed here; the hold ends all the same. + out.Action = "unchanged" + out.Detail = "retired: the take of " + t.Interface + " is " + proven + " and " + t.Config + + " was already gone; original kept at " + held.Kept + } + return out, true +} + // takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since // a machine has one private network. func takesOver(d *declaration.Declaration) *declaration.Service { diff --git a/internal/apply/takeover_test.go b/internal/apply/takeover_test.go index 18fde16..058c1d2 100644 --- a/internal/apply/takeover_test.go +++ b/internal/apply/takeover_test.go @@ -4,6 +4,7 @@ import ( "crypto/ecdh" "crypto/rand" "encoding/base64" + "errors" "os" "path/filepath" "strings" @@ -77,7 +78,10 @@ func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T t.Fatalf("the found unit was not stopped and disabled: %+v", u) } for _, asked := range m.asked { - if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") { + // Only ever asked about: which interfaces are up, and whether a peer has handshaken with + // the mesh's own (novox/hq ADR 0119). + if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") && + asked != "wg show mesh0 latest-handshakes" { t.Errorf("the found interface was touched with %q; it is stopped, never flushed", asked) } if strings.HasPrefix(asked, "wg-quick") || strings.Contains(asked, "peer remove") { @@ -254,3 +258,254 @@ func TestATakeoverIsRefusedOnAConvergedDeclaration(t *testing.T) { t.Fatalf("a takeover on a converged node was accepted: %v", err) } } + +// novox/hq ADR 0119: once the take is proven — taken, and a peer handshaken on the mesh's +// interface — the found configuration is removed from where its unit reads it, its original stays +// kept and the hold on it ends. Never before, and never brought back. + +const takesOverID = "mesh-wireguard.overlay-up.takes-over" + +// handshaken is `wg show mesh0 latest-handshakes` with one of the two peers through. +const handshaken = "PEER-A=\t1790000000\nPEER-B=\t0\n" + +func TestAProvenTakeRetiresTheFoundConfiguration(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes = handshaken + report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir) + + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Fatalf("a proven take left the found configuration where its unit reads it: %v", err) + } + retired, ok := state.RetiredAt(config) + if !ok || retired.Kept == "" || retired.ID != takesOverID { + t.Fatalf("the retirement was not recorded: %+v", state.Retired) + } + if kept, _ := os.ReadFile(retired.Kept); string(kept) != foundConf { + t.Fatalf("the kept original did not survive the retirement: %q", kept) + } + if _, held := state.HeldAt(takesOverID); held { + t.Error("the hold on the found configuration did not end with its retirement") + } + if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" { + t.Errorf("the found unit is not left down and disabled: %+v", u) + } + if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Kept != retired.Kept || + !strings.Contains(report.Tunnel.Note, "proven: 1 peer(s) handshaken on mesh0") || + !strings.Contains(report.Tunnel.Note, "is retired") { + t.Fatalf("the account does not say the take is proven and the configuration retired: %+v", report.Tunnel) + } + if o := outcomeOf(report, takesOverID); o.Action != "removed" || !strings.Contains(o.Detail, "retired") { + t.Errorf("the retirement is not what the apply says it did to the file: %+v", o) + } + // And the account still carries what was found, read from the kept original. + if report.Tunnel.Port != 51900 || report.Tunnel.Peers != 2 { + t.Errorf("the account lost what the tunnel was: %+v", report.Tunnel) + } +} + +func TestATakeNotProvenKeepsTheFoundConfigurationAndSaysSo(t *testing.T) { + cases := map[string]struct { + handshakes string + fail error + says string + }{ + "no peer at all": {"", nil, "no peer has handshaken on mesh0"}, + "every handshake at zero": {"PEER-A=\t0\nPEER-B=\t0\n", nil, "no peer has handshaken on mesh0"}, + "wg is not there": {"", errors.New(`exec: "wg": executable file not found in $PATH`), "executable file not found"}, + "the answer is nonsense": {"unable to access interface\n", nil, "not a peer and a time"}, + } + for name, c := range cases { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes, m.handshakesFail = c.handshakes, c.fail + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + report, state := applyAdopted(t, d, store.State{}, m, dir) + + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatalf("%s: a take not proven lost the found configuration", name) + } + if _, held := state.HeldAt(takesOverID); !held { + t.Errorf("%s: the hold ended although the take is not proven", name) + } + if _, retired := state.RetiredAt(config); retired { + t.Errorf("%s: recorded as retired", name) + } + if report.Tunnel == nil || report.Tunnel.State != Taken || + !strings.Contains(report.Tunnel.Note, "taken, not yet proven") || + !strings.Contains(report.Tunnel.Note, c.says) || !strings.Contains(report.Tunnel.Note, "is kept") { + t.Errorf("%s: the account does not say the take is not proven and why: %+v", name, report.Tunnel) + } + + // A later apply that finds a peer through retires it: the take itself need not be the one. + m.handshakes, m.handshakesFail = handshaken, nil + _, state = applyAdopted(t, d, state, m, dir) + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Errorf("%s: the apply after the take was proven kept the found configuration", name) + } + if _, retired := state.RetiredAt(config); !retired { + t.Errorf("%s: the later retirement was not recorded", name) + } + } +} + +func TestAFoundConfigurationWhoseKeptOriginalIsMissingIsNotRetired(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + held, _ := state.HeldAt(takesOverID) + if err := os.Remove(held.Kept); err != nil { + t.Fatal(err) + } + + m.handshakes = handshaken + report, state := applyAdopted(t, d, state, m, dir) + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatal("the found configuration was removed with no kept original left of it") + } + if _, still := state.HeldAt(takesOverID); !still { + t.Error("the hold ended although nothing was retired") + } + if _, retired := state.RetiredAt(config); retired { + t.Error("recorded as retired") + } + if report.Tunnel == nil || !strings.Contains(report.Tunnel.Note, "is not retired") || + !strings.Contains(report.Tunnel.Note, held.Kept+" is missing") { + t.Errorf("the account does not say the kept original is missing: %+v", report.Tunnel) + } +} + +func TestAFoundConfigurationRewrittenSinceItWasFoundIsKeptAgainBeforeItGoes(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + rewritten := foundConf + "\n[Peer]\nPublicKey = PEER-C=\nAllowedIPs = 192.0.2.4/32\n" + if err := os.WriteFile(config, []byte(rewritten), 0o600); err != nil { + t.Fatal(err) + } + + m.handshakes = handshaken + _, state = applyAdopted(t, d, state, m, dir) + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Fatal("a proven take kept a rewritten configuration") + } + retired, _ := state.RetiredAt(config) + if first, _ := os.ReadFile(retired.Kept); string(first) != foundConf { + t.Errorf("the first original was overwritten: %q", first) + } + kept, _ := filepath.Glob(filepath.Join(dir, "kept", "*-wg0.conf")) + var found bool + for _, k := range kept { + if got, _ := os.ReadFile(k); string(got) == rewritten { + found = true + } + } + if !found { + t.Errorf("what the file held when it was retired was not kept: %v", kept) + } +} + +func TestARetiredTakeIsSteadyAndItsFoundUnitFindsNothingToDo(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes = handshaken + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + retired, _ := state.RetiredAt(config) + + // wg-quick@wg0 with no configuration: inactive, and disabled — the check of it every apply + // makes must find nothing to do and fail on nothing. + m.asked = nil + report, again := applyAdopted(t, d, state, m, dir) + if report.Changed() { + t.Errorf("an apply after the retirement moved the machine: %+v", report.Outcomes) + } + if m.did("systemctl stop wg-quick@wg0") || m.did("systemctl start wg-quick@wg0") { + t.Errorf("the retired tunnel's unit was acted on: %v", m.asked) + } + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Error("the found configuration came back") + } + if _, held := again.HeldAt(takesOverID); held { + t.Error("a retired configuration is held again") + } + if r, ok := again.RetiredAt(config); !ok || r != retired { + t.Errorf("the retirement was not kept as it was: %+v", again.Retired) + } + if o := outcomeOf(report, takesOverID); o.Action != "unchanged" || !strings.Contains(o.Detail, "retired") { + t.Errorf("the retired configuration is not said as retired: %+v", o) + } + if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Kept != retired.Kept || + !strings.Contains(report.Tunnel.Note, "retired") || report.Tunnel.Port != 51900 { + t.Errorf("the account of a retired take does not say so: %+v", report.Tunnel) + } + + // Enabled at boot again by a person: disabled again, as any take does, and still no error. + m.units["wg-quick@wg0"].enabled = "enabled" + _, _ = applyAdopted(t, d, again, m, dir) + if m.units["wg-quick@wg0"].enabled != "disabled" { + t.Error("the found unit enabled again by hand was left to start at boot") + } +} + +func TestUndeclaringThePrivateNetworkAfterRetirementBringsNothingBack(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes = handshaken + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + + // The private network unassigned: only something else is declared. + other := adopted(t, `{"taken":[],"untaken":{}}`, + `{"id":"other.file","type":"file","path":"`+filepath.Join(dir, "other.conf")+`","content":"x\n"}`) + m.asked = nil + _, after := applyAdopted(t, other, state, m, dir) + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Fatal("undeclaring the private network brought the found configuration back") + } + if m.did("systemctl start wg-quick@wg0") || m.did("systemctl enable wg-quick@wg0") { + t.Errorf("undeclaring the private network started the found tunnel: %v", m.asked) + } + if _, ok := after.RetiredAt(config); !ok { + t.Error("the retirement was forgotten with the private network") + } + + // Assigned again, it finds the configuration retired rather than missing, and raises the + // mesh's interface. + report, _ := applyAdopted(t, d, after, m, dir) + if report.Tunnel == nil || report.Tunnel.State != Taken { + t.Errorf("the private network assigned again did not take the tunnel: %+v", report.Tunnel) + } + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Error("assigning the private network again brought the found configuration back") + } +} + +func TestAPlanSaysTheFoundConfigurationIsRetiredWhenTheTakeIsProven(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + + plan := Plan(d, store.State{}, store.OriginDeclared) + report, state := applyAdopted(t, d, store.State{}, m, dir) + if got, want := strings.Join(ids(plan), " "), strings.Join(outcomeIDs(report), " "); got != want { + t.Errorf("the plan said %q and the apply did %q", got, want) + } + var take Step + for _, s := range plan { + if s.ID == takesOverID { + take = s + } + } + if take.Verb != "hold" || take.Target != config || !strings.Contains(take.Why, "retired — removed from "+config) || + !strings.Contains(take.Why, "handshaking on mesh0") { + t.Errorf("the plan does not say the found configuration is retired once proven: %+v", take) + } + // Held and still declared: never planned as forgotten. + if strings.Contains(verbs(Plan(d, state, store.OriginDeclared)), "forget "+takesOverID) { + t.Error("the plan forgets a hold the apply keeps") + } + + m.handshakes = handshaken + _, state = applyAdopted(t, d, state, m, dir) + for _, s := range Plan(d, state, store.OriginDeclared) { + if s.ID == takesOverID && (s.Verb != "check" || !strings.Contains(s.Why, "retired once the take")) { + t.Errorf("a retired configuration is planned as %+v", s) + } + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index 5cc738f..d0c6e63 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -706,9 +706,10 @@ type Service struct { // TakesOver names the found tunnel this service replaces (novox/hq ADR 0105): before this unit // is started, the named unit is stopped and disabled — never flushed — and its configuration - // file is kept like any held file. Only on an adopted node, and only said by the controller, - // which knows the found tunnel's key is this node's own: without that, starting this unit on - // the found one's port would drop every peer's packets. + // file is kept like any held file — until the take is proven by a peer's handshake, and then + // retired, its original staying kept (novox/hq ADR 0119). Only on an adopted node, and only + // said by the controller, which knows the found tunnel's key is this node's own: without that, + // starting this unit on the found one's port would drop every peer's packets. TakesOver *TakeOver `json:"takes-over,omitempty"` } diff --git a/internal/store/store.go b/internal/store/store.go index 71485a0..87c7b90 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -170,6 +170,28 @@ type State struct { // the bundle carried in the binary is not applied again: what genesis applied was rewritten // for this machine, and the mesh has said more since (novox/hq issue 104). Genesis *Genesis `json:"genesis,omitempty"` + + // Retired is each found tunnel configuration the mesh removed from where its unit reads it, + // once the private network's take of that tunnel was proven (novox/hq ADR 0119). + // + // **Not a hold, and never released with one.** The hold on the found configuration ends at the + // retirement — what it held for has been replaced, and the node stops reporting it — so without + // this the next apply would find no hold and no file and read the take as one whose + // configuration vanished before it could be kept. It is kept whether or not the private + // network stays declared: undeclaring brings nothing back (ADR 0118), and a private network + // assigned again finds the tunnel's configuration retired rather than missing. + Retired []Retired `json:"retired,omitempty"` +} + +// Retired is a found configuration the mesh removed once what replaced it was proven (novox/hq +// ADR 0119): where it was, under which hold it had been kept, and where its original still is. +type Retired struct { + ID string `json:"id"` + Path string `json:"path"` + // Kept is the original as found (novox/hq ADR 0100) — the record of what the predecessor was, + // and a person's way back if one is ever wanted. The mesh never copies it back. + Kept string `json:"kept"` + At time.Time `json:"at"` } // Modes a node can be in (novox/hq ADR 0100). @@ -312,6 +334,42 @@ func (s *State) Release(id string) { } } +// RetiredAt returns the retirement of the found configuration at a path, if the mesh retired one. +func (s State) RetiredAt(path string) (Retired, bool) { + for _, r := range s.Retired { + if r.Path == path { + return r, true + } + } + return Retired{}, false +} + +// RecordRetired adds or replaces the retirement of the configuration at one path. +func (s *State) RecordRetired(r Retired) { + for i, existing := range s.Retired { + if existing.Path == r.Path { + s.Retired[i] = r + return + } + } + s.Retired = append(s.Retired, r) +} + +// Unretire forgets a retirement: the configuration is at its path again, put back by a person, and +// is found — and kept — afresh. +func (s *State) Unretire(path string) { + kept := s.Retired[:0] + for _, r := range s.Retired { + if r.Path != path { + kept = append(kept, r) + } + } + s.Retired = kept + if len(s.Retired) == 0 { + s.Retired = nil + } +} + // Find returns what was applied under an identity. func (s State) Find(id string) (Applied, bool) { for _, r := range s.Resources { diff --git a/internal/tunnel/tunnel.go b/internal/tunnel/tunnel.go index 9e1793d..5590304 100644 --- a/internal/tunnel/tunnel.go +++ b/internal/tunnel/tunnel.go @@ -3,7 +3,9 @@ // // On an adopted node that is the hub, the mesh's interface is raised with the found interface's // private key, on its port, with its address and range, and every peer it had. The found interface -// is stopped, never flushed; its configuration stays on disk. What this package does is the +// is stopped, never flushed; its configuration stays on disk until the take is proven — a peer +// has handshaken with the mesh's interface — and is then retired (novox/hq ADR 0119). What this +// package does is the // reading: which interface is there, what its file says, and what of that travels to the mesh — // everything but the private key, which becomes the node's own overlay key and is stored the way // that key is stored. @@ -151,6 +153,47 @@ func Find(ctx context.Context, run Runner, named string) (Found, error) { return found, nil } +// Handshaken is how many peers of an interface have completed a handshake with it: the proof that +// the interface carries the tunnel, rather than merely being up (novox/hq ADR 0119). +// +// Asked of the running interface, since a handshake is a fact about the kernel's tunnel that no +// file records. A question that cannot be asked — no `wg` on the machine, no such interface, a +// permission refused — is an error and never a zero: "no peer has handshaken" retires nothing +// either, but it is a different thing to tell a person. +func Handshaken(ctx context.Context, run Runner, iface string) (int, error) { + out, err := run(ctx, "wg", "show", iface, "latest-handshakes") + if err != nil { + return 0, fmt.Errorf("cannot ask %s which peers have handshaken: %w", iface, err) + } + return ParseHandshakes(out) +} + +// ParseHandshakes reads `wg show latest-handshakes`: one line per peer, its public key +// and the Unix time of its latest handshake, tab-separated — zero for a peer that never has. What +// is counted is the peers with a time. A line that is not a key and a time is refused rather than +// skipped: output this does not understand is not evidence of anything. +func ParseHandshakes(out string) (int, error) { + n := 0 + for i, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) != 2 { + return 0, fmt.Errorf("line %d of the handshakes is not a peer and a time: %q", i+1, line) + } + at, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil || at < 0 { + return 0, fmt.Errorf("line %d of the handshakes does not end in a time: %q", i+1, line) + } + if at > 0 { + n++ + } + } + return n, nil +} + func orNone(names []string) string { if len(names) == 0 { return "none" diff --git a/internal/tunnel/tunnel_test.go b/internal/tunnel/tunnel_test.go index 19d73b3..c35ca4b 100644 --- a/internal/tunnel/tunnel_test.go +++ b/internal/tunnel/tunnel_test.go @@ -194,3 +194,49 @@ func TestAFoundTunnelReadsItsMTU(t *testing.T) { t.Fatalf("a config with no MTU must leave it zero; got %d", f2.MTU) } } + +// novox/hq ADR 0119: a take is proven by a handshake on the mesh's interface, read from `wg show +// latest-handshakes` — as wg prints it, a key and a Unix time per peer, zero for never. +func TestAHandshakeIsAPeerWithATime(t *testing.T) { + cases := map[string]struct { + out string + want int + }{ + "two peers, one handshaken": {"PEER-A=\t1790000000\nPEER-B=\t0\n", 1}, + "every peer handshaken": {"PEER-A=\t1790000000\nPEER-B=\t1790000042\n", 2}, + "no peer ever": {"PEER-A=\t0\nPEER-B=\t0\n", 0}, + "an interface with no peer": {"", 0}, + "spaces, a trailing line": {"PEER-A= 1790000000\n\n", 1}, + } + for name, c := range cases { + got, err := ParseHandshakes(c.out) + if err != nil || got != c.want { + t.Errorf("%s: %d peer(s) handshaken (%v), want %d", name, got, err, c.want) + } + } + // Output that is not a key and a time is not evidence of anything, and not a zero either. + for _, nonsense := range []string{"PEER-A=\n", "PEER-A=\tyesterday\n", "PEER-A=\t-1\n", "a b c\n"} { + if _, err := ParseHandshakes(nonsense); err == nil { + t.Errorf("%q was read as handshakes", nonsense) + } + } +} + +func TestHandshakesThatCannotBeAskedAreAnErrorNotAZero(t *testing.T) { + var asked string + ok := func(_ context.Context, name string, args ...string) (string, error) { + asked = name + " " + strings.Join(args, " ") + return "PEER-A=\t1790000000\n", nil + } + if n, err := Handshaken(context.Background(), ok, "mesh0"); err != nil || n != 1 || + asked != "wg show mesh0 latest-handshakes" { + t.Fatalf("asked %q and read %d (%v)", asked, n, err) + } + missing := func(context.Context, string, ...string) (string, error) { + return "", errors.New(`exec: "wg": executable file not found in $PATH`) + } + if _, err := Handshaken(context.Background(), missing, "mesh0"); err == nil || + !strings.Contains(err.Error(), "mesh0") { + t.Fatalf("a machine with no wg was read as one with no handshake: %v", err) + } +}