From b462f461c6c161dfa8347c944790ff99331e2bf2 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 00:47:57 +0200 Subject: [PATCH 1/2] A taken tunnel's found configuration is retired once the take is proven (hq ADR 0119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kept on disk it was the take's fallback; once the mesh's interface is up in its place and a peer has handshaken with it, it is an unmaintained way back onto the network, held for ever. It is now removed from where its unit reads it, its kept original verified first and left as it is, and the hold ends. Until proven — no handshake, or wg not answering — it is kept and the report says why. The retirement is recorded apart from holds, so later applies, an undeclare, and a reassignment find it retired rather than missing, and nothing writes it back. --- internal/apply/apply.go | 16 +- internal/apply/hold_test.go | 7 + internal/apply/plan.go | 38 +++- internal/apply/takeover.go | 168 +++++++++++++++++- internal/apply/takeover_test.go | 257 +++++++++++++++++++++++++++- internal/declaration/declaration.go | 7 +- internal/store/store.go | 58 +++++++ internal/tunnel/tunnel.go | 45 ++++- internal/tunnel/tunnel_test.go | 46 +++++ 9 files changed, 628 insertions(+), 14 deletions(-) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 6bdb549..af94644 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -364,6 +364,7 @@ func ApplyKeeping( // flushed. A failure here fails the service too — the mesh's interface is not started on a // port the found one still holds. stoppedFound := false + tookAt := -1 if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil { var outcome Outcome var facts TakenTunnel @@ -388,8 +389,11 @@ func ApplyKeeping( log(fmt.Sprintf(" failed %s (%s): %v", svc.Identity(), svc.Unit, err)) continue } + tookAt = len(report.Outcomes) report.Outcomes = append(report.Outcomes, outcome) - log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) + if outcome.Action == "held" { + log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) + } } was, _ := known.Find(resource.Identity()) @@ -510,6 +514,16 @@ func ApplyKeeping( // The found interface is down and the mesh's is up in its place: the tunnel changed // hands (novox/hq ADR 0105). Read from the machine, not assumed. report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run) + // And once a peer has handshaken with it, the take is proven and the found + // configuration is retired — here, after the mesh's service applied, so in the apply + // of the take itself only if a peer is already through; otherwise a later apply + // retires it (novox/hq ADR 0119). What it did replaces what the take said of the file. + if o, did := retireFound(ctx, svc, &known, run, keep, report.Tunnel, time.Now().UTC()); did { + if tookAt >= 0 { + report.Outcomes[tookAt] = o + } + log(fmt.Sprintf(" %s %s (%s): %s", o.Action, o.ID, o.Target, o.Detail)) + } } report.Outcomes = append(report.Outcomes, outcome) if outcome.Action != "unchanged" { diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 850f254..5f67996 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -21,6 +21,10 @@ type machine struct { asked []string // wgUp is what `wg show interfaces` answers: the tunnels up on the machine. wgUp string + // handshakes is what `wg show latest-handshakes` answers, and handshakesFail the + // error it fails with instead — a machine with no `wg`, say (novox/hq ADR 0119). + handshakes string + handshakesFail error // units are service units by name, as systemd would report them; volumes are the runtime's // named volumes. @@ -101,6 +105,9 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e return m.systemctl(args) } if name == "wg" { + if len(args) > 0 && args[len(args)-1] == "latest-handshakes" { + return m.handshakes, m.handshakesFail + } return m.wgUp, nil } if name == "getent" { diff --git a/internal/apply/plan.go b/internal/apply/plan.go index 1937617..475a34c 100644 --- a/internal/apply/plan.go +++ b/internal/apply/plan.go @@ -56,6 +56,12 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step { for _, r := range d.Resources { declared[r.Identity()] = true } + // The found tunnel's configuration is held under an id of its own, declared for as long as the + // service taking it over is — as ApplyKeeping counts it, or a plan would forget a hold the + // apply keeps (novox/hq ADR 0105). + if svc := takesOver(d); svc != nil { + declared[takeOverID(svc)] = true + } rec := known.Firewall ufw := rec != nil && rec.Kind == string(firewall.UFW) @@ -136,7 +142,12 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step { } steps = append(steps, orphans...) for _, r := range rest { - steps = append(steps, planned(r, d, known)) + step := planned(r, d, known) + if svc, ok := r.(*declaration.Service); ok && svc.TakesOver != nil && d.Adoption != nil && step.Verb != "hold" { + // The take comes before the service that replaces the tunnel, as it does in the apply. + steps = append(steps, plannedTake(svc, known)) + } + steps = append(steps, step) } // Only a declaration from the mesh converges a node; a bundle or a file never retires the @@ -239,6 +250,31 @@ func planned(r declaration.Resource, d *declaration.Declaration, known store.Sta return step } +// plannedTake is what the take of a found tunnel would do to its configuration (novox/hq ADR 0105, +// ADR 0119): kept as found while the take is not proven, and retired — removed from where its unit +// reads it, its original staying kept — by the first apply that finds the mesh's interface up in +// its place with a peer handshaken. Whether that is this apply is read from the machine, which a +// plan does not do, so it says when rather than whether. One the mesh retired already is said as +// retired: nothing brings it back. +func plannedTake(svc *declaration.Service, known store.State) Step { + t := svc.TakesOver + step := Step{Verb: "hold", Type: string(declaration.TypeFile), ID: takeOverID(svc), Target: t.Config} + if r, ok := known.RetiredAt(t.Config); ok { + step.Verb = "check" + step.Why = "retired once the take of " + t.Interface + " was proven; its original stays at " + r.Kept + + " and the mesh never brings it back" + return step + } + step.Why = "the configuration of the tunnel " + t.Interface + ", kept as found while " + svc.Unit + + " takes it over (" + t.Unit + " stopped and disabled, never flushed); retired — removed from " + + t.Config + ", its original staying kept — once the take is proven by a peer handshaking on " + + strings.TrimPrefix(svc.Unit, "wg-quick@") + if h, ok := known.HeldAt(takeOverID(svc)); ok && h.Kept != "" { + step.Why += "; the original is at " + h.Kept + } + return step +} + // readsChanged is which of the files a container was created reading the apply will hand it // changed — the same comparison applyContainer makes (novox/hq 04-ISSUES/103), settled from the // declaration and the record alone. diff --git a/internal/apply/takeover.go b/internal/apply/takeover.go index 359412c..8bc7130 100644 --- a/internal/apply/takeover.go +++ b/internal/apply/takeover.go @@ -27,6 +27,16 @@ import ( // Every apply, not once: a found unit somebody starts again would take the port back from the // mesh's interface, so it is stopped again and said so. That is the one place an adopted node // undoes something done by hand, and it is because the tunnel is the mesh's now. +// +// **Until the take is proven, and then the found configuration is retired** (novox/hq ADR 0119). +// Keeping it on disk was the caution the take needed: if the mesh's interface does not come up, +// the found unit is started again and the peers never notice. That caution is spent once the +// tunnel is taken — the found unit down and disabled, the mesh's interface up — and a peer has +// handshaken with the mesh's interface. From then on a configuration nothing maintains, one +// command away from raising a second way onto the network, is not a rollback path but a door +// nobody watches. So it is removed from where its unit reads it; its original, kept before +// anything happened to it (ADR 0100), stays kept; and the hold on it ends. A take never proven +// keeps it, and says so — a broken take is visible, not silently retired. // TakenTunnel is what an apply says about a tunnel it took over, for the node's report. type TakenTunnel struct { @@ -36,7 +46,9 @@ type TakenTunnel struct { Peers int // State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one // down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's - // not up: the peers reach nothing). Note is what this apply did about it. + // not up: the peers reach nothing). Note is what this apply did about it — and, for a taken + // tunnel, whether the take is proven and its found configuration retired (novox/hq ADR 0119). + // Kept is where the found configuration's original is, retired or not. State string Note string Kept string @@ -84,14 +96,35 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, // 1. The configuration, kept like any held file. A synthetic file resource stands for it, so // the same code keeps its original, digests it and notices it changing. + // + // **Unless it was retired** (novox/hq ADR 0119): the take was proven and the mesh removed + // it, so there is nothing to hold and nothing missing — only where its original is, which + // the retirement recorded. A hold still standing is let go: that is what retiring it meant. + // One put back at its path by a person is on the machine again, with no hold, and is found + // and kept afresh — the same content kept once, as any original is — and retired again by + // the first apply that finds the take still proven. file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config} - was, already := known.HeldAt(id) - out, held, err := hold(ctx, sys, file, module, was, already, - "the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now) - if err != nil { - return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err) + var held store.Held + retired, wasRetired := known.RetiredAt(t.Config) + if wasRetired && !present(t.Config) { + known.Release(id) + held = store.Held{Kept: retired.Kept} + out = begin(file) + out.Action = "unchanged" + facts.Note = "the found configuration " + t.Config + " was retired once the take was proven; " + + "its original is kept at " + retired.Kept + " and the mesh never brings it back" + } else { + if wasRetired { + known.Unretire(t.Config) + } + was, already := known.HeldAt(id) + out, held, err = hold(ctx, sys, file, module, was, already, + "the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now) + if err != nil { + return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err) + } + known.RecordHeld(held) } - known.RecordHeld(held) facts.Kept = held.Kept // What the file says, for the report: from the machine, or from the kept original when the // machine's copy is gone. The private key stays in the file; nothing here keeps it. @@ -185,6 +218,9 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, } out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found" + if wasRetired && out.Action == "unchanged" { + out.Detail = "the tunnel " + t.Interface + "'s configuration, retired once the take was proven" + } if held.Kept != "" { out.Detail += " (original at " + held.Kept + ")" } @@ -335,6 +371,124 @@ func restoreFound(ctx context.Context, sys system.System, unit string, run Runne facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had" } +// retireFound removes the found tunnel's configuration from where its unit reads it, once the take +// is proven, and ends the hold on it (novox/hq ADR 0119). Asked after the mesh's service applied +// and the tunnel reads as taken; retired says whether this apply retired it, and out is then what +// replaces the take's outcome for the configuration. +// +// **Proven is taken and a handshake.** Taken alone — the found unit down and disabled, the mesh's +// interface up — says the mesh's interface exists, not that any peer reaches it: an interface up +// with the wrong key is taken and carries nothing. A peer that has completed a handshake with it +// has checked its key, so that is the proof, asked of the kernel through `wg`. Anything short of +// one — no peer yet, every time zero, `wg` missing or failing — keeps the file, and the account +// says which: a take that never proves itself is visible rather than silently retired. +// +// **The original must still be kept.** It is the record of what the predecessor was and a +// person's only way back (ADR 0100); a kept copy that has gone missing is said, and the file is +// not removed, since removing it then would lose the only copy. What is on disk now, if something +// other than the mesh rewrote it since it was found, is kept too before it goes — by content, so +// the first original is never overwritten. +// +// The found unit is left disabled; without its configuration it cannot raise the interface, so +// every later apply's check of it finds nothing to do. Nothing here ever writes the file back. +func retireFound(ctx context.Context, svc *declaration.Service, known *store.State, run Runner, keep Keep, + facts *TakenTunnel, now time.Time) (out Outcome, retired bool) { + t := svc.TakesOver + id := takeOverID(svc) + if facts.State != Taken { + return out, false + } + held, isHeld := known.HeldAt(id) + if !isHeld { + // Retired already (takeOver let any hold go and said so), or never held: nothing to do. + return out, false + } + say := func(note string) { + if facts.Note != "" { + facts.Note += "; " + } + facts.Note += note + } + mesh := strings.TrimPrefix(svc.Unit, "wg-quick@") + peers, err := tunnel.Handshaken(ctx, tunnel.Runner(run), mesh) + if err != nil { + say("taken, not yet proven: " + err.Error() + "; the found configuration " + t.Config + " is kept") + return out, false + } + if peers == 0 { + say("taken, not yet proven: no peer has handshaken on " + mesh + "; the found configuration " + + t.Config + " is kept") + return out, false + } + proven := fmt.Sprintf("proven: %d peer(s) handshaken on %s", peers, mesh) + + // The kept original, read back — not just named in a record. + if held.Kept == "" { + say(proven + ", and the found configuration " + t.Config + " is not retired: no original of it " + + "was kept, so removing it would leave no record of what the predecessor was") + return out, false + } + original, err := os.ReadFile(held.Kept) + if err != nil || (held.Digest != "" && digestOf(string(original)) != held.Digest) { + why := "is missing" + if err == nil { + why = "no longer holds what was found" + } else if !errors.Is(err, os.ErrNotExist) { + why = "cannot be read (" + err.Error() + ")" + } + say(proven + ", and the found configuration " + t.Config + " is not retired: its kept original " + + held.Kept + " " + why + ", so removing it would lose the only copy") + return out, false + } + + gone := !present(t.Config) + if !gone { + current, err := os.ReadFile(t.Config) + if err != nil { + say(proven + ", and the found configuration " + t.Config + " is not retired: it cannot be read (" + + err.Error() + ")") + return out, false + } + if held.Digest != "" && digestOf(string(current)) != held.Digest { + if keep == nil { + say(proven + ", and the found configuration " + t.Config + " is not retired: it was rewritten " + + "since it was found and this host has nowhere to keep what it holds now") + return out, false + } + if _, err := keep(t.Config, current, 0o600); err != nil { + say(proven + ", and the found configuration " + t.Config + " is not retired: keeping what it " + + "holds now failed (" + err.Error() + ")") + return out, false + } + } + if err := os.Remove(t.Config); err != nil && !errors.Is(err, os.ErrNotExist) { + say(proven + ", and the found configuration " + t.Config + " could not be removed (" + err.Error() + ")") + return out, false + } + if present(t.Config) { + say(proven + ", and the found configuration " + t.Config + " is still there after it was removed") + return out, false + } + } + + known.RecordRetired(store.Retired{ID: id, Path: t.Config, Kept: held.Kept, At: now}) + known.Release(id) + facts.Kept = held.Kept + say(proven + "; the found configuration " + t.Config + " is retired — its original kept at " + + held.Kept + ", " + t.Unit + " left disabled, and the mesh never brings it back") + + out = Outcome{ID: id, Type: string(declaration.TypeFile), Target: t.Config, Action: "removed", + Detail: "retired: the take of " + t.Interface + " is " + proven + "; original kept at " + held.Kept} + if gone { + // Already gone — removed by something other than the mesh, or by an apply whose record was + // never saved. Nothing removed here; the hold ends all the same. + out.Action = "unchanged" + out.Detail = "retired: the take of " + t.Interface + " is " + proven + " and " + t.Config + + " was already gone; original kept at " + held.Kept + } + return out, true +} + // takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since // a machine has one private network. func takesOver(d *declaration.Declaration) *declaration.Service { diff --git a/internal/apply/takeover_test.go b/internal/apply/takeover_test.go index 18fde16..058c1d2 100644 --- a/internal/apply/takeover_test.go +++ b/internal/apply/takeover_test.go @@ -4,6 +4,7 @@ import ( "crypto/ecdh" "crypto/rand" "encoding/base64" + "errors" "os" "path/filepath" "strings" @@ -77,7 +78,10 @@ func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T t.Fatalf("the found unit was not stopped and disabled: %+v", u) } for _, asked := range m.asked { - if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") { + // Only ever asked about: which interfaces are up, and whether a peer has handshaken with + // the mesh's own (novox/hq ADR 0119). + if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") && + asked != "wg show mesh0 latest-handshakes" { t.Errorf("the found interface was touched with %q; it is stopped, never flushed", asked) } if strings.HasPrefix(asked, "wg-quick") || strings.Contains(asked, "peer remove") { @@ -254,3 +258,254 @@ func TestATakeoverIsRefusedOnAConvergedDeclaration(t *testing.T) { t.Fatalf("a takeover on a converged node was accepted: %v", err) } } + +// novox/hq ADR 0119: once the take is proven — taken, and a peer handshaken on the mesh's +// interface — the found configuration is removed from where its unit reads it, its original stays +// kept and the hold on it ends. Never before, and never brought back. + +const takesOverID = "mesh-wireguard.overlay-up.takes-over" + +// handshaken is `wg show mesh0 latest-handshakes` with one of the two peers through. +const handshaken = "PEER-A=\t1790000000\nPEER-B=\t0\n" + +func TestAProvenTakeRetiresTheFoundConfiguration(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes = handshaken + report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir) + + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Fatalf("a proven take left the found configuration where its unit reads it: %v", err) + } + retired, ok := state.RetiredAt(config) + if !ok || retired.Kept == "" || retired.ID != takesOverID { + t.Fatalf("the retirement was not recorded: %+v", state.Retired) + } + if kept, _ := os.ReadFile(retired.Kept); string(kept) != foundConf { + t.Fatalf("the kept original did not survive the retirement: %q", kept) + } + if _, held := state.HeldAt(takesOverID); held { + t.Error("the hold on the found configuration did not end with its retirement") + } + if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" { + t.Errorf("the found unit is not left down and disabled: %+v", u) + } + if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Kept != retired.Kept || + !strings.Contains(report.Tunnel.Note, "proven: 1 peer(s) handshaken on mesh0") || + !strings.Contains(report.Tunnel.Note, "is retired") { + t.Fatalf("the account does not say the take is proven and the configuration retired: %+v", report.Tunnel) + } + if o := outcomeOf(report, takesOverID); o.Action != "removed" || !strings.Contains(o.Detail, "retired") { + t.Errorf("the retirement is not what the apply says it did to the file: %+v", o) + } + // And the account still carries what was found, read from the kept original. + if report.Tunnel.Port != 51900 || report.Tunnel.Peers != 2 { + t.Errorf("the account lost what the tunnel was: %+v", report.Tunnel) + } +} + +func TestATakeNotProvenKeepsTheFoundConfigurationAndSaysSo(t *testing.T) { + cases := map[string]struct { + handshakes string + fail error + says string + }{ + "no peer at all": {"", nil, "no peer has handshaken on mesh0"}, + "every handshake at zero": {"PEER-A=\t0\nPEER-B=\t0\n", nil, "no peer has handshaken on mesh0"}, + "wg is not there": {"", errors.New(`exec: "wg": executable file not found in $PATH`), "executable file not found"}, + "the answer is nonsense": {"unable to access interface\n", nil, "not a peer and a time"}, + } + for name, c := range cases { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes, m.handshakesFail = c.handshakes, c.fail + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + report, state := applyAdopted(t, d, store.State{}, m, dir) + + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatalf("%s: a take not proven lost the found configuration", name) + } + if _, held := state.HeldAt(takesOverID); !held { + t.Errorf("%s: the hold ended although the take is not proven", name) + } + if _, retired := state.RetiredAt(config); retired { + t.Errorf("%s: recorded as retired", name) + } + if report.Tunnel == nil || report.Tunnel.State != Taken || + !strings.Contains(report.Tunnel.Note, "taken, not yet proven") || + !strings.Contains(report.Tunnel.Note, c.says) || !strings.Contains(report.Tunnel.Note, "is kept") { + t.Errorf("%s: the account does not say the take is not proven and why: %+v", name, report.Tunnel) + } + + // A later apply that finds a peer through retires it: the take itself need not be the one. + m.handshakes, m.handshakesFail = handshaken, nil + _, state = applyAdopted(t, d, state, m, dir) + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Errorf("%s: the apply after the take was proven kept the found configuration", name) + } + if _, retired := state.RetiredAt(config); !retired { + t.Errorf("%s: the later retirement was not recorded", name) + } + } +} + +func TestAFoundConfigurationWhoseKeptOriginalIsMissingIsNotRetired(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + held, _ := state.HeldAt(takesOverID) + if err := os.Remove(held.Kept); err != nil { + t.Fatal(err) + } + + m.handshakes = handshaken + report, state := applyAdopted(t, d, state, m, dir) + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatal("the found configuration was removed with no kept original left of it") + } + if _, still := state.HeldAt(takesOverID); !still { + t.Error("the hold ended although nothing was retired") + } + if _, retired := state.RetiredAt(config); retired { + t.Error("recorded as retired") + } + if report.Tunnel == nil || !strings.Contains(report.Tunnel.Note, "is not retired") || + !strings.Contains(report.Tunnel.Note, held.Kept+" is missing") { + t.Errorf("the account does not say the kept original is missing: %+v", report.Tunnel) + } +} + +func TestAFoundConfigurationRewrittenSinceItWasFoundIsKeptAgainBeforeItGoes(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + rewritten := foundConf + "\n[Peer]\nPublicKey = PEER-C=\nAllowedIPs = 192.0.2.4/32\n" + if err := os.WriteFile(config, []byte(rewritten), 0o600); err != nil { + t.Fatal(err) + } + + m.handshakes = handshaken + _, state = applyAdopted(t, d, state, m, dir) + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Fatal("a proven take kept a rewritten configuration") + } + retired, _ := state.RetiredAt(config) + if first, _ := os.ReadFile(retired.Kept); string(first) != foundConf { + t.Errorf("the first original was overwritten: %q", first) + } + kept, _ := filepath.Glob(filepath.Join(dir, "kept", "*-wg0.conf")) + var found bool + for _, k := range kept { + if got, _ := os.ReadFile(k); string(got) == rewritten { + found = true + } + } + if !found { + t.Errorf("what the file held when it was retired was not kept: %v", kept) + } +} + +func TestARetiredTakeIsSteadyAndItsFoundUnitFindsNothingToDo(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes = handshaken + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + retired, _ := state.RetiredAt(config) + + // wg-quick@wg0 with no configuration: inactive, and disabled — the check of it every apply + // makes must find nothing to do and fail on nothing. + m.asked = nil + report, again := applyAdopted(t, d, state, m, dir) + if report.Changed() { + t.Errorf("an apply after the retirement moved the machine: %+v", report.Outcomes) + } + if m.did("systemctl stop wg-quick@wg0") || m.did("systemctl start wg-quick@wg0") { + t.Errorf("the retired tunnel's unit was acted on: %v", m.asked) + } + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Error("the found configuration came back") + } + if _, held := again.HeldAt(takesOverID); held { + t.Error("a retired configuration is held again") + } + if r, ok := again.RetiredAt(config); !ok || r != retired { + t.Errorf("the retirement was not kept as it was: %+v", again.Retired) + } + if o := outcomeOf(report, takesOverID); o.Action != "unchanged" || !strings.Contains(o.Detail, "retired") { + t.Errorf("the retired configuration is not said as retired: %+v", o) + } + if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Kept != retired.Kept || + !strings.Contains(report.Tunnel.Note, "retired") || report.Tunnel.Port != 51900 { + t.Errorf("the account of a retired take does not say so: %+v", report.Tunnel) + } + + // Enabled at boot again by a person: disabled again, as any take does, and still no error. + m.units["wg-quick@wg0"].enabled = "enabled" + _, _ = applyAdopted(t, d, again, m, dir) + if m.units["wg-quick@wg0"].enabled != "disabled" { + t.Error("the found unit enabled again by hand was left to start at boot") + } +} + +func TestUndeclaringThePrivateNetworkAfterRetirementBringsNothingBack(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes = handshaken + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + + // The private network unassigned: only something else is declared. + other := adopted(t, `{"taken":[],"untaken":{}}`, + `{"id":"other.file","type":"file","path":"`+filepath.Join(dir, "other.conf")+`","content":"x\n"}`) + m.asked = nil + _, after := applyAdopted(t, other, state, m, dir) + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Fatal("undeclaring the private network brought the found configuration back") + } + if m.did("systemctl start wg-quick@wg0") || m.did("systemctl enable wg-quick@wg0") { + t.Errorf("undeclaring the private network started the found tunnel: %v", m.asked) + } + if _, ok := after.RetiredAt(config); !ok { + t.Error("the retirement was forgotten with the private network") + } + + // Assigned again, it finds the configuration retired rather than missing, and raises the + // mesh's interface. + report, _ := applyAdopted(t, d, after, m, dir) + if report.Tunnel == nil || report.Tunnel.State != Taken { + t.Errorf("the private network assigned again did not take the tunnel: %+v", report.Tunnel) + } + if _, err := os.Lstat(config); !os.IsNotExist(err) { + t.Error("assigning the private network again brought the found configuration back") + } +} + +func TestAPlanSaysTheFoundConfigurationIsRetiredWhenTheTakeIsProven(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + + plan := Plan(d, store.State{}, store.OriginDeclared) + report, state := applyAdopted(t, d, store.State{}, m, dir) + if got, want := strings.Join(ids(plan), " "), strings.Join(outcomeIDs(report), " "); got != want { + t.Errorf("the plan said %q and the apply did %q", got, want) + } + var take Step + for _, s := range plan { + if s.ID == takesOverID { + take = s + } + } + if take.Verb != "hold" || take.Target != config || !strings.Contains(take.Why, "retired — removed from "+config) || + !strings.Contains(take.Why, "handshaking on mesh0") { + t.Errorf("the plan does not say the found configuration is retired once proven: %+v", take) + } + // Held and still declared: never planned as forgotten. + if strings.Contains(verbs(Plan(d, state, store.OriginDeclared)), "forget "+takesOverID) { + t.Error("the plan forgets a hold the apply keeps") + } + + m.handshakes = handshaken + _, state = applyAdopted(t, d, state, m, dir) + for _, s := range Plan(d, state, store.OriginDeclared) { + if s.ID == takesOverID && (s.Verb != "check" || !strings.Contains(s.Why, "retired once the take")) { + t.Errorf("a retired configuration is planned as %+v", s) + } + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index 5cc738f..d0c6e63 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -706,9 +706,10 @@ type Service struct { // TakesOver names the found tunnel this service replaces (novox/hq ADR 0105): before this unit // is started, the named unit is stopped and disabled — never flushed — and its configuration - // file is kept like any held file. Only on an adopted node, and only said by the controller, - // which knows the found tunnel's key is this node's own: without that, starting this unit on - // the found one's port would drop every peer's packets. + // file is kept like any held file — until the take is proven by a peer's handshake, and then + // retired, its original staying kept (novox/hq ADR 0119). Only on an adopted node, and only + // said by the controller, which knows the found tunnel's key is this node's own: without that, + // starting this unit on the found one's port would drop every peer's packets. TakesOver *TakeOver `json:"takes-over,omitempty"` } diff --git a/internal/store/store.go b/internal/store/store.go index 71485a0..87c7b90 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -170,6 +170,28 @@ type State struct { // the bundle carried in the binary is not applied again: what genesis applied was rewritten // for this machine, and the mesh has said more since (novox/hq issue 104). Genesis *Genesis `json:"genesis,omitempty"` + + // Retired is each found tunnel configuration the mesh removed from where its unit reads it, + // once the private network's take of that tunnel was proven (novox/hq ADR 0119). + // + // **Not a hold, and never released with one.** The hold on the found configuration ends at the + // retirement — what it held for has been replaced, and the node stops reporting it — so without + // this the next apply would find no hold and no file and read the take as one whose + // configuration vanished before it could be kept. It is kept whether or not the private + // network stays declared: undeclaring brings nothing back (ADR 0118), and a private network + // assigned again finds the tunnel's configuration retired rather than missing. + Retired []Retired `json:"retired,omitempty"` +} + +// Retired is a found configuration the mesh removed once what replaced it was proven (novox/hq +// ADR 0119): where it was, under which hold it had been kept, and where its original still is. +type Retired struct { + ID string `json:"id"` + Path string `json:"path"` + // Kept is the original as found (novox/hq ADR 0100) — the record of what the predecessor was, + // and a person's way back if one is ever wanted. The mesh never copies it back. + Kept string `json:"kept"` + At time.Time `json:"at"` } // Modes a node can be in (novox/hq ADR 0100). @@ -312,6 +334,42 @@ func (s *State) Release(id string) { } } +// RetiredAt returns the retirement of the found configuration at a path, if the mesh retired one. +func (s State) RetiredAt(path string) (Retired, bool) { + for _, r := range s.Retired { + if r.Path == path { + return r, true + } + } + return Retired{}, false +} + +// RecordRetired adds or replaces the retirement of the configuration at one path. +func (s *State) RecordRetired(r Retired) { + for i, existing := range s.Retired { + if existing.Path == r.Path { + s.Retired[i] = r + return + } + } + s.Retired = append(s.Retired, r) +} + +// Unretire forgets a retirement: the configuration is at its path again, put back by a person, and +// is found — and kept — afresh. +func (s *State) Unretire(path string) { + kept := s.Retired[:0] + for _, r := range s.Retired { + if r.Path != path { + kept = append(kept, r) + } + } + s.Retired = kept + if len(s.Retired) == 0 { + s.Retired = nil + } +} + // Find returns what was applied under an identity. func (s State) Find(id string) (Applied, bool) { for _, r := range s.Resources { diff --git a/internal/tunnel/tunnel.go b/internal/tunnel/tunnel.go index 9e1793d..5590304 100644 --- a/internal/tunnel/tunnel.go +++ b/internal/tunnel/tunnel.go @@ -3,7 +3,9 @@ // // On an adopted node that is the hub, the mesh's interface is raised with the found interface's // private key, on its port, with its address and range, and every peer it had. The found interface -// is stopped, never flushed; its configuration stays on disk. What this package does is the +// is stopped, never flushed; its configuration stays on disk until the take is proven — a peer +// has handshaken with the mesh's interface — and is then retired (novox/hq ADR 0119). What this +// package does is the // reading: which interface is there, what its file says, and what of that travels to the mesh — // everything but the private key, which becomes the node's own overlay key and is stored the way // that key is stored. @@ -151,6 +153,47 @@ func Find(ctx context.Context, run Runner, named string) (Found, error) { return found, nil } +// Handshaken is how many peers of an interface have completed a handshake with it: the proof that +// the interface carries the tunnel, rather than merely being up (novox/hq ADR 0119). +// +// Asked of the running interface, since a handshake is a fact about the kernel's tunnel that no +// file records. A question that cannot be asked — no `wg` on the machine, no such interface, a +// permission refused — is an error and never a zero: "no peer has handshaken" retires nothing +// either, but it is a different thing to tell a person. +func Handshaken(ctx context.Context, run Runner, iface string) (int, error) { + out, err := run(ctx, "wg", "show", iface, "latest-handshakes") + if err != nil { + return 0, fmt.Errorf("cannot ask %s which peers have handshaken: %w", iface, err) + } + return ParseHandshakes(out) +} + +// ParseHandshakes reads `wg show latest-handshakes`: one line per peer, its public key +// and the Unix time of its latest handshake, tab-separated — zero for a peer that never has. What +// is counted is the peers with a time. A line that is not a key and a time is refused rather than +// skipped: output this does not understand is not evidence of anything. +func ParseHandshakes(out string) (int, error) { + n := 0 + for i, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) != 2 { + return 0, fmt.Errorf("line %d of the handshakes is not a peer and a time: %q", i+1, line) + } + at, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil || at < 0 { + return 0, fmt.Errorf("line %d of the handshakes does not end in a time: %q", i+1, line) + } + if at > 0 { + n++ + } + } + return n, nil +} + func orNone(names []string) string { if len(names) == 0 { return "none" diff --git a/internal/tunnel/tunnel_test.go b/internal/tunnel/tunnel_test.go index 19d73b3..c35ca4b 100644 --- a/internal/tunnel/tunnel_test.go +++ b/internal/tunnel/tunnel_test.go @@ -194,3 +194,49 @@ func TestAFoundTunnelReadsItsMTU(t *testing.T) { t.Fatalf("a config with no MTU must leave it zero; got %d", f2.MTU) } } + +// novox/hq ADR 0119: a take is proven by a handshake on the mesh's interface, read from `wg show +// latest-handshakes` — as wg prints it, a key and a Unix time per peer, zero for never. +func TestAHandshakeIsAPeerWithATime(t *testing.T) { + cases := map[string]struct { + out string + want int + }{ + "two peers, one handshaken": {"PEER-A=\t1790000000\nPEER-B=\t0\n", 1}, + "every peer handshaken": {"PEER-A=\t1790000000\nPEER-B=\t1790000042\n", 2}, + "no peer ever": {"PEER-A=\t0\nPEER-B=\t0\n", 0}, + "an interface with no peer": {"", 0}, + "spaces, a trailing line": {"PEER-A= 1790000000\n\n", 1}, + } + for name, c := range cases { + got, err := ParseHandshakes(c.out) + if err != nil || got != c.want { + t.Errorf("%s: %d peer(s) handshaken (%v), want %d", name, got, err, c.want) + } + } + // Output that is not a key and a time is not evidence of anything, and not a zero either. + for _, nonsense := range []string{"PEER-A=\n", "PEER-A=\tyesterday\n", "PEER-A=\t-1\n", "a b c\n"} { + if _, err := ParseHandshakes(nonsense); err == nil { + t.Errorf("%q was read as handshakes", nonsense) + } + } +} + +func TestHandshakesThatCannotBeAskedAreAnErrorNotAZero(t *testing.T) { + var asked string + ok := func(_ context.Context, name string, args ...string) (string, error) { + asked = name + " " + strings.Join(args, " ") + return "PEER-A=\t1790000000\n", nil + } + if n, err := Handshaken(context.Background(), ok, "mesh0"); err != nil || n != 1 || + asked != "wg show mesh0 latest-handshakes" { + t.Fatalf("asked %q and read %d (%v)", asked, n, err) + } + missing := func(context.Context, string, ...string) (string, error) { + return "", errors.New(`exec: "wg": executable file not found in $PATH`) + } + if _, err := Handshaken(context.Background(), missing, "mesh0"); err == nil || + !strings.Contains(err.Error(), "mesh0") { + t.Fatalf("a machine with no wg was read as one with no handshake: %v", err) + } +} -- 2.54.0 From 50776b86133c23bc27daf501babd0efca83a6966 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 00:55:50 +0200 Subject: [PATCH 2/2] review: a retired configuration that comes back is retired again on its first original, and only wg-quick's own file is ever removed (hq ADR 0119) Put back by hand, it was found afresh and its copy became the hold's original, so a machine that kept restoring it kept growing copies and lost which one was first. The first original now stays the record's, content that differs is kept once beside it, and the note says a rollback means unassigning the private network. Nothing is removed unless it is /.conf, not a path the mesh writes, and not a link, which would leave the key-bearing target behind. --- internal/apply/apply.go | 2 +- internal/apply/takeover.go | 166 +++++++++++++++++++------- internal/apply/takeover_test.go | 204 ++++++++++++++++++++++++++++++++ internal/store/store.go | 33 +++--- 4 files changed, 340 insertions(+), 65 deletions(-) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index af94644..aced7bd 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -518,7 +518,7 @@ func ApplyKeeping( // configuration is retired — here, after the mesh's service applied, so in the apply // of the take itself only if a peer is already through; otherwise a later apply // retires it (novox/hq ADR 0119). What it did replaces what the take said of the file. - if o, did := retireFound(ctx, svc, &known, run, keep, report.Tunnel, time.Now().UTC()); did { + if o, did := retireFound(ctx, svc, d, &known, run, keep, report.Tunnel, time.Now().UTC()); did { if tookAt >= 0 { report.Outcomes[tookAt] = o } diff --git a/internal/apply/takeover.go b/internal/apply/takeover.go index 8bc7130..0669746 100644 --- a/internal/apply/takeover.go +++ b/internal/apply/takeover.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "strings" "time" @@ -100,30 +101,48 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, // **Unless it was retired** (novox/hq ADR 0119): the take was proven and the mesh removed // it, so there is nothing to hold and nothing missing — only where its original is, which // the retirement recorded. A hold still standing is let go: that is what retiring it meant. - // One put back at its path by a person is on the machine again, with no hold, and is found - // and kept afresh — the same content kept once, as any original is — and retired again by - // the first apply that finds the take still proven. + // + // **One that comes back is held again on its FIRST original** — the one kept before anything + // happened to it (ADR 0100), never whatever was put back — and retired again by the first + // apply that finds the take still proven, keeping what came back only if it differs from + // what is already kept. Put back by hand while the private network is assigned, it is not a + // rollback: that means unassigning the private network first, and the note says so. file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config} var held store.Held retired, wasRetired := known.RetiredAt(t.Config) - if wasRetired && !present(t.Config) { + cameBack := wasRetired && present(t.Config) + switch { + case wasRetired && !cameBack: known.Release(id) held = store.Held{Kept: retired.Kept} out = begin(file) out.Action = "unchanged" facts.Note = "the found configuration " + t.Config + " was retired once the take was proven; " + "its original is kept at " + retired.Kept + " and the mesh never brings it back" - } else { - if wasRetired { - known.Unretire(t.Config) - } + default: was, already := known.HeldAt(id) - out, held, err = hold(ctx, sys, file, module, was, already, - "the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now) + why := "the configuration of the tunnel " + t.Interface + ", taken over by " + svc.Unit + if cameBack && !already { + digest := retired.Digest + if digest == "" { + if raw, err := os.ReadFile(retired.Kept); err == nil { + digest = digestOf(string(raw)) + } + } + was = store.Held{ID: id, Module: module, Kind: string(declaration.TypeFile), Target: t.Config, + Since: now, Why: why, Kept: retired.Kept, Digest: digest} + already = true + } + out, held, err = hold(ctx, sys, file, module, was, already, why, run, keep, now) if err != nil { return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err) } known.RecordHeld(held) + if cameBack { + facts.Note = "the found configuration " + t.Config + " came back after it was retired; while the " + + "private network is assigned the mesh retires it again, so rolling back to the found tunnel " + + "means unassigning the private network first" + } } facts.Kept = held.Kept // What the file says, for the report: from the machine, or from the kept original when the @@ -218,7 +237,11 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, } out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found" - if wasRetired && out.Action == "unchanged" { + switch { + case cameBack: + out.Detail = "the tunnel " + t.Interface + "'s configuration, back after it was retired; held until " + + "it is retired again" + case wasRetired: out.Detail = "the tunnel " + t.Interface + "'s configuration, retired once the take was proven" } if held.Kept != "" { @@ -371,6 +394,10 @@ func restoreFound(ctx context.Context, sys system.System, unit string, run Runne facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had" } +// wireguardDir is where a found tunnel's configuration may be retired from: wg-quick's own, and +// nowhere else. A variable so a test can hand in a directory. +var wireguardDir = tunnel.ConfigDir + // retireFound removes the found tunnel's configuration from where its unit reads it, once the take // is proven, and ends the hold on it (novox/hq ADR 0119). Asked after the mesh's service applied // and the tunnel reads as taken; retired says whether this apply retired it, and out is then what @@ -379,20 +406,29 @@ func restoreFound(ctx context.Context, sys system.System, unit string, run Runne // **Proven is taken and a handshake.** Taken alone — the found unit down and disabled, the mesh's // interface up — says the mesh's interface exists, not that any peer reaches it: an interface up // with the wrong key is taken and carries nothing. A peer that has completed a handshake with it -// has checked its key, so that is the proof, asked of the kernel through `wg`. Anything short of -// one — no peer yet, every time zero, `wg` missing or failing — keeps the file, and the account -// says which: a take that never proves itself is visible rather than silently retired. +// has checked its key, so that is the proof, asked of the kernel through `wg`. Any handshake counts, +// however old: a change to the mesh's configuration restarts its unit, which recreates the +// interface and resets its counters, so a time that is there at all was made by this interface. +// Anything short of one — no peer yet, every time zero, `wg` missing or failing — keeps the file, +// and the account says which: a take that never proves itself is visible rather than silently +// retired. +// +// **Only what the take names, and only wg-quick's own file.** Nothing is removed unless the path +// is exactly `/.conf`, is not a path the mesh itself writes, and is +// a file rather than a link: removing a link would leave the key-bearing file it points at where it +// is, a retirement in name only, so that one is said and left to a person. // // **The original must still be kept.** It is the record of what the predecessor was and a // person's only way back (ADR 0100); a kept copy that has gone missing is said, and the file is -// not removed, since removing it then would lose the only copy. What is on disk now, if something -// other than the mesh rewrote it since it was found, is kept too before it goes — by content, so -// the first original is never overwritten. +// not removed, since removing it then would lose the only copy. What is on disk now, if it differs +// from the first original and from what was kept at the last retirement, is kept too before it +// goes — by content, so the first original is never overwritten and a file that keeps coming back +// the same keeps nothing more. // // The found unit is left disabled; without its configuration it cannot raise the interface, so // every later apply's check of it finds nothing to do. Nothing here ever writes the file back. -func retireFound(ctx context.Context, svc *declaration.Service, known *store.State, run Runner, keep Keep, - facts *TakenTunnel, now time.Time) (out Outcome, retired bool) { +func retireFound(ctx context.Context, svc *declaration.Service, d *declaration.Declaration, known *store.State, + run Runner, keep Keep, facts *TakenTunnel, now time.Time) (out Outcome, retired bool) { t := svc.TakesOver id := takeOverID(svc) if facts.State != Taken { @@ -409,6 +445,10 @@ func retireFound(ctx context.Context, svc *declaration.Service, known *store.Sta } facts.Note += note } + notRetired := func(why string) (Outcome, bool) { + say("the found configuration " + t.Config + " is not retired: " + why) + return Outcome{}, false + } mesh := strings.TrimPrefix(svc.Unit, "wg-quick@") peers, err := tunnel.Handshaken(ctx, tunnel.Runner(run), mesh) if err != nil { @@ -421,12 +461,26 @@ func retireFound(ctx context.Context, svc *declaration.Service, known *store.Sta return out, false } proven := fmt.Sprintf("proven: %d peer(s) handshaken on %s", peers, mesh) + say(proven) + + // What may be removed at all. + if want := filepath.Join(wireguardDir, t.Interface+".conf"); t.Config != want { + return notRetired("only " + want + ", the found interface's own wg-quick configuration, is ever " + + "retired by the mesh, and the take names " + t.Config) + } + if known.Recorded(string(declaration.TypeFile), t.Config) || declaresFile(d, t.Config) { + return notRetired("it is a path the mesh itself writes") + } + if info, err := os.Lstat(t.Config); err == nil && info.Mode()&os.ModeSymlink != 0 { + target, _ := os.Readlink(t.Config) + return notRetired("it is a link to " + target + "; removing the link would leave the key-bearing file " + + "it points at, so it must be retired by hand — both are kept") + } // The kept original, read back — not just named in a record. if held.Kept == "" { - say(proven + ", and the found configuration " + t.Config + " is not retired: no original of it " + - "was kept, so removing it would leave no record of what the predecessor was") - return out, false + return notRetired("no original of it was kept, so removing it would leave no record of what the " + + "predecessor was") } original, err := os.ReadFile(held.Kept) if err != nil || (held.Digest != "" && digestOf(string(original)) != held.Digest) { @@ -436,49 +490,59 @@ func retireFound(ctx context.Context, svc *declaration.Service, known *store.Sta } else if !errors.Is(err, os.ErrNotExist) { why = "cannot be read (" + err.Error() + ")" } - say(proven + ", and the found configuration " + t.Config + " is not retired: its kept original " + - held.Kept + " " + why + ", so removing it would lose the only copy") - return out, false + return notRetired("its kept original " + held.Kept + " " + why + ", so removing it would lose the only copy") } + before, cameBack := known.RetiredAt(t.Config) + record := store.Retired{ID: id, Path: t.Config, Kept: held.Kept, Digest: digestOf(string(original)), At: now} + if cameBack { + // The first original stays the record's, and so does what the last retirement kept. + record.Extra, record.ExtraDigest, record.Again = before.Extra, before.ExtraDigest, before.Again+1 + } + newCopy := "" gone := !present(t.Config) if !gone { current, err := os.ReadFile(t.Config) if err != nil { - say(proven + ", and the found configuration " + t.Config + " is not retired: it cannot be read (" + - err.Error() + ")") - return out, false + return notRetired("it cannot be read (" + err.Error() + ")") } - if held.Digest != "" && digestOf(string(current)) != held.Digest { + if sum := digestOf(string(current)); sum != record.Digest && sum != record.ExtraDigest { if keep == nil { - say(proven + ", and the found configuration " + t.Config + " is not retired: it was rewritten " + - "since it was found and this host has nowhere to keep what it holds now") - return out, false + return notRetired("it holds something other than its kept original and this host has nowhere " + + "to keep it") } - if _, err := keep(t.Config, current, 0o600); err != nil { - say(proven + ", and the found configuration " + t.Config + " is not retired: keeping what it " + - "holds now failed (" + err.Error() + ")") - return out, false + where, err := keep(t.Config, current, 0o600) + if err != nil { + return notRetired("keeping what it holds now failed (" + err.Error() + ")") } + record.Extra, record.ExtraDigest, newCopy = where, sum, where } if err := os.Remove(t.Config); err != nil && !errors.Is(err, os.ErrNotExist) { - say(proven + ", and the found configuration " + t.Config + " could not be removed (" + err.Error() + ")") - return out, false + return notRetired("removing it failed (" + err.Error() + ")") } if present(t.Config) { - say(proven + ", and the found configuration " + t.Config + " is still there after it was removed") - return out, false + return notRetired("it is still there after it was removed") } } - known.RecordRetired(store.Retired{ID: id, Path: t.Config, Kept: held.Kept, At: now}) + known.RecordRetired(record) known.Release(id) facts.Kept = held.Kept - say(proven + "; the found configuration " + t.Config + " is retired — its original kept at " + - held.Kept + ", " + t.Unit + " left disabled, and the mesh never brings it back") - - out = Outcome{ID: id, Type: string(declaration.TypeFile), Target: t.Config, Action: "removed", - Detail: "retired: the take of " + t.Interface + " is " + proven + "; original kept at " + held.Kept} + copied := "" + if newCopy != "" { + copied = "; what it held, which differed from the original, is kept at " + newCopy + } + out = Outcome{ID: id, Type: string(declaration.TypeFile), Target: t.Config, Action: "removed"} + switch { + case cameBack: + say("the found configuration came back and was retired again — its original still kept at " + + held.Kept + copied + "; rolling back to the found tunnel means unassigning the private network first") + out.Detail = "the found configuration came back and was retired again; original kept at " + held.Kept + copied + default: + say("the found configuration " + t.Config + " is retired — its original kept at " + held.Kept + copied + + ", " + t.Unit + " left disabled, and the mesh never brings it back") + out.Detail = "retired: the take of " + t.Interface + " is " + proven + "; original kept at " + held.Kept + copied + } if gone { // Already gone — removed by something other than the mesh, or by an apply whose record was // never saved. Nothing removed here; the hold ends all the same. @@ -489,6 +553,16 @@ func retireFound(ctx context.Context, svc *declaration.Service, known *store.Sta return out, true } +// declaresFile is whether a declaration writes a file at a path. +func declaresFile(d *declaration.Declaration, path string) bool { + for _, r := range d.Resources { + if f, ok := r.(*declaration.File); ok && filepath.Clean(f.Path) == filepath.Clean(path) { + return true + } + } + return false +} + // takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since // a machine has one private network. func takesOver(d *declaration.Declaration) *declaration.Service { diff --git a/internal/apply/takeover_test.go b/internal/apply/takeover_test.go index 058c1d2..8b6c24e 100644 --- a/internal/apply/takeover_test.go +++ b/internal/apply/takeover_test.go @@ -66,6 +66,10 @@ func aHubInUse(t *testing.T) (dir, config, mesh, keyFile string, m *machine) { "wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"}, }} takeoverRecheck = 0 + // The found configuration lives in this test's own wireguard directory (novox/hq ADR 0119). + was := wireguardDir + wireguardDir = dir + t.Cleanup(func() { wireguardDir = was }) return dir, config, mesh, keyFile, m } @@ -509,3 +513,203 @@ func TestAPlanSaysTheFoundConfigurationIsRetiredWhenTheTakeIsProven(t *testing.T } } } + +// keptCopies is every copy kept of the found configuration. +func keptCopies(t *testing.T, dir string) []string { + t.Helper() + kept, err := filepath.Glob(filepath.Join(dir, "kept", "*-wg0.conf")) + if err != nil { + t.Fatal(err) + } + return kept +} + +func TestAConfigurationPutBackIsRetiredAgainOnItsFirstOriginalAndSettles(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.handshakes = handshaken + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + first, _ := state.RetiredAt(config) + + // Put back by hand with the original, while no peer is through yet: held on the first + // original, and the account says what a rollback takes. + write(t, config, foundConf) + m.handshakes = "" + report, state := applyAdopted(t, d, state, m, dir) + if held, ok := state.HeldAt(takesOverID); !ok || held.Kept != first.Kept { + t.Fatalf("what came back is not held on the first original: %+v", held) + } + if report.Tunnel == nil || !strings.Contains(report.Tunnel.Note, "came back after it was retired") || + !strings.Contains(report.Tunnel.Note, "unassigning the private network first") { + t.Errorf("the account does not say a rollback means unassigning the private network: %+v", report.Tunnel) + } + + // Proven: retired again, nothing more kept, said once. + m.handshakes = handshaken + report, state = applyAdopted(t, d, state, m, dir) + again, _ := state.RetiredAt(config) + if _, err := os.Lstat(config); !os.IsNotExist(err) || again.Kept != first.Kept || again.Extra != "" { + t.Fatalf("put back as it was, it was not retired again on the first original: %+v", again) + } + if o := outcomeOf(report, takesOverID); o.Action != "removed" || + !strings.HasPrefix(o.Detail, "the found configuration came back and was retired again") || + strings.Contains(o.Detail, "differed") { + t.Errorf("the second retirement is not said as one: %+v", o) + } + if !strings.Contains(report.Tunnel.Note, "unassigning the private network first") { + t.Errorf("the account does not say what a rollback takes: %q", report.Tunnel.Note) + } + if n := len(keptCopies(t, dir)); n != 1 { + t.Errorf("%d copies kept of one content", n) + } + if report, _ := applyAdopted(t, d, state, m, dir); report.Changed() { + t.Errorf("a steady machine moved after the second retirement: %+v", report.Outcomes) + } + + // Put back with something else: that is kept beside the first original, which stays the record's. + other := strings.Replace(foundConf, "PEER-B=", "PEER-Z=", 1) + write(t, config, other) + report, state = applyAdopted(t, d, state, m, dir) + third, _ := state.RetiredAt(config) + if third.Kept != first.Kept || third.Extra == "" || third.Extra == first.Kept { + t.Fatalf("other content was not kept apart from the first original: %+v", third) + } + if got, _ := os.ReadFile(third.Extra); string(got) != other { + t.Errorf("the extra copy does not hold what was put back: %q", got) + } + if o := outcomeOf(report, takesOverID); !strings.Contains(o.Detail, third.Extra) || + !strings.Contains(report.Tunnel.Note, third.Extra) { + t.Errorf("where the extra copy is was not said: %+v / %q", o, report.Tunnel.Note) + } + + // And the same other content again: nothing more kept, the record as it was. + write(t, config, other) + report, state = applyAdopted(t, d, state, m, dir) + fourth, _ := state.RetiredAt(config) + if fourth.Kept != first.Kept || fourth.Extra != third.Extra || len(keptCopies(t, dir)) != 2 { + t.Errorf("the same content put back again grew the copies: %+v, %v", fourth, keptCopies(t, dir)) + } + if o := outcomeOf(report, takesOverID); strings.Contains(o.Detail, "differed") { + t.Errorf("a copy already kept was said as new: %+v", o) + } + if report, _ := applyAdopted(t, d, state, m, dir); report.Changed() { + t.Errorf("a steady machine moved: %+v", report.Outcomes) + } +} + +func TestOnlyWgQuicksOwnConfigurationIsRetired(t *testing.T) { + // Not under the wireguard directory. + dir, config, mesh, keyFile, m := aHubInUse(t) + wireguardDir = filepath.Join(dir, "elsewhere") + m.handshakes = handshaken + report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir) + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatal("a configuration outside wg-quick's directory was removed") + } + if _, held := state.HeldAt(takesOverID); !held || !strings.Contains(report.Tunnel.Note, "is not retired: only ") { + t.Errorf("the refusal is not said, or the hold ended: %+v", report.Tunnel) + } + + // A path the mesh itself writes. + dir, config, mesh, keyFile, m = aHubInUse(t) + m.handshakes = handshaken + known := store.State{} + known.Record(store.Applied{ID: "bundle.wg0", Type: "file", Target: config, Origin: store.OriginCarried}) + report, _ = applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), known, m, dir) + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatal("a path the mesh writes was retired") + } + if !strings.Contains(report.Tunnel.Note, "a path the mesh itself writes") { + t.Errorf("the refusal is not said: %+v", report.Tunnel) + } +} + +func TestAFoundConfigurationThatIsALinkIsKeptAndLeftToAPerson(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + target := filepath.Join(dir, "predecessor", "hub.conf") + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + t.Fatal(err) + } + write(t, target, foundConf) + if err := os.Remove(config); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, config); err != nil { + t.Fatal(err) + } + m.handshakes = handshaken + report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir) + if _, err := os.Lstat(config); err != nil { + t.Fatal("the link was removed, leaving the key-bearing file it points at") + } + if got, _ := os.ReadFile(target); string(got) != foundConf { + t.Fatal("the file the link points at was touched") + } + held, ok := state.HeldAt(takesOverID) + if !ok { + t.Fatal("the hold ended") + } + if kept, _ := os.ReadFile(held.Kept); string(kept) != foundConf { + t.Errorf("what was kept is not what the link points at: %q", kept) + } + if !strings.Contains(report.Tunnel.Note, "is a link to "+target) || !strings.Contains(report.Tunnel.Note, "by hand") { + t.Errorf("the account does not say the link must be retired by hand: %q", report.Tunnel.Note) + } +} + +func TestARetirementWhoseRecordWasNeverSavedIsRecordedByTheNextApply(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + held, _ := state.HeldAt(takesOverID) + // An apply removed the file and stopped before its state was saved. + if err := os.Remove(config); err != nil { + t.Fatal(err) + } + m.handshakes = handshaken + report, state := applyAdopted(t, d, state, m, dir) + if r, ok := state.RetiredAt(config); !ok || r.Kept != held.Kept { + t.Fatalf("the retirement was not recorded: %+v", state.Retired) + } + if _, still := state.HeldAt(takesOverID); still { + t.Error("the hold did not end") + } + if o := outcomeOf(report, takesOverID); o.Action != "unchanged" || !strings.Contains(o.Detail, "already gone") { + t.Errorf("a file already gone is not said as such: %+v", o) + } +} + +func TestARemovalThatFailsKeepsTheFileAndTheHold(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root removes from a directory it may not write to") + } + dir, _, mesh, keyFile, m := aHubInUse(t) + wg := filepath.Join(dir, "wireguard") + if err := os.MkdirAll(wg, 0o700); err != nil { + t.Fatal(err) + } + config := filepath.Join(wg, "wg0.conf") + write(t, config, foundConf) + wireguardDir = wg + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + + if err := os.Chmod(wg, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(wg, 0o700) }) + m.handshakes = handshaken + report, state := applyAdopted(t, d, state, m, dir) + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatal("the found configuration is gone although it could not be removed") + } + if _, held := state.HeldAt(takesOverID); !held { + t.Error("the hold ended although nothing was retired") + } + if _, retired := state.RetiredAt(config); retired { + t.Error("recorded as retired") + } + if !strings.Contains(report.Tunnel.Note, "removing it failed") || !strings.Contains(report.Tunnel.Note, "permission denied") { + t.Errorf("the failed removal is not said: %q", report.Tunnel.Note) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 87c7b90..7306725 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -189,9 +189,21 @@ type Retired struct { ID string `json:"id"` Path string `json:"path"` // Kept is the original as found (novox/hq ADR 0100) — the record of what the predecessor was, - // and a person's way back if one is ever wanted. The mesh never copies it back. - Kept string `json:"kept"` - At time.Time `json:"at"` + // and a person's way back if one is ever wanted. The mesh never copies it back. It is the FIRST + // original, and stays so however often the file comes back: a retirement repeated never moves + // it. Digest is what it holds. + Kept string `json:"kept"` + Digest string `json:"digest,omitempty"` + // Extra is where what was at the path when it was last retired is kept, when that differed from + // the first original — rewritten since it was found, or put back with other content — and + // ExtraDigest what it holds. One copy per distinct content: a file that comes back as it was + // last retired keeps nothing more. + Extra string `json:"extra,omitempty"` + ExtraDigest string `json:"extra_digest,omitempty"` + At time.Time `json:"at"` + // Again is how many times the configuration came back after it was retired, and was retired + // again (novox/hq ADR 0119). + Again int `json:"again,omitempty"` } // Modes a node can be in (novox/hq ADR 0100). @@ -355,21 +367,6 @@ func (s *State) RecordRetired(r Retired) { s.Retired = append(s.Retired, r) } -// Unretire forgets a retirement: the configuration is at its path again, put back by a person, and -// is found — and kept — afresh. -func (s *State) Unretire(path string) { - kept := s.Retired[:0] - for _, r := range s.Retired { - if r.Path != path { - kept = append(kept, r) - } - } - s.Retired = kept - if len(s.Retired) == 0 { - s.Retired = nil - } -} - // Find returns what was applied under an identity. func (s State) Find(id string) (Applied, bool) { for _, r := range s.Resources { -- 2.54.0