novox/mesh-host
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

review: a retired configuration that comes back is retired again on its first original, and only wg-quick's own file is ever removed (hq ADR 0119)

Browse Source 
Put back by hand, it was found afresh and its copy became the hold's original, so a machine that
kept restoring it kept growing copies and lost which one was first. The first original now stays
the record's, content that differs is kept once beside it, and the note says a rollback means
unassigning the private network. Nothing is removed unless it is <wireguard dir>/<iface>.conf,
not a path the mesh writes, and not a link, which would leave the key-bearing target behind.
This commit is contained in:
jochen
2026-09-27 00:55:50 +02:00
parent b462f461c6
commit 50776b8613
4 changed files with 340 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/apply/apply.go
+1 -1
View File
@@ -518,7 +518,7 @@ func ApplyKeeping(
// configuration is retired — here, after the mesh's service applied, so in the apply
// of the take itself only if a peer is already through; otherwise a later apply
// retires it (novox/hq ADR 0119). What it did replaces what the take said of the file.
if o, did := retireFound(ctx, svc, &known, run, keep, report.Tunnel, time.Now().UTC()); did {
if o, did := retireFound(ctx, svc, d, &known, run, keep, report.Tunnel, time.Now().UTC()); did {
if tookAt >= 0 {
report.Outcomes[tookAt] = o
}
internal/apply/takeover.go
+120 -46
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
@@ -100,30 +101,48 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service,
// **Unless it was retired** (novox/hq ADR 0119): the take was proven and the mesh removed
// it, so there is nothing to hold and nothing missing — only where its original is, which
// the retirement recorded. A hold still standing is let go: that is what retiring it meant.
// One put back at its path by a person is on the machine again, with no hold, and is found
// and kept afresh — the same content kept once, as any original is — and retired again by
// the first apply that finds the take still proven.
//
// **One that comes back is held again on its FIRST original** — the one kept before anything
// happened to it (ADR 0100), never whatever was put back — and retired again by the first
// apply that finds the take still proven, keeping what came back only if it differs from
// what is already kept. Put back by hand while the private network is assigned, it is not a
// rollback: that means unassigning the private network first, and the note says so.
file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config}
var held store.Held
retired, wasRetired := known.RetiredAt(t.Config)
if wasRetired && !present(t.Config) {
cameBack := wasRetired && present(t.Config)
switch {
case wasRetired && !cameBack:
known.Release(id)
held = store.Held{Kept: retired.Kept}
out = begin(file)
out.Action = "unchanged"
facts.Note = "the found configuration " + t.Config + " was retired once the take was proven; " +
"its original is kept at " + retired.Kept + " and the mesh never brings it back"
} else {
if wasRetired {
known.Unretire(t.Config)
}
default:
was, already := known.HeldAt(id)
out, held, err = hold(ctx, sys, file, module, was, already,
"the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now)
why := "the configuration of the tunnel " + t.Interface + ", taken over by " + svc.Unit
if cameBack && !already {
digest := retired.Digest
if digest == "" {
if raw, err := os.ReadFile(retired.Kept); err == nil {
digest = digestOf(string(raw))
}
}
was = store.Held{ID: id, Module: module, Kind: string(declaration.TypeFile), Target: t.Config,
Since: now, Why: why, Kept: retired.Kept, Digest: digest}
already = true
}
out, held, err = hold(ctx, sys, file, module, was, already, why, run, keep, now)
if err != nil {
return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err)
}
known.RecordHeld(held)
if cameBack {
facts.Note = "the found configuration " + t.Config + " came back after it was retired; while the " +
"private network is assigned the mesh retires it again, so rolling back to the found tunnel " +
"means unassigning the private network first"
}
}
facts.Kept = held.Kept
// What the file says, for the report: from the machine, or from the kept original when the
@@ -218,7 +237,11 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service,
}
out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found"
if wasRetired && out.Action == "unchanged" {
switch {
case cameBack:
out.Detail = "the tunnel " + t.Interface + "'s configuration, back after it was retired; held until " +
"it is retired again"
case wasRetired:
out.Detail = "the tunnel " + t.Interface + "'s configuration, retired once the take was proven"
}
if held.Kept != "" {
@@ -371,6 +394,10 @@ func restoreFound(ctx context.Context, sys system.System, unit string, run Runne
facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had"
}
// wireguardDir is where a found tunnel's configuration may be retired from: wg-quick's own, and
// nowhere else. A variable so a test can hand in a directory.
var wireguardDir = tunnel.ConfigDir
// retireFound removes the found tunnel's configuration from where its unit reads it, once the take
// is proven, and ends the hold on it (novox/hq ADR 0119). Asked after the mesh's service applied
// and the tunnel reads as taken; retired says whether this apply retired it, and out is then what
@@ -379,20 +406,29 @@ func restoreFound(ctx context.Context, sys system.System, unit string, run Runne
// **Proven is taken and a handshake.** Taken alone — the found unit down and disabled, the mesh's
// interface up — says the mesh's interface exists, not that any peer reaches it: an interface up
// with the wrong key is taken and carries nothing. A peer that has completed a handshake with it
// has checked its key, so that is the proof, asked of the kernel through `wg`. Anything short of
// one — no peer yet, every time zero, `wg` missing or failing — keeps the file, and the account
// says which: a take that never proves itself is visible rather than silently retired.
// has checked its key, so that is the proof, asked of the kernel through `wg`. Any handshake counts,
// however old: a change to the mesh's configuration restarts its unit, which recreates the
// interface and resets its counters, so a time that is there at all was made by this interface.
// Anything short of one — no peer yet, every time zero, `wg` missing or failing — keeps the file,
// and the account says which: a take that never proves itself is visible rather than silently
// retired.
//
// **Only what the take names, and only wg-quick's own file.** Nothing is removed unless the path
// is exactly `<wireguard dir>/<found interface>.conf`, is not a path the mesh itself writes, and is
// a file rather than a link: removing a link would leave the key-bearing file it points at where it
// is, a retirement in name only, so that one is said and left to a person.
//
// **The original must still be kept.** It is the record of what the predecessor was and a
// person's only way back (ADR 0100); a kept copy that has gone missing is said, and the file is
// not removed, since removing it then would lose the only copy. What is on disk now, if something
// other than the mesh rewrote it since it was found, is kept too before it goes — by content, so
// the first original is never overwritten.
// not removed, since removing it then would lose the only copy. What is on disk now, if it differs
// from the first original and from what was kept at the last retirement, is kept too before it
// goes — by content, so the first original is never overwritten and a file that keeps coming back
// the same keeps nothing more.
//
// The found unit is left disabled; without its configuration it cannot raise the interface, so
// every later apply's check of it finds nothing to do. Nothing here ever writes the file back.
func retireFound(ctx context.Context, svc *declaration.Service, known *store.State, run Runner, keep Keep,
facts *TakenTunnel, now time.Time) (out Outcome, retired bool) {
func retireFound(ctx context.Context, svc *declaration.Service, d *declaration.Declaration, known *store.State,
run Runner, keep Keep, facts *TakenTunnel, now time.Time) (out Outcome, retired bool) {
t := svc.TakesOver
id := takeOverID(svc)
if facts.State != Taken {
@@ -409,6 +445,10 @@ func retireFound(ctx context.Context, svc *declaration.Service, known *store.Sta
}
facts.Note += note
}
notRetired := func(why string) (Outcome, bool) {
say("the found configuration " + t.Config + " is not retired: " + why)
return Outcome{}, false
}
mesh := strings.TrimPrefix(svc.Unit, "wg-quick@")
peers, err := tunnel.Handshaken(ctx, tunnel.Runner(run), mesh)
if err != nil {
@@ -421,12 +461,26 @@ func retireFound(ctx context.Context, svc *declaration.Service, known *store.Sta
return out, false
}
proven := fmt.Sprintf("proven: %d peer(s) handshaken on %s", peers, mesh)
say(proven)
// What may be removed at all.
if want := filepath.Join(wireguardDir, t.Interface+".conf"); t.Config != want {
return notRetired("only " + want + ", the found interface's own wg-quick configuration, is ever " +
"retired by the mesh, and the take names " + t.Config)
}
if known.Recorded(string(declaration.TypeFile), t.Config) || declaresFile(d, t.Config) {
return notRetired("it is a path the mesh itself writes")
}
if info, err := os.Lstat(t.Config); err == nil && info.Mode()&os.ModeSymlink != 0 {
target, _ := os.Readlink(t.Config)
return notRetired("it is a link to " + target + "; removing the link would leave the key-bearing file " +
"it points at, so it must be retired by hand — both are kept")
}
// The kept original, read back — not just named in a record.
if held.Kept == "" {
say(proven + ", and the found configuration " + t.Config + " is not retired: no original of it " +
"was kept, so removing it would leave no record of what the predecessor was")
return out, false
return notRetired("no original of it was kept, so removing it would leave no record of what the " +
"predecessor was")
}
original, err := os.ReadFile(held.Kept)
if err != nil || (held.Digest != "" && digestOf(string(original)) != held.Digest) {
@@ -436,49 +490,59 @@ func retireFound(ctx context.Context, svc *declaration.Service, known *store.Sta
} else if !errors.Is(err, os.ErrNotExist) {
why = "cannot be read (" + err.Error() + ")"
}
say(proven + ", and the found configuration " + t.Config + " is not retired: its kept original " +
held.Kept + " " + why + ", so removing it would lose the only copy")
return out, false
return notRetired("its kept original " + held.Kept + " " + why + ", so removing it would lose the only copy")
}
before, cameBack := known.RetiredAt(t.Config)
record := store.Retired{ID: id, Path: t.Config, Kept: held.Kept, Digest: digestOf(string(original)), At: now}
if cameBack {
// The first original stays the record's, and so does what the last retirement kept.
record.Extra, record.ExtraDigest, record.Again = before.Extra, before.ExtraDigest, before.Again+1
}
newCopy := ""
gone := !present(t.Config)
if !gone {
current, err := os.ReadFile(t.Config)
if err != nil {
say(proven + ", and the found configuration " + t.Config + " is not retired: it cannot be read (" +
err.Error() + ")")
return out, false
return notRetired("it cannot be read (" + err.Error() + ")")
}
if held.Digest != "" && digestOf(string(current)) != held.Digest {
if sum := digestOf(string(current)); sum != record.Digest && sum != record.ExtraDigest {
if keep == nil {
say(proven + ", and the found configuration " + t.Config + " is not retired: it was rewritten " +
"since it was found and this host has nowhere to keep what it holds now")
return out, false
return notRetired("it holds something other than its kept original and this host has nowhere " +
"to keep it")
}
if _, err := keep(t.Config, current, 0o600); err != nil {
say(proven + ", and the found configuration " + t.Config + " is not retired: keeping what it " +
"holds now failed (" + err.Error() + ")")
return out, false
where, err := keep(t.Config, current, 0o600)
if err != nil {
return notRetired("keeping what it holds now failed (" + err.Error() + ")")
}
record.Extra, record.ExtraDigest, newCopy = where, sum, where
}
if err := os.Remove(t.Config); err != nil && !errors.Is(err, os.ErrNotExist) {
say(proven + ", and the found configuration " + t.Config + " could not be removed (" + err.Error() + ")")
return out, false
return notRetired("removing it failed (" + err.Error() + ")")
}
if present(t.Config) {
say(proven + ", and the found configuration " + t.Config + " is still there after it was removed")
return out, false
return notRetired("it is still there after it was removed")
}
}
known.RecordRetired(store.Retired{ID: id, Path: t.Config, Kept: held.Kept, At: now})
known.RecordRetired(record)
known.Release(id)
facts.Kept = held.Kept
say(proven + "; the found configuration " + t.Config + " is retired — its original kept at " +
held.Kept + ", " + t.Unit + " left disabled, and the mesh never brings it back")
out = Outcome{ID: id, Type: string(declaration.TypeFile), Target: t.Config, Action: "removed",
Detail: "retired: the take of " + t.Interface + " is " + proven + "; original kept at " + held.Kept}
copied := ""
if newCopy != "" {
copied = "; what it held, which differed from the original, is kept at " + newCopy
}
out = Outcome{ID: id, Type: string(declaration.TypeFile), Target: t.Config, Action: "removed"}
switch {
case cameBack:
say("the found configuration came back and was retired again — its original still kept at " +
held.Kept + copied + "; rolling back to the found tunnel means unassigning the private network first")
out.Detail = "the found configuration came back and was retired again; original kept at " + held.Kept + copied
default:
say("the found configuration " + t.Config + " is retired — its original kept at " + held.Kept + copied +
", " + t.Unit + " left disabled, and the mesh never brings it back")
out.Detail = "retired: the take of " + t.Interface + " is " + proven + "; original kept at " + held.Kept + copied
}
if gone {
// Already gone — removed by something other than the mesh, or by an apply whose record was
// never saved. Nothing removed here; the hold ends all the same.
@@ -489,6 +553,16 @@ func retireFound(ctx context.Context, svc *declaration.Service, known *store.Sta
return out, true
}
// declaresFile is whether a declaration writes a file at a path.
func declaresFile(d *declaration.Declaration, path string) bool {
for _, r := range d.Resources {
if f, ok := r.(*declaration.File); ok && filepath.Clean(f.Path) == filepath.Clean(path) {
return true
}
}
return false
}
// takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since
// a machine has one private network.
func takesOver(d *declaration.Declaration) *declaration.Service {
internal/apply/takeover_test.go
+204
View File
@@ -66,6 +66,10 @@ func aHubInUse(t *testing.T) (dir, config, mesh, keyFile string, m *machine) {
"wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"},
}}
takeoverRecheck = 0
// The found configuration lives in this test's own wireguard directory (novox/hq ADR 0119).
was := wireguardDir
wireguardDir = dir
t.Cleanup(func() { wireguardDir = was })
return dir, config, mesh, keyFile, m
}
@@ -509,3 +513,203 @@ func TestAPlanSaysTheFoundConfigurationIsRetiredWhenTheTakeIsProven(t *testing.T
}
}
}
// keptCopies is every copy kept of the found configuration.
func keptCopies(t *testing.T, dir string) []string {
t.Helper()
kept, err := filepath.Glob(filepath.Join(dir, "kept", "*-wg0.conf"))
if err != nil {
t.Fatal(err)
}
return kept
}
func TestAConfigurationPutBackIsRetiredAgainOnItsFirstOriginalAndSettles(t *testing.T) {
dir, config, mesh, keyFile, m := aHubInUse(t)
m.handshakes = handshaken
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
_, state := applyAdopted(t, d, store.State{}, m, dir)
first, _ := state.RetiredAt(config)
// Put back by hand with the original, while no peer is through yet: held on the first
// original, and the account says what a rollback takes.
write(t, config, foundConf)
m.handshakes = ""
report, state := applyAdopted(t, d, state, m, dir)
if held, ok := state.HeldAt(takesOverID); !ok || held.Kept != first.Kept {
t.Fatalf("what came back is not held on the first original: %+v", held)
}
if report.Tunnel == nil || !strings.Contains(report.Tunnel.Note, "came back after it was retired") ||
!strings.Contains(report.Tunnel.Note, "unassigning the private network first") {
t.Errorf("the account does not say a rollback means unassigning the private network: %+v", report.Tunnel)
}
// Proven: retired again, nothing more kept, said once.
m.handshakes = handshaken
report, state = applyAdopted(t, d, state, m, dir)
again, _ := state.RetiredAt(config)
if _, err := os.Lstat(config); !os.IsNotExist(err) || again.Kept != first.Kept || again.Extra != "" {
t.Fatalf("put back as it was, it was not retired again on the first original: %+v", again)
}
if o := outcomeOf(report, takesOverID); o.Action != "removed" ||
!strings.HasPrefix(o.Detail, "the found configuration came back and was retired again") ||
strings.Contains(o.Detail, "differed") {
t.Errorf("the second retirement is not said as one: %+v", o)
}
if !strings.Contains(report.Tunnel.Note, "unassigning the private network first") {
t.Errorf("the account does not say what a rollback takes: %q", report.Tunnel.Note)
}
if n := len(keptCopies(t, dir)); n != 1 {
t.Errorf("%d copies kept of one content", n)
}
if report, _ := applyAdopted(t, d, state, m, dir); report.Changed() {
t.Errorf("a steady machine moved after the second retirement: %+v", report.Outcomes)
}
// Put back with something else: that is kept beside the first original, which stays the record's.
other := strings.Replace(foundConf, "PEER-B=", "PEER-Z=", 1)
write(t, config, other)
report, state = applyAdopted(t, d, state, m, dir)
third, _ := state.RetiredAt(config)
if third.Kept != first.Kept || third.Extra == "" || third.Extra == first.Kept {
t.Fatalf("other content was not kept apart from the first original: %+v", third)
}
if got, _ := os.ReadFile(third.Extra); string(got) != other {
t.Errorf("the extra copy does not hold what was put back: %q", got)
}
if o := outcomeOf(report, takesOverID); !strings.Contains(o.Detail, third.Extra) ||
!strings.Contains(report.Tunnel.Note, third.Extra) {
t.Errorf("where the extra copy is was not said: %+v / %q", o, report.Tunnel.Note)
}
// And the same other content again: nothing more kept, the record as it was.
write(t, config, other)
report, state = applyAdopted(t, d, state, m, dir)
fourth, _ := state.RetiredAt(config)
if fourth.Kept != first.Kept || fourth.Extra != third.Extra || len(keptCopies(t, dir)) != 2 {
t.Errorf("the same content put back again grew the copies: %+v, %v", fourth, keptCopies(t, dir))
}
if o := outcomeOf(report, takesOverID); strings.Contains(o.Detail, "differed") {
t.Errorf("a copy already kept was said as new: %+v", o)
}
if report, _ := applyAdopted(t, d, state, m, dir); report.Changed() {
t.Errorf("a steady machine moved: %+v", report.Outcomes)
}
}
func TestOnlyWgQuicksOwnConfigurationIsRetired(t *testing.T) {
// Not under the wireguard directory.
dir, config, mesh, keyFile, m := aHubInUse(t)
wireguardDir = filepath.Join(dir, "elsewhere")
m.handshakes = handshaken
report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)
if got, _ := os.ReadFile(config); string(got) != foundConf {
t.Fatal("a configuration outside wg-quick's directory was removed")
}
if _, held := state.HeldAt(takesOverID); !held || !strings.Contains(report.Tunnel.Note, "is not retired: only ") {
t.Errorf("the refusal is not said, or the hold ended: %+v", report.Tunnel)
}
// A path the mesh itself writes.
dir, config, mesh, keyFile, m = aHubInUse(t)
m.handshakes = handshaken
known := store.State{}
known.Record(store.Applied{ID: "bundle.wg0", Type: "file", Target: config, Origin: store.OriginCarried})
report, _ = applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), known, m, dir)
if got, _ := os.ReadFile(config); string(got) != foundConf {
t.Fatal("a path the mesh writes was retired")
}
if !strings.Contains(report.Tunnel.Note, "a path the mesh itself writes") {
t.Errorf("the refusal is not said: %+v", report.Tunnel)
}
}
func TestAFoundConfigurationThatIsALinkIsKeptAndLeftToAPerson(t *testing.T) {
dir, config, mesh, keyFile, m := aHubInUse(t)
target := filepath.Join(dir, "predecessor", "hub.conf")
if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil {
t.Fatal(err)
}
write(t, target, foundConf)
if err := os.Remove(config); err != nil {
t.Fatal(err)
}
if err := os.Symlink(target, config); err != nil {
t.Fatal(err)
}
m.handshakes = handshaken
report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)
if _, err := os.Lstat(config); err != nil {
t.Fatal("the link was removed, leaving the key-bearing file it points at")
}
if got, _ := os.ReadFile(target); string(got) != foundConf {
t.Fatal("the file the link points at was touched")
}
held, ok := state.HeldAt(takesOverID)
if !ok {
t.Fatal("the hold ended")
}
if kept, _ := os.ReadFile(held.Kept); string(kept) != foundConf {
t.Errorf("what was kept is not what the link points at: %q", kept)
}
if !strings.Contains(report.Tunnel.Note, "is a link to "+target) || !strings.Contains(report.Tunnel.Note, "by hand") {
t.Errorf("the account does not say the link must be retired by hand: %q", report.Tunnel.Note)
}
}
func TestARetirementWhoseRecordWasNeverSavedIsRecordedByTheNextApply(t *testing.T) {
dir, config, mesh, keyFile, m := aHubInUse(t)
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
_, state := applyAdopted(t, d, store.State{}, m, dir)
held, _ := state.HeldAt(takesOverID)
// An apply removed the file and stopped before its state was saved.
if err := os.Remove(config); err != nil {
t.Fatal(err)
}
m.handshakes = handshaken
report, state := applyAdopted(t, d, state, m, dir)
if r, ok := state.RetiredAt(config); !ok || r.Kept != held.Kept {
t.Fatalf("the retirement was not recorded: %+v", state.Retired)
}
if _, still := state.HeldAt(takesOverID); still {
t.Error("the hold did not end")
}
if o := outcomeOf(report, takesOverID); o.Action != "unchanged" || !strings.Contains(o.Detail, "already gone") {
t.Errorf("a file already gone is not said as such: %+v", o)
}
}
func TestARemovalThatFailsKeepsTheFileAndTheHold(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("root removes from a directory it may not write to")
}
dir, _, mesh, keyFile, m := aHubInUse(t)
wg := filepath.Join(dir, "wireguard")
if err := os.MkdirAll(wg, 0o700); err != nil {
t.Fatal(err)
}
config := filepath.Join(wg, "wg0.conf")
write(t, config, foundConf)
wireguardDir = wg
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
_, state := applyAdopted(t, d, store.State{}, m, dir)
if err := os.Chmod(wg, 0o500); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(wg, 0o700) })
m.handshakes = handshaken
report, state := applyAdopted(t, d, state, m, dir)
if got, _ := os.ReadFile(config); string(got) != foundConf {
t.Fatal("the found configuration is gone although it could not be removed")
}
if _, held := state.HeldAt(takesOverID); !held {
t.Error("the hold ended although nothing was retired")
}
if _, retired := state.RetiredAt(config); retired {
t.Error("recorded as retired")
}
if !strings.Contains(report.Tunnel.Note, "removing it failed") || !strings.Contains(report.Tunnel.Note, "permission denied") {
t.Errorf("the failed removal is not said: %q", report.Tunnel.Note)
}
}