novox/mesh-host
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into feat/nats-genesis

Browse Source
This commit is contained in:
jschoubben
2026-09-27 18:38:45 +02:00
parent 6a3435629e 54f6eb661a
commit 48e2ff2de5
26 changed files with 3620 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/main.go
+1 -1
View File
@@ -915,7 +915,7 @@ func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, l
// carried is a found tunnel as it is presented to the mesh: everything but its private key.
func carried(t tunnel.Found) *link.Tunnel {
out := &link.Tunnel{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port,
Address: t.Address, Range: t.Range, PublicKey: t.PublicKey}
Address: t.Address, Range: t.Range, MTU: t.MTU, PublicKey: t.PublicKey}
for _, p := range t.Peers {
out.Peers = append(out.Peers, link.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address})
}
internal/apply/apply.go
+358 -25
View File
@@ -52,6 +52,12 @@ type Outcome struct {
wrote string
// into is what a file written into held before the mesh's keys (novox/hq ADR 0102).
into *store.Into
// kept is where this apply kept the original of a file it wrote over (novox/hq ADR 0100).
kept string
// stateless is a service whose unit's lifecycle is the machine's (novox/hq ADR 0117).
stateless bool
// found is, for a service, its unit as the host first found it (novox/hq ADR 0118).
found *store.FoundUnit
// reads is, for a container, the digest of each file it was created reading, by path — so
// the next apply can say which one changed (novox/hq 04-ISSUES/103).
reads map[string]string
@@ -164,6 +170,14 @@ func ApplyKeeping(
// as the service that took it over is (novox/hq ADR 0105).
declared[takeOverID(svc)] = true
}
// What an unfinished apply found a unit as is kept only while its service is declared: one
// declared again later is read afresh, as any resource with no record is (novox/hq ADR 0118).
known.DropFoundUndeclared(declared, origin)
// Which units the mesh made, read before any removal can take a unit file's record away — so
// whichever order a module declared a unit and its file in, the unit is known for the mesh's
// when its service goes (novox/hq ADR 0118).
made := meshMadeUnits(known)
// Which firewall is found here, before anything else, since an unsupported one refuses the
// whole declaration (novox/hq ADR 0100). Nothing for a converged node.
@@ -182,7 +196,7 @@ func ApplyKeeping(
if declaration.Type(orphan.Type) == declaration.TypeOpening {
action, detail, err = removeOpening(ctx, orphan, run, known.Firewall)
} else {
action, detail, err = remove(ctx, sys, orphan, run)
action, detail, err = remove(ctx, sys, orphan, run, made)
}
if err != nil {
return &Error{Resource: orphan.ID, Err: err, Done: report}
@@ -350,6 +364,7 @@ func ApplyKeeping(
// flushed. A failure here fails the service too — the mesh's interface is not started on a
// port the found one still holds.
stoppedFound := false
tookAt := -1
if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil {
var outcome Outcome
var facts TakenTunnel
@@ -374,11 +389,22 @@ func ApplyKeeping(
log(fmt.Sprintf(" failed %s (%s): %v", svc.Identity(), svc.Unit, err))
continue
}
tookAt = len(report.Outcomes)
report.Outcomes = append(report.Outcomes, outcome)
if outcome.Action == "held" {
log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail))
}
}
was, _ := known.Find(resource.Identity())
// What an earlier apply of this service found its unit as and could not yet record is
// newer than anything the record says — the record's own finding with what was read
// since — so it is what this apply goes on from (novox/hq ADR 0118).
previous := was
if p, ok := known.FoundFirst[resource.Identity()]; ok {
f := p.FoundUnit
previous.Found = &f
}
var outcome Outcome
var err error
if o, isOpening := resource.(*declaration.Opening); isOpening {
@@ -391,9 +417,15 @@ func ApplyKeeping(
!known.Recorded(string(declaration.TypeFile), f.Path) {
keepFound = keep
}
outcome, err = applyOne(ctx, sys, resource, run, changed, in, was, unseal, keepFound)
outcome, err = applyOne(ctx, sys, resource, run, changed, in, previous, unseal, keepFound)
}
if err != nil {
// **What was found is kept whatever the apply then did** (novox/hq ADR 0118). The
// failure may have come after the mesh enabled or started the unit, and the next apply
// would otherwise read that as the machine's own.
if outcome.found != nil {
known.KeepFound(resource.Identity(), origin, *outcome.found)
}
failed := &Error{Resource: resource.Identity(), Err: err, Done: report}
failures = append(failures, failed)
log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), outcome.Target, err))
@@ -441,6 +473,16 @@ func ApplyKeeping(
continue
}
// Where the original of what this file replaced was kept, carried for as long as the
// resource is recorded: kept by this apply, by a hold its module's cutover ends, or before.
held, wasHeld := known.HeldAt(resource.Identity())
kept := outcome.kept
if kept == "" && wasHeld {
kept = held.Kept
}
if kept == "" {
kept = was.Kept
}
// Only now. The record follows the fact, never leads it.
known.Record(store.Applied{
Origin: origin,
@@ -448,18 +490,40 @@ func ApplyKeeping(
Target: outcome.Target, AppliedAt: time.Now().UTC(),
Wrote: outcome.wrote,
Into: outcome.into,
Kept: kept,
Reads: outcome.reads,
Stateless: outcome.stateless,
Found: outcome.found,
Holds: holds(resource),
})
// Its module has been taken, and what was held for it is now the mesh's.
if held, wasHeld := known.HeldAt(resource.Identity()); wasHeld {
if outcome.found != nil {
known.DropFound(resource.Identity())
}
// Its module has been taken, and what was held for it is now the mesh's. A file written
// into, or a service whose lifecycle is the machine's, replaced nothing that was found, so
// its outcome says what the apply did, not that a cutover happened; a hold from when it was
// declared otherwise goes all the same — here, after the apply worked, so a failed one
// keeps the hold and where its original is.
if wasHeld {
known.Release(held.ID)
if !replacesNothing(resource) {
outcome.Detail = takenDetail(held)
}
}
if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil {
// The found interface is down and the mesh's is up in its place: the tunnel changed
// hands (novox/hq ADR 0105). Read from the machine, not assumed.
report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run)
// And once a peer has handshaken with it, the take is proven and the found
// configuration is retired — here, after the mesh's service applied, so in the apply
// of the take itself only if a peer is already through; otherwise a later apply
// retires it (novox/hq ADR 0119). What it did replaces what the take said of the file.
if o, did := retireFound(ctx, svc, d, &known, run, keep, report.Tunnel, time.Now().UTC()); did {
if tookAt >= 0 {
report.Outcomes[tookAt] = o
}
log(fmt.Sprintf(" %s %s (%s): %s", o.Action, o.ID, o.Target, o.Detail))
}
}
report.Outcomes = append(report.Outcomes, outcome)
if outcome.Action != "unchanged" {
@@ -523,7 +587,7 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru
case *declaration.File:
return applyFile(res, previous, unseal, keepFound)
case *declaration.Service:
return applyService(ctx, sys, res, run, changed)
return applyService(ctx, sys, res, run, changed, previous)
case *declaration.Package:
return applyPackage(ctx, sys, res, run)
case *declaration.Container:
@@ -669,6 +733,9 @@ func applyAccess(r *declaration.Access) (Outcome, error) {
// keepFound, when not nil, is where the original of a file this host has no record of is kept
// before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome.
func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) {
if r.Into == declaration.IntoBlock {
return applyBlock(r, previous)
}
if r.Into != "" {
return applyInto(r, previous)
}
@@ -837,6 +904,7 @@ func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepF
default:
out.Action = "unchanged"
}
out.kept = kept
if kept != "" {
if out.Detail != "" {
out.Detail += "; "
@@ -896,7 +964,10 @@ type unitReloader interface {
}
func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
changed map[string]bool) (Outcome, error) {
changed map[string]bool, previous store.Applied) (Outcome, error) {
if r.Stateless() {
return reflectOnly(ctx, sys, r, run, changed)
}
out := begin(r)
var changes []string
@@ -911,6 +982,21 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
}
}
// **What the unit was before the mesh touched it**, read once and carried in the record from
// then on (novox/hq ADR 0118). Undeclared, the unit is given back to exactly this: it is the one
// fact that separates the container runtime, running before the mesh arrived and to be left
// running, from the mesh's packet filter, stopped until the mesh started it and to be stopped
// again. Read after the reload above, never before it: a unit whose file this same apply wrote
// is not a unit the service manager knows until then, and reading it first would find nothing.
found, gaveBack, err := foundAs(ctx, sys, r, run, previous)
if err != nil {
return out, err
}
out.found = found
if gaveBack != "" {
changes = append(changes, gaveBack)
}
// Boot first. A unit asked to be running and enabled should survive this apply failing
// half way in the more useful direction: enabled-and-stopped comes back at the next boot,
// where running-and-disabled does not.
@@ -919,6 +1005,15 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
if err != nil {
return out, err
}
// Whether it started at boot is found the first time the mesh is about to change that —
// which is not always the first apply: a declaration that said nothing about boot never
// touched it, so what is there when one first does is still the machine's (novox/hq ADR
// 0118). Kept before the change, so a failure after it still has it.
if out.found != nil && out.found.Boot == "" {
f := *out.found
f.Boot = bootBefore
out.found = &f
}
if bootBefore != r.Boot {
if err := sys.SetServiceBoot(ctx, run, r.Unit, r.Boot); err != nil {
return out, fmt.Errorf("setting %s to %s at boot: %w", r.Unit, r.Boot, err)
@@ -1007,6 +1102,73 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
return out, nil
}
// reflectOnly is a service whose unit's lifecycle is the machine's (novox/hq ADR 0117): nothing is
// started, stopped, enabled or disabled, and a changed trigger is acted on only where the unit is
// already running. An inactive unit is left so — started, it would be a second network manager on
// a machine that uses another — and it reads the change when whatever starts it does.
func reflectOnly(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
changed map[string]bool) (Outcome, error) {
out := begin(r)
out.stateless = true
restart := reflected(r, changed)
reload := restartedBy(r.ReloadOn, changed)
if len(restart) == 0 && len(reload) == 0 {
out.Action = "unchanged"
out.Detail = "its lifecycle is the machine's; nothing it reflects changed"
return out, nil
}
state, err := sys.ServiceState(ctx, run, r.Unit)
if err != nil {
return out, err
}
if state != "running" {
out.Action = "unchanged"
out.Detail = "not running; the change applies at its next start"
return out, nil
}
if len(restart) > 0 {
// The same as a stated service: the unit's own file may be what changed, and the manager
// reads that again only when told to.
if u, ok := sys.(unitReloader); ok {
if err := u.ReloadUnits(ctx, run); err != nil {
return out, fmt.Errorf("reloading the service manager's units for %s: %w", r.Unit, err)
}
}
if err := sys.SetServiceState(ctx, run, r.Unit, "stopped"); err != nil {
return out, fmt.Errorf("restarting %s: stopping it: %w", r.Unit, err)
}
if err := sys.SetServiceState(ctx, run, r.Unit, "running"); err != nil {
return out, fmt.Errorf("restarting %s: starting it again: %w", r.Unit, err)
}
} else {
reloader, ok := sys.(serviceReloader)
if !ok {
return out, fmt.Errorf("%s must be reloaded for %s and this machine's service manager "+
"cannot reload a unit", r.Unit, strings.Join(reload, ", "))
}
if err := reloader.ReloadService(ctx, run, r.Unit); err != nil {
return out, fmt.Errorf("reloading %s: %w", r.Unit, err)
}
}
// Read back: it was running, and a restart or reload that left it otherwise is a failure —
// the machine's network manager down is not a change to report and move past.
after, err := sys.ServiceState(ctx, run, r.Unit)
if err != nil {
return out, err
}
out.Action = "updated"
if len(restart) > 0 {
out.Detail = "restarted for " + strings.Join(restart, ", ")
} else {
out.Detail = "reloaded for " + strings.Join(reload, ", ")
}
if after != "running" {
return out, fmt.Errorf("%s was %s to pick up a change and is %s", r.Unit,
strings.Fields(out.Detail)[0], after)
}
return out, nil
}
// remove undoes one resource the host applied and the declaration no longer names, and reports
// what it actually did.
//
@@ -1016,7 +1178,10 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
// It returns the action rather than assuming "removed", because for half the vocabulary the
// honest word is "forgotten". A host that reported a package removed when it left the package
// installed would be describing an effect it declined to have.
func remove(ctx context.Context, sys system.System, a store.Applied, run Runner) (string, string, error) {
//
// made is the units whose unit file this host wrote where there was none (meshMadeUnits).
func remove(ctx context.Context, sys system.System, a store.Applied, run Runner,
made map[string]bool) (string, string, error) {
switch declaration.Type(a.Type) {
case declaration.TypeDirectory:
// **A directory with anything left in it is kept, and that is the rule that protects
@@ -1049,6 +1214,9 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner)
return "removed", "no longer declared, and empty", nil
case declaration.TypeFile:
if a.Into != nil && a.Into.Format == declaration.IntoBlock {
return removeBlock(a)
}
if a.Into != nil {
return removeInto(a)
}
@@ -1061,24 +1229,12 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner)
return "removed", "no longer declared", nil
case declaration.TypeService:
// A unit that is no longer declared is stopped, not deleted. The host did not install
// it and does not own the unit file — only the state it put the unit into.
//
// A unit that no longer EXISTS is already in the state removal is trying to reach, and
// saying so matters: stopping it fails, and a failure here fails the whole apply. A
// host holding a record of an uninstalled unit would then be unable to apply anything,
// ever, with no way out but editing its state by hand. Removal is idempotent for the
// same reason `os.RemoveAll` is.
if _, err := sys.ServiceState(ctx, run, a.Target); err != nil {
if strings.Contains(err.Error(), "does not exist on this machine") {
return "forgotten", "the unit no longer exists", nil
}
return "", "", err
}
if err := sys.SetServiceState(ctx, run, a.Target, "stopped"); err != nil {
return "", "", fmt.Errorf("stopping %s: %w", a.Target, err)
}
return "removed", "stopped; the unit file is not the host's to delete", nil
return removeService(ctx, sys, a, run, made[a.Target])
case declaration.TypeProcess:
// The other side of the same line: a process's unit is the host's own — it wrote the unit
// file and unpacked the bundle — so it goes with its declaration (novox/hq ADR 0118).
return removeProcess(ctx, a, run)
case declaration.TypeContainer:
// The host CREATED this one, so the host removes it. That is the line: it removes what
@@ -1914,3 +2070,180 @@ func declaredDigest(r declaration.Resource) string {
}
return fmt.Sprintf("%x", sha256.Sum256([]byte(material)))
}
// removeService gives a unit back the state the host first found it in, and nothing more.
//
// **Removes what it made, gives back what it changed, leaves what was the machine's**
// (novox/hq ADR 0118). A service resource never installs a unit; it puts one that already existed
// into a state. So undeclaring it cannot mean stopping it — that is how unassigning the private
// network stopped the container runtime and every container with it, how unassigning sshd would
// have stopped ssh, and how an uplink module would have taken a machine off its only link
// (novox/hq issue 130). It means undoing what the mesh did to it: a unit found running is left
// running; a unit the mesh started is stopped again, and disabled again if the mesh enabled it.
//
// **A unit whose file the mesh wrote is the mesh's, whatever was found** — made is that. The
// adoption guard and the converge filter are units of exactly this kind: their unit files are the
// mesh's own `file` resources, written where there was none, and records written before the host
// kept what it found say nothing about them. Forgetting one would leave its table loaded — the
// guard beside a converged node's own filter, or the filter beside the predecessor's firewall
// re-enabled — and its unit file then deleted from under a running unit. So it is stopped and
// disabled at boot, as a process's unit is, before its file goes: orphans are removed newest
// first, and a unit's file is declared before the service that starts it.
//
// **Never started on the way out.** A unit the mesh stopped is not started again when its
// declaration goes: starting something is a decision, and the operator makes it. The report says
// what was actually done — a unit already as it was found is forgotten, not "restored".
func removeService(ctx context.Context, sys system.System, a store.Applied, run Runner,
made bool) (string, string, error) {
if a.Stateless {
// Declared with no state (novox/hq ADR 0117): its lifecycle was never the mesh's, and the
// declaration that said so is the operator's word to hold to, a file of the mesh's or not.
return "forgotten", "its state was never the mesh's", nil
}
if !made && a.Found == nil {
// Recorded before the host kept what it found. Not knowing, it leaves the unit as it is:
// a unit left running can be stopped by the operator, and one stopped by mistake may be
// the link the operator would use to do it.
return "forgotten", "recorded before the host kept what it found; left as it is", nil
}
stop := made || a.Found.State == "stopped"
disable := made || a.Found.Boot == "disabled"
if !stop && !disable {
// Found running, and not disabled by anyone but the mesh: nothing to give back. What the
// mesh did since is said, so a unit left stopped is not reported as the machine's doing —
// read as well as the machine answers, since nothing here acts on the answer.
state, err := sys.ServiceState(ctx, run, a.Target)
if err != nil && strings.Contains(err.Error(), "does not exist on this machine") {
return "forgotten", "the unit no longer exists", nil
}
var did []string
if err == nil && state == "stopped" {
did = append(did, "stopped it")
}
if a.Found.Boot == "enabled" {
if boot, err := sys.ServiceBoot(ctx, run, a.Target); err == nil && boot == "disabled" {
did = append(did, "disabled it at boot")
}
}
if len(did) > 0 {
return "forgotten", "left as it is; the mesh " + strings.Join(did, " and ") +
" and does not start anything on the way out", nil
}
return "forgotten", "it was running before the mesh; left as it is", nil
}
// Something may be given back, so the unit must still be there to give it to. One since
// uninstalled is already as far from the mesh as it can be, and saying so keeps a record of it
// from failing every apply.
state, err := sys.ServiceState(ctx, run, a.Target)
if err != nil {
if strings.Contains(err.Error(), "does not exist on this machine") {
return "forgotten", "the unit no longer exists", nil
}
return "", "", err
}
var did, already []string
if stop {
if state != "stopped" {
if err := sys.SetServiceState(ctx, run, a.Target, "stopped"); err != nil {
return "", "", fmt.Errorf("stopping %s, which the mesh started: %w", a.Target, err)
}
did = append(did, "stopped")
} else {
already = append(already, "stopped")
}
}
if disable {
// Disabled unless it reads as disabled: a unit the service manager cannot say a boot state
// for — one with no install section — takes a disable as a no-op, and asking is how the
// host stays sure the mesh's enable did not outlive it.
if boot, err := sys.ServiceBoot(ctx, run, a.Target); err == nil && boot == "disabled" {
already = append(already, "disabled at boot")
} else {
if err := sys.SetServiceBoot(ctx, run, a.Target, "disabled"); err != nil {
return "", "", fmt.Errorf("disabling %s, which the mesh enabled: %w", a.Target, err)
}
did = append(did, "disabled at boot")
}
}
if made {
if len(did) == 0 {
return "forgotten", "already stopped and disabled at boot; the mesh wrote its unit file", nil
}
return "removed", strings.Join(did, ", ") + "; the mesh wrote its unit file, so the unit is the mesh's own", nil
}
if len(did) == 0 {
return "forgotten", "already as the host found it (" + strings.Join(already, ", ") + ")", nil
}
detail := strings.Join(did, ", ") + ", as the host found it"
if len(already) > 0 {
detail += "; already " + strings.Join(already, ", ")
}
return "restored", detail, nil
}
// foundAs is what a service's unit was before the mesh touched it, as far as this host can know:
// what an earlier apply recorded, or — the first time the mesh is about to act on the unit — what
// is there now (novox/hq ADR 0118). Nil when it cannot be known: a record written before the host
// kept this has had the mesh acting on the unit since, and a reading now would be the mesh's doing.
//
// Read now as well, besides on a first apply, where the mesh has never acted on this unit's state
// although a record exists: the record is of a service declared with no state (novox/hq ADR 0117),
// which the mesh never starts or stops, or of another unit altogether. What was found about one
// unit says nothing about another, so a service moved to a new unit gives the old one back, just as
// if it had been undeclared, and is read afresh for the new one; gave says what that gave back.
func foundAs(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
previous store.Applied) (found *store.FoundUnit, gave string, err error) {
if f := previous.Found; f != nil {
unit := f.Unit
if unit == "" {
unit = previous.Target
}
if unit == "" || unit == r.Unit {
return f, "", nil
}
// Given back as if undeclared, never as the mesh's own: whether the mesh wrote the old
// unit's file is known to the removal of orphans, and that file's own record goes with it.
action, detail, err := removeService(ctx, sys,
store.Applied{Type: string(declaration.TypeService), Target: unit, Found: f}, run, false)
if err != nil {
return nil, "", fmt.Errorf("giving %s back as the host found it, now that %s is declared instead: %w",
unit, r.Unit, err)
}
if action == "restored" {
gave = unit + " " + detail
}
} else if previous.ID != "" && !previous.Stateless && previous.Target == r.Unit {
return nil, "", nil
}
state, err := sys.ServiceState(ctx, run, r.Unit)
if err != nil {
return nil, gave, err
}
return &store.FoundUnit{Unit: r.Unit, State: state}, gave, nil
}
// meshMadeUnits is every unit whose unit file this host wrote whole where there was none: a `file`
// record with no kept original and not written into, at a unit's own path under a directory the
// service manager loads administrators' units from — or the one the host writes a process's unit
// in. Such a unit is the mesh's, whatever was found (novox/hq ADR 0118); see removeService.
//
// A drop-in is not the unit's own file, and a file the host wrote over is the machine's unit with
// the mesh's text in it: its original is kept, and put back when the file's record goes.
func meshMadeUnits(known store.State) map[string]bool {
made := map[string]bool{}
dirs := map[string]bool{"/etc/systemd/system": true, "/run/systemd/system": true,
filepath.Clean(unitDir): true}
for _, r := range known.Resources {
if r.Type != string(declaration.TypeFile) || r.Kept != "" || r.Into != nil {
continue
}
path := filepath.Clean(r.Target)
if dirs[filepath.Dir(path)] {
made[filepath.Base(path)] = true
}
}
return made
}
internal/apply/apply_test.go
+88 -10
View File
@@ -348,9 +348,27 @@ func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) {
}
}
func TestADroppedServiceIsStoppedNotDeleted(t *testing.T) {
// The host did not install the unit and does not own the unit file — only the state it put
// the unit into.
func TestADroppedServiceIsGivenBackTheStateItWasFoundIn(t *testing.T) {
// novox/hq ADR 0118: undeclaring removes what the mesh made, gives back what it changed, and
// leaves what was the machine's. A service resource never installs a unit — so undeclaring it
// undoes what the mesh did to the unit, and nothing more. Stopping every undeclared unit is
// how unassigning the private network stopped the container runtime (novox/hq issue 130).
cases := []struct {
name string
found *store.FoundUnit
action string
stop bool
disable bool
}{
{"recorded before the host kept what it found", nil, "forgotten", false, false},
{"running before the mesh", &store.FoundUnit{State: "running"}, "forgotten", false, false},
{"running and enabled before the mesh", &store.FoundUnit{State: "running", Boot: "enabled"}, "forgotten", false, false},
{"started by the mesh", &store.FoundUnit{State: "stopped"}, "restored", true, false},
{"started and enabled by the mesh", &store.FoundUnit{State: "stopped", Boot: "disabled"}, "restored", true, true},
{"enabled by the mesh, running before it", &store.FoundUnit{State: "running", Boot: "disabled"}, "restored", false, true},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
var commands []string
run := func(ctx context.Context, name string, args ...string) (string, error) {
commands = append(commands, strings.Join(args, " "))
@@ -360,21 +378,81 @@ func TestADroppedServiceIsStoppedNotDeleted(t *testing.T) {
return "", nil
}
state := store.State{Resources: []store.Applied{
{ID: "s", Type: "service", Target: "gone.service"},
{ID: "s", Type: "service", Target: "unit.service", Found: c.found},
}}
d := parse(t, `{"declaration":1,"resources":[
{"id":"other","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
]}`)
if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil, nil); err != nil {
report, after, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil, nil)
if err != nil {
t.Fatal(err)
}
joined := strings.Join(commands, "; ")
if !strings.Contains(joined, "stop gone.service") {
t.Errorf("the dropped service was not stopped: %s", joined)
if stopped := strings.Contains(joined, "stop unit.service"); stopped != c.stop {
t.Errorf("stopped %v, want %v: %s", stopped, c.stop, joined)
}
if strings.Contains(joined, "disable") || strings.Contains(joined, "mask") {
t.Errorf("the host did more than stop a unit it does not own: %s", joined)
if disabled := strings.Contains(joined, "disable unit.service"); disabled != c.disable {
t.Errorf("disabled %v, want %v: %s", disabled, c.disable, joined)
}
if strings.Contains(joined, "start unit.service") || strings.Contains(joined, "mask") {
t.Errorf("the host started or masked a unit on its way out: %s", joined)
}
if o := outcomeOf(report, "s"); o.Action != c.action {
t.Errorf("outcome %+v, want %s", o, c.action)
}
if _, still := after.Find("s"); still {
t.Error("the host still believes it owns the undeclared service")
}
})
}
}
func TestAServiceRecordsTheStateItWasFoundInOnceAndCarriesIt(t *testing.T) {
// Read the first time the host applies the unit — before it starts or enables anything —
// and never again: by the next apply, the unit's state is the mesh's doing.
active := "inactive"
enabled := "disabled"
run := func(ctx context.Context, name string, args ...string) (string, error) {
switch args[0] {
case "show":
return "LoadState=loaded\nActiveState=" + active + "\n", nil
case "is-enabled":
return enabled, nil
case "start":
active = "active"
case "enable":
enabled = "enabled"
}
return "", nil
}
d := parse(t, `{"declaration":1,"resources":[
{"id":"s","type":"service","unit":"filter.service","state":"running","boot":"enabled"}
]}`)
_, first, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
if err != nil {
t.Fatal(err)
}
rec, _ := first.Find("s")
if rec.Found == nil || rec.Found.State != "stopped" || rec.Found.Boot != "disabled" {
t.Fatalf("first apply recorded %+v, want stopped and disabled", rec.Found)
}
_, second, err := Apply(context.Background(), archHost(t), d, first, store.OriginCarried, run, nil, nil)
if err != nil {
t.Fatal(err)
}
rec, _ = second.Find("s")
if rec.Found == nil || rec.Found.State != "stopped" {
t.Errorf("a later apply replaced what was found with what the mesh made: %+v", rec.Found)
}
// A record from before the host kept what it found is not given one later.
old := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "filter.service"}}}
_, third, err := Apply(context.Background(), archHost(t), d, old, store.OriginCarried, run, nil, nil)
if err != nil {
t.Fatal(err)
}
if rec, _ = third.Find("s"); rec.Found != nil {
t.Errorf("a record that predates the field was given one after the mesh had acted: %+v", rec.Found)
}
}
internal/apply/block.go
+433
View File
@@ -0,0 +1,433 @@
package apply
import (
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// A file written into a marked block, never over (novox/hq issue 128, ADR 0102).
//
// **The file is the machine's; the mesh owns lines in it.** The machine's hosts file is the case
// that needed it. The mesh wrote it whole — its own header, localhost, the machine's name and every
// name in the mesh — and on a workstation that file is shared: the distribution's lines, a local
// development tool's own marked blocks rewritten whenever its projects change, the operator's
// hand-added names. Written whole, all of those went at the next change to the mesh's names, with
// no failure anywhere: the tool believed it had written its block, and the mesh believed it owned
// the file. It is ADR 0102's failure exactly, in a file ADR 0102's JSON verb cannot speak.
//
// So the host finds the lines between `# BEGIN mesh <id>` and `# END mesh <id>`, rewrites those and
// nothing else, and records what they held before. Every line outside the markers is kept byte for
// byte — including another tool's `# BEGIN …` blocks, which are that tool's. Undeclared, the region
// is given back what it held, or taken out with its markers when it held nothing, and a file the
// mesh created goes only if nothing but whitespace is left.
// applyBlock writes a file's declared lines into its region of the file already at its path.
//
// **A link stays a link.** Where the path is a symbolic link — a hosts file some distributions keep
// elsewhere and link into /etc — the file read, written and renamed over is the one it points to,
// so the link and whatever manages it are left as they were. A file written whole, or into JSON,
// still replaces a link with a file; that is unchanged here.
func applyBlock(r *declaration.File, previous store.Applied) (Outcome, error) {
out := begin(r)
opening, closing := declaration.BlockMarkers(r.ID)
want := blockBody(r.Content)
real, err := realPath(r.Path)
if err != nil {
return out, err
}
raw, err := os.ReadFile(real)
existed := err == nil
if err != nil && !errors.Is(err, os.ErrNotExist) {
return out, err
}
// What the file is, taken once with what it holds: its mode and owner are the machine's and
// go back onto what is written. A file read and then not there to stat is a failure, never a
// file with no owner.
var info os.FileInfo
if existed {
if info, err = os.Stat(real); err != nil {
return out, fmt.Errorf("read %s and cannot see it: %w", r.Path, err)
}
}
existing := string(raw)
rec := store.Into{Format: declaration.IntoBlock}
var note string
rebuilt := false
// **A file the mesh once wrote whole** (novox/hq issue 128). The resource keeps its id when its
// module moves from writing the file whole to writing into it, and the file on the machine is
// then the mesh's own old write — its header, its loopback lines, its names. Adding the region
// after that would leave the old names above the new ones, and a resolver takes the first
// line that answers: the region would be shadowed by what it replaced. So the file is rebuilt:
// the original the mesh kept before its first write, with the region in it; or, where the mesh
// made the file itself, the loopback lines every machine needs, kept as the machine's, with the
// region beside them. Changed since the mesh wrote it, the file is somebody's again and is
// written into as it stands, and the outcome says so.
if existed && previous.Into == nil && previous.Wrote != "" {
if digestOf(existing) == previous.Wrote {
if previous.Kept != "" {
original, err := os.ReadFile(previous.Kept)
if err != nil {
return out, fmt.Errorf("%s was written whole by the mesh over an original kept at %s, "+
"which cannot be read to give it back: %w; it was left as it is", r.Path, previous.Kept, err)
}
existing = string(original)
note = "the mesh's old whole file replaced by the original kept at " + previous.Kept + ", with the region in it"
} else {
existing = loopbackOf(existing)
rec.Created = true
note = "the mesh's old whole file replaced by its loopback lines and the region"
}
// Not what was read: the whole of it was the mesh's, and the file is written afresh.
rebuilt = true
} else {
note = "a file the mesh once wrote whole, changed since; its old lines were kept"
}
}
lines := linesOf(existing)
at, found, err := regionIn(lines, opening, closing)
if err != nil {
// Refused, never guessed at: markers the host cannot pair are markers it cannot write
// between without risking lines that are not the mesh's.
return out, fmt.Errorf("%s: %w; it was left as it is", r.Path, err)
}
// A record of a block is carried; anything else — no record, a file once written whole, one
// once written into as JSON — is a file the host is seeing for the first time as a block.
recorded := previous.Into != nil && previous.Into.Format == declaration.IntoBlock
if recorded && existed {
rec.Created = previous.Into.Created
rec.Region = previous.Into.Region
rec.Separated = previous.Into.Separated
rec.At = previous.Into.At
rec.Ended = previous.Into.Ended
} else if !rebuilt {
// A file gone since the last apply is made again, and made by the mesh: what it held
// before went with it, so there is nothing to give back but the file's absence.
rec.Created = !existed
if found {
// **What the host may have written itself is not the machine's** — the same reasoning
// as a key in a JSON file (novox/hq ADR 0102). With no record, a region already holding
// exactly the declared lines cannot be told from one this host wrote a moment ago and
// died before saving; remembered as the machine's, it would be put back on undeclare
// for ever. So it is the mesh's, and undeclaring takes it out.
if held := at.body(lines); held != want {
rec.Region = &held
}
}
}
// Drift: the machine no longer holds, between the mesh's markers, what this host last put
// there. Judged only against a record of a block: a digest of a whole file says nothing about
// a region of it.
drifted := recorded && previous.Wrote != "" && existed &&
(!found || digestOf(at.body(lines)) != previous.Wrote)
var next string
switch {
case !existed:
next = regionOf(opening, closing, want)
case found:
// Where it is, whatever At says: the region is never moved, because moving it moves the
// machine's lines around it.
next = strings.Join(lines[:at.begin+1], "") + want + strings.Join(lines[at.end:], "")
case r.At == declaration.AtStart:
// Above everything, and one blank line between the region and the machine's first line
// unless there is one already — a line in some files means what the lines above it say.
rec.At, rec.Separated, rec.Ended = declaration.AtStart, false, false
next = regionOf(opening, closing, want)
if existing != "" && !strings.HasPrefix(existing, "\n") {
next += "\n"
rec.Separated = true
}
next += existing
default:
// At the end, apart from whatever is there: the file's last line is ended if it was not,
// and one blank line separates the region from the machine's lines unless there is one.
rec.At, rec.Separated, rec.Ended = "", false, false
next = existing
if next != "" && !strings.HasSuffix(next, "\n") {
next += "\n"
rec.Ended = true
}
if next != "" && next != "\n" && !strings.HasSuffix(next, "\n\n") {
next += "\n"
rec.Separated = true
}
next += regionOf(opening, closing, want)
}
// What was not the mesh's is what it was. By construction — and checked, because a slip in
// splicing lines is exactly the fault this mode exists to prevent, and it must never be written.
if found {
after := linesOf(next)
if where, ok, err := regionIn(after, opening, closing); err != nil || !ok || outside(after, where) != outside(lines, at) {
return out, fmt.Errorf("%s: writing the region would change lines outside it; it was left as it is", r.Path)
}
}
same := existed && next == string(raw)
if !same {
mode := os.FileMode(0o644)
if info != nil {
mode = info.Mode().Perm() // the machine's file keeps the machine's mode
} else if mode, err = modeOf(r.Mode, mode); err != nil {
return out, err
}
if err := os.MkdirAll(filepath.Dir(real), 0o755); err != nil {
return out, err
}
if err := writeAtomically(real, []byte(next), mode); err != nil {
return out, err
}
if info != nil {
// The write is a new file renamed over the old, so it belongs to whoever wrote it. The
// machine's file keeps the machine's owner, as it keeps its mode.
if err := keepOwner(real, info); err != nil {
return out, err
}
} else if err := own(real, r.Owner); err != nil {
return out, err
}
}
// Read back: the region holds what was declared. Only the region — another tool writing its
// own lines in the moment after the rename is not a failed write. What remains is the moment
// between reading the file and renaming over it: a line another tool writes there is lost, and
// found again at its next write. Nothing short of a lock every writer honours closes that, and
// the other writers of a hosts file honour none.
written, err := os.ReadFile(real)
if err != nil {
return out, fmt.Errorf("wrote into %s and cannot read it back: %w", r.Path, err)
}
back := linesOf(string(written))
if where, ok, err := regionIn(back, opening, closing); err != nil || !ok || where.body(back) != want {
return out, fmt.Errorf("%s does not hold the mesh's region after writing into it", r.Path)
}
out.into = &rec
out.wrote = digestOf(want)
switch {
case note != "" && !same:
out.Action = "updated"
out.Detail = note
case !existed:
out.Action = "created"
out.Detail = "written into; the file was not there"
case same:
out.Action = "unchanged"
case drifted:
out.Action = "corrected"
out.Detail = "the mesh's region had been changed on the machine; every line outside it was kept"
case !found:
out.Action = "updated"
where := "end"
if rec.At == declaration.AtStart {
where = "start"
}
out.Detail = "the mesh's region added at the " + where + "; every other line kept as it was"
default:
out.Action = "updated"
out.Detail = "the mesh's region rewritten; every line outside it kept as it was"
}
return out, nil
}
// removeBlock gives back what a file written into a block held before the mesh's region.
func removeBlock(a store.Applied) (string, string, error) {
real, err := realPath(a.Target)
if err != nil {
return "", "", err
}
raw, err := os.ReadFile(real)
if errors.Is(err, os.ErrNotExist) {
return "forgotten", "no longer there", nil
}
if err != nil {
return "", "", err
}
info, err := os.Stat(real)
if err != nil {
return "", "", fmt.Errorf("read %s and cannot see it: %w", a.Target, err)
}
opening, closing := declaration.BlockMarkers(a.ID)
lines := linesOf(string(raw))
at, found, err := regionIn(lines, opening, closing)
if err != nil {
return "kept", err.Error() + ", so nothing was taken out of it; remove the mesh's region by hand", nil
}
next, action, detail := string(raw), "forgotten", "the mesh's region was no longer in it"
switch {
case found && a.Into.Region != nil:
next = strings.Join(lines[:at.begin+1], "") + *a.Into.Region + strings.Join(lines[at.end:], "")
action, detail = "restored", "no longer declared; the region was given back what it held"
case found:
from, to := at.begin, at.end+1
// The blank line the host added beside the region, when a blank line still stands there.
// Whether it is the same one the host added cannot be known from the file; a blank line
// is the one line whose going changes nothing any program reads, so it is taken. A line
// that is not blank is never taken, whoever put it there.
if a.Into.Separated {
if a.Into.At == declaration.AtStart {
if to < len(lines) && lines[to] == "\n" {
to++
}
} else if from > 0 && lines[from-1] == "\n" {
from--
}
}
next = strings.Join(lines[:from], "") + strings.Join(lines[to:], "")
// And the line end the host gave the machine's last line, if that line is still last.
if a.Into.Ended && strings.Join(lines[to:], "") == "" {
next = strings.TrimSuffix(next, "\n")
}
action, detail = "restored", "no longer declared; the mesh's region was taken out and every other line kept"
}
if a.Into.Created && strings.TrimSpace(next) == "" && real == a.Target {
if err := os.Remove(real); err != nil {
return "", "", err
}
return "removed", "no longer declared; the mesh had created it and nothing else was in it", nil
}
if next == string(raw) {
return action, detail, nil
}
if err := writeAtomically(real, []byte(next), info.Mode().Perm()); err != nil {
return "", "", err
}
if err := keepOwner(real, info); err != nil {
return "", "", err
}
return action, detail, nil
}
// realPath is the file a path names, through any links; a path that is not there yet is itself.
// A link to nothing is refused: writing through it would replace the link with a file.
func realPath(path string) (string, error) {
real, err := filepath.EvalSymlinks(path)
if err == nil {
return real, nil
}
if _, lerr := os.Lstat(path); errors.Is(lerr, os.ErrNotExist) {
return path, nil
}
return "", fmt.Errorf("%s is a link the host cannot follow to a file: %w; it was left as it is", path, err)
}
// loopbackOf is the lines of a file that answer for the machine itself — localhost, its own name on
// 127.0.1.1, ::1 — and nothing else: what the mesh's old whole hosts file carried that the machine
// needs, without the mesh's header or its names.
func loopbackOf(text string) string {
var b strings.Builder
for _, line := range linesOf(text) {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
if ip := net.ParseIP(fields[0]); ip != nil && ip.IsLoopback() {
b.WriteString(strings.TrimSuffix(line, "\n") + "\n")
}
}
return b.String()
}
// outside is every line of a file but the mesh's region, markers included, as one string.
func outside(lines []string, at region) string {
end := at.end + 1
if end > len(lines) {
end = len(lines)
}
return strings.Join(lines[:at.begin], "") + "\x00" + strings.Join(lines[end:], "")
}
// blockBody is the declared lines as they stand in the region: ending in exactly one line end, or
// nothing at all when there are no lines.
func blockBody(content string) string {
trimmed := strings.TrimRight(content, "\n")
if trimmed == "" {
return ""
}
return trimmed + "\n"
}
func regionOf(begin, end, body string) string {
return begin + "\n" + body + end + "\n"
}
// linesOf splits text into lines that keep their line ends, so joining them again gives back
// exactly the bytes that were read — a last line without one included.
func linesOf(text string) []string {
return strings.SplitAfter(text, "\n")
}
// region is where the mesh's markers stand, as indices into the lines of a file.
type region struct{ begin, end int }
// body is what stands between the markers.
func (r region) body(lines []string) string {
return strings.Join(lines[r.begin+1:r.end], "")
}
// regionIn finds the mesh's markers for one resource. A line is a marker only if it is exactly the
// marker, so another tool's block and another resource's region are never it. Markers that do not
// form one pair — a begin with no end, an end before its begin, either twice — are an error rather
// than a best guess, because a guess is how the host would rewrite lines that are not its own.
func regionIn(lines []string, begin, end string) (region, bool, error) {
at := region{begin: -1, end: -1}
for i, line := range lines {
switch strings.TrimSuffix(line, "\n") {
case begin:
if at.begin >= 0 {
return at, false, fmt.Errorf("%q is in it more than once", begin)
}
at.begin = i
case end:
if at.end >= 0 {
return at, false, fmt.Errorf("%q is in it more than once", end)
}
at.end = i
}
}
switch {
case at.begin < 0 && at.end < 0:
return at, false, nil
case at.begin < 0:
return at, false, fmt.Errorf("%q is in it with no %q before it", end, begin)
case at.end < 0:
return at, false, fmt.Errorf("%q is in it with no %q after it", begin, end)
case at.end < at.begin:
return at, false, fmt.Errorf("%q stands before %q", end, begin)
}
return at, true, nil
}
// keepOwner gives a file rewritten through a new one back to whoever owned what it replaced.
// Changed only where it differs, so a host that is not root can still write a file it owns.
func keepOwner(path string, was os.FileInfo) error {
uid, gid, ok := ownerOf(was)
if !ok {
return nil
}
now, err := os.Stat(path)
if err != nil {
return err
}
if u, g, ok := ownerOf(now); ok && u == uid && g == gid {
return nil
}
if err := os.Chown(path, uid, gid); err != nil {
return fmt.Errorf("cannot give %s back to its owner %d:%d: %w", path, uid, gid, err)
}
return nil
}
internal/apply/block_test.go
+618
View File
@@ -0,0 +1,618 @@
package apply
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq issue 128 and ADR 0102: a text file the mesh shares with software it did not
// install is written into a marked block, never over — every line outside the mesh's markers is
// the machine's and is kept byte for byte, and undeclaring gives the file back.
const namesID = "mesh-wireguard.fact-node-names"
func blockDecl(t *testing.T, path, content string, extra ...string) string {
t.Helper()
more := ""
for _, e := range extra {
more += "," + e
}
return fmt.Sprintf(`{"declaration":1,"resources":[
{"id":%q,"type":"file","path":%q,"into":"block","content":%q%s}
]}`, namesID, path, content, more)
}
func applyBlockDecl(t *testing.T, raw string, known store.State) (Report, store.State) {
t.Helper()
report, state, err := Apply(context.Background(), archHost(t), parse(t, raw), known, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
return report, state
}
func undeclare(t *testing.T, known store.State) (Report, store.State) {
t.Helper()
report, state, err := Apply(context.Background(), archHost(t), somethingElse(t), known, store.OriginDeclared, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
return report, state
}
func readText(t *testing.T, path string) string {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
return string(raw)
}
func marked(id, body string) string {
return "# BEGIN mesh " + id + "\n" + body + "# END mesh " + id + "\n"
}
// A workstation's hosts file, the way issue 128 found it: the distribution's lines, a development
// tool's own marked blocks, and the operator's hand-added names.
const workstationHosts = "127.0.0.1\tlocalhost\n" +
"127.0.1.1\tg14.localdomain g14\n" +
"\n" +
"# BEGIN devtool project-a\n" +
"127.0.0.1 a.test api.a.test\n" +
"# END devtool project-a\n" +
"# BEGIN devtool project-b\n" +
"127.0.0.1 b.test\n" +
"# END devtool project-b\n" +
"192.168.1.20 printer # the operator's\n"
const meshNames = "10.42.0.1 ace\n10.42.0.2 novox\n"
func TestABlockKeepsEveryLineOutsideItsMarkers(t *testing.T) {
path := filepath.Join(t.TempDir(), "hosts")
if err := os.WriteFile(path, []byte(workstationHosts), 0o640); err != nil {
t.Fatal(err)
}
report, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
want := workstationHosts + "\n" + marked(namesID, meshNames)
if got := readText(t, path); got != want {
t.Fatalf("the file after writing into it:\n%q\nwant\n%q", got, want)
}
if got := report.Outcomes[0].Action; got != "updated" {
t.Errorf("adding the region was %q", got)
}
if info, _ := os.Stat(path); info.Mode().Perm() != 0o640 {
t.Errorf("the machine's file mode was changed to %o", info.Mode().Perm())
}
rec, _ := state.Find(namesID)
if rec.Into == nil || rec.Into.Format != "block" || rec.Into.Region != nil || rec.Into.Created {
t.Fatalf("recorded as %+v", rec.Into)
}
// Again, with nothing changed: nothing written.
report, state = applyBlockDecl(t, blockDecl(t, path, meshNames), state)
if got := report.Outcomes[0].Action; got != "unchanged" {
t.Errorf("a second apply was %q", got)
}
// The development tool rewrites its block, and the operator adds a line after the mesh's
// region; the mesh's names change. Only the region moves.
edited := strings.Replace(readText(t, path), "127.0.0.1 b.test\n", "127.0.0.1 b.test c.test\n", 1) +
"10.0.0.5 nas # added after\n"
_ = os.WriteFile(path, []byte(edited), 0o640)
changed := meshNames + "10.42.0.3 shanks\n"
report, state = applyBlockDecl(t, blockDecl(t, path, changed), state)
want = strings.Replace(edited, marked(namesID, meshNames), marked(namesID, changed), 1)
if got := readText(t, path); got != want {
t.Fatalf("rewriting the region moved something else:\n%q\nwant\n%q", got, want)
}
if got := report.Outcomes[0].Action; got != "updated" {
t.Errorf("rewriting the region was %q", got)
}
// Undeclared: the region, its markers and the blank line the host put before it go; every
// other line is where it was.
report, _ = undeclare(t, state)
want = strings.Replace(edited, "\n"+marked(namesID, meshNames), "", 1)
if got := readText(t, path); got != want {
t.Fatalf("undeclaring left:\n%q\nwant\n%q", got, want)
}
if got := report.Outcomes[0].Action; got != "restored" {
t.Errorf("undeclaring was %q", got)
}
}
func TestUndeclaringABlockAddedAtTheEndGivesTheFileBackExactly(t *testing.T) {
for name, original := range map[string]string{
"ending in a line": "127.0.0.1 localhost\n",
"ending in a blank line": "127.0.0.1 localhost\n\n",
"empty": "",
} {
t.Run(name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "hosts")
_ = os.WriteFile(path, []byte(original), 0o644)
_, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
undeclare(t, state)
if got := readText(t, path); got != original {
t.Errorf("undeclaring left %q, the machine had %q", got, original)
}
})
}
}
func TestABlockAddedAtTheEndIsSetApartFromTheMachinesLines(t *testing.T) {
for name, c := range map[string]struct{ before, after string }{
"no line end": {"127.0.0.1 localhost", "127.0.0.1 localhost\n\n" + marked(namesID, meshNames)},
"a line end": {"127.0.0.1 localhost\n", "127.0.0.1 localhost\n\n" + marked(namesID, meshNames)},
"a blank line already": {"127.0.0.1 localhost\n\n", "127.0.0.1 localhost\n\n" + marked(namesID, meshNames)},
"empty": {"", marked(namesID, meshNames)},
"only a blank line": {"\n", "\n" + marked(namesID, meshNames)},
"content without an end": {"x\n\n", "x\n\n" + marked(namesID, meshNames)},
} {
t.Run(name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "hosts")
_ = os.WriteFile(path, []byte(c.before), 0o644)
applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
if got := readText(t, path); got != c.after {
t.Errorf("got %q, want %q", got, c.after)
}
})
}
}
func TestTheRegionEndsInExactlyOneLineEnd(t *testing.T) {
for content, body := range map[string]string{
"10.42.0.1 ace": "10.42.0.1 ace\n",
"10.42.0.1 ace\n": "10.42.0.1 ace\n",
"10.42.0.1 ace\n\n\n": "10.42.0.1 ace\n",
"": "",
"\n\n": "",
} {
path := filepath.Join(t.TempDir(), "hosts")
_, state := applyBlockDecl(t, blockDecl(t, path, content), store.State{})
if got := readText(t, path); got != marked(namesID, body) {
t.Errorf("content %q was written as %q", content, got)
}
// And the same content again is not a change.
report, _ := applyBlockDecl(t, blockDecl(t, path, content), state)
if got := report.Outcomes[0].Action; got != "unchanged" {
t.Errorf("content %q applied twice was %q", content, got)
}
}
}
func TestARegionAlreadyThereIsRewrittenInPlaceAndGivenBack(t *testing.T) {
path := filepath.Join(t.TempDir(), "hosts")
before := "127.0.0.1 localhost\n"
after := "# BEGIN devtool x\n127.0.0.1 x.test\n# END devtool x\n192.168.1.20 printer\n"
found := "10.42.0.9 old-name\n"
original := before + marked(namesID, found) + after
_ = os.WriteFile(path, []byte(original), 0o644)
report, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
if got := readText(t, path); got != before+marked(namesID, meshNames)+after {
t.Fatalf("the region was not rewritten in place: %q", got)
}
if got := report.Outcomes[0].Action; got != "updated" {
t.Errorf("rewriting a found region was %q", got)
}
rec, _ := state.Find(namesID)
if rec.Into.Region == nil || *rec.Into.Region != found {
t.Fatalf("what the region held before was recorded as %v", rec.Into.Region)
}
// Undeclared: what the region held goes back, where it was.
report, _ = undeclare(t, state)
if got := readText(t, path); got != original {
t.Errorf("undeclaring left %q, the machine had %q", got, original)
}
if got := report.Outcomes[0].Action; got != "restored" {
t.Errorf("undeclaring was %q", got)
}
}
func TestARegionWithNoRecordHoldingExactlyTheDeclaredLinesIsTheMeshs(t *testing.T) {
// A host that wrote the region and died before saving its state: what is between the markers
// is exactly what the mesh declares, and remembered as the machine's it would never go.
path := filepath.Join(t.TempDir(), "hosts")
original := "127.0.0.1 localhost\n"
_ = os.WriteFile(path, []byte(original+"\n"+marked(namesID, meshNames)), 0o644)
_, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
if rec, _ := state.Find(namesID); rec.Into.Region != nil {
t.Fatalf("the mesh's own lines were recorded as the machine's: %q", *rec.Into.Region)
}
undeclare(t, state)
if got := readText(t, path); !strings.HasPrefix(got, original) || strings.Contains(got, "BEGIN mesh") {
t.Errorf("undeclaring left the mesh's region behind: %q", got)
}
}
func TestADriftedRegionIsCorrected(t *testing.T) {
path := filepath.Join(t.TempDir(), "hosts")
_ = os.WriteFile(path, []byte(workstationHosts), 0o644)
_, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
written := readText(t, path)
_ = os.WriteFile(path, []byte(strings.Replace(written, "10.42.0.2 novox\n", "10.42.0.2 novox\n6.6.6.6 evil\n", 1)), 0o644)
report, _ := applyBlockDecl(t, blockDecl(t, path, meshNames), state)
if got := report.Outcomes[0].Action; got != "corrected" {
t.Errorf("a region edited on the machine was %q", got)
}
if got := readText(t, path); got != written {
t.Errorf("the region was not put back: %q", got)
}
// The region taken out by hand is drift too, and it is put back.
_ = os.WriteFile(path, []byte(workstationHosts), 0o644)
report, _ = applyBlockDecl(t, blockDecl(t, path, meshNames), state)
if got := report.Outcomes[0].Action; got != "corrected" {
t.Errorf("a region removed on the machine was %q", got)
}
if got := readText(t, path); got != written {
t.Errorf("the region was not put back: %q", got)
}
}
func TestTwoRegionsInOneFileAreEachTheirOwn(t *testing.T) {
path := filepath.Join(t.TempDir(), "hosts")
_ = os.WriteFile(path, []byte(workstationHosts), 0o644)
raw := fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"a.names","type":"file","path":%q,"into":"block","content":"10.42.0.1 ace\n"},
{"id":"b.names","type":"file","path":%q,"into":"block","content":"10.43.0.1 lab\n"}
]}`, path, path)
_, state := applyBlockDecl(t, raw, store.State{})
want := workstationHosts + "\n" + marked("a.names", "10.42.0.1 ace\n") + "\n" + marked("b.names", "10.43.0.1 lab\n")
if got := readText(t, path); got != want {
t.Fatalf("two regions:\n%q\nwant\n%q", got, want)
}
report, state := applyBlockDecl(t, raw, state)
for _, o := range report.Outcomes {
if o.Action != "unchanged" {
t.Errorf("%s applied twice was %q", o.ID, o.Action)
}
}
// One undeclared: only its region goes.
only := fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"b.names","type":"file","path":%q,"into":"block","content":"10.43.0.1 lab\n"}
]}`, path)
applyBlockDecl(t, only, state)
want = workstationHosts + "\n" + marked("b.names", "10.43.0.1 lab\n")
if got := readText(t, path); got != want {
t.Errorf("undeclaring one region:\n%q\nwant\n%q", got, want)
}
}
func TestABlockInAFileThatWasNotThereIsCreatedAndRemovedWithIt(t *testing.T) {
path := filepath.Join(t.TempDir(), "conf.d", "mesh.conf")
report, state := applyBlockDecl(t, blockDecl(t, path, meshNames, `"mode":"0600"`), store.State{})
if got := readText(t, path); got != marked(namesID, meshNames) {
t.Fatalf("a created file holds %q", got)
}
if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 {
t.Errorf("a created file is mode %o, declared 0600", info.Mode().Perm())
}
if got := report.Outcomes[0].Action; got != "created" {
t.Errorf("creating was %q", got)
}
if rec, _ := state.Find(namesID); !rec.Into.Created {
t.Error("the mesh creating the file was not recorded")
}
report, _ = undeclare(t, state)
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Errorf("a file the mesh created, holding only its region, was left behind")
}
if got := report.Outcomes[0].Action; got != "removed" {
t.Errorf("undeclaring was %q", got)
}
// Somebody else wrote into it meanwhile: it is no longer only the mesh's, and it stays.
_, state = applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
_ = os.WriteFile(path, []byte(readText(t, path)+"their = line\n"), 0o600)
undeclare(t, state)
if got := readText(t, path); got != "their = line\n" {
t.Errorf("undeclaring a created file somebody wrote into left %q", got)
}
}
func TestMarkersThatDoNotPairAreRefusedAndLeftAlone(t *testing.T) {
for name, text := range map[string]string{
"a begin with no end": "a\n# BEGIN mesh " + namesID + "\nb\n",
"an end with no begin": "a\n# END mesh " + namesID + "\n",
"an end before a begin": "# END mesh " + namesID + "\n# BEGIN mesh " + namesID + "\n",
"a begin twice": marked(namesID, "x\n") + "# BEGIN mesh " + namesID + "\n",
} {
t.Run(name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "hosts")
_ = os.WriteFile(path, []byte(text), 0o644)
if _, _, err := Apply(context.Background(), archHost(t), parse(t, blockDecl(t, path, meshNames)),
store.State{}, store.OriginDeclared, nil, nil, nil); err == nil {
t.Fatal("markers that do not pair were written between")
}
if got := readText(t, path); got != text {
t.Errorf("the file was changed: %q", got)
}
})
}
}
func TestAMarkerOfAnotherIDIsNotThisRegion(t *testing.T) {
// The id is part of the marker: a region whose id merely starts with this one is not it.
path := filepath.Join(t.TempDir(), "hosts")
other := marked(namesID+"-extra", "10.9.9.9 other\n")
_ = os.WriteFile(path, []byte(other), 0o644)
_, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
if got := readText(t, path); got != other+"\n"+marked(namesID, meshNames) {
t.Fatalf("got %q", got)
}
undeclare(t, state)
if got := readText(t, path); got != other {
t.Errorf("undeclaring touched the other region: %q", got)
}
}
func TestABlockAtTheStartStandsAboveEverything(t *testing.T) {
// dhcpcd scopes every line after `interface X` to that interface, so the mesh's global options
// go above all of it.
dhcpcd := "hostname\nduid\n\ninterface enp6s0\nstatic ip_address=192.168.1.5/24\n"
opts := "nohook resolv.conf\ndenyinterfaces mesh0\n"
for name, c := range map[string]struct{ before, after string }{
"a file with content": {dhcpcd, marked(namesID, opts) + "\n" + dhcpcd},
"an empty file": {"", marked(namesID, opts)},
"a file opening blank": {"\n" + dhcpcd, marked(namesID, opts) + "\n" + dhcpcd},
"a last line with no end": {"interface enp6s0", marked(namesID, opts) + "\ninterface enp6s0"},
} {
t.Run(name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "dhcpcd.conf")
_ = os.WriteFile(path, []byte(c.before), 0o644)
report, state := applyBlockDecl(t, blockDecl(t, path, opts, `"at":"start"`), store.State{})
if got := readText(t, path); got != c.after {
t.Fatalf("got %q, want %q", got, c.after)
}
if got := report.Outcomes[0].Action; got != "updated" {
t.Errorf("adding the region was %q", got)
}
report, state = applyBlockDecl(t, blockDecl(t, path, opts, `"at":"start"`), state)
if got := report.Outcomes[0].Action; got != "unchanged" {
t.Errorf("a second apply was %q", got)
}
undeclare(t, state)
if got := readText(t, path); got != c.before {
t.Errorf("undeclaring left %q, the machine had %q", got, c.before)
}
})
}
t.Run("a file that was not there", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "dhcpcd.conf")
applyBlockDecl(t, blockDecl(t, path, opts, `"at":"start"`), store.State{})
if got := readText(t, path); got != marked(namesID, opts) {
t.Errorf("got %q", got)
}
})
}
func TestARegionAlreadyThereIsNotMovedWhereverAtSaysItGoes(t *testing.T) {
for _, at := range []string{`"at":"start"`, `"at":"end"`} {
path := filepath.Join(t.TempDir(), "dhcpcd.conf")
original := "hostname\n" + marked(namesID, "old\n") + "interface enp6s0\n"
_ = os.WriteFile(path, []byte(original), 0o644)
applyBlockDecl(t, blockDecl(t, path, "nohook resolv.conf\n", at), store.State{})
want := "hostname\n" + marked(namesID, "nohook resolv.conf\n") + "interface enp6s0\n"
if got := readText(t, path); got != want {
t.Errorf("%s: a found region was moved: %q", at, got)
}
}
}
func TestAFileWrittenIntoABlockIsNeverHeldOnAnAdoptedNode(t *testing.T) {
path := filepath.Join(t.TempDir(), "hosts")
_ = os.WriteFile(path, []byte(workstationHosts), 0o644)
resource := fmt.Sprintf(`{"id":%q,"type":"file","path":%q,"into":"block","content":%q}`, namesID, path, meshNames)
d := adopted(t, `{"taken":[],"untaken":{"mesh-wireguard":["`+namesID+`"]}}`, resource)
report, state := applyAdopted(t, d, store.State{}, &machine{}, t.TempDir())
if got := outcomeOf(report, namesID).Action; got == "held" {
t.Fatal("a file written into a block was held, though it replaces nothing that was found")
}
if len(state.Held) != 0 {
t.Errorf("something was held: %+v", state.Held)
}
if got := readText(t, path); got != workstationHosts+"\n"+marked(namesID, meshNames) {
t.Errorf("the adopted node's file was not written into: %q", got)
}
// Held from when it was declared whole, the hold does not keep the region out.
path2 := filepath.Join(t.TempDir(), "hosts")
_ = os.WriteFile(path2, []byte(workstationHosts), 0o644)
known := store.State{Held: []store.Held{{ID: namesID, Module: "mesh-wireguard", Kind: "file", Target: path2}}}
resource2 := fmt.Sprintf(`{"id":%q,"type":"file","path":%q,"into":"block","content":%q}`, namesID, path2, meshNames)
d2 := adopted(t, `{"taken":[],"untaken":{"mesh-wireguard":["`+namesID+`"]}}`, resource2)
report, state = applyAdopted(t, d2, known, &machine{}, t.TempDir())
if got := outcomeOf(report, namesID).Action; got != "updated" {
t.Errorf("the file was %q, not written into", got)
}
if len(state.Held) != 0 {
t.Errorf("the old hold outlived the block declaration: %+v", state.Held)
}
// And the preview says the same: written into, not held.
for _, s := range Plan(d2, known, store.OriginDeclared) {
if s.ID == namesID && s.Verb == "hold" {
t.Errorf("the preview holds a file written into a block: %+v", s)
}
}
}
func TestTheRecordOfABlockSurvivesTheStateFile(t *testing.T) {
// What undeclaring needs is in the state a host saves, not only in memory.
path := filepath.Join(t.TempDir(), "hosts")
original := "a\n" + marked(namesID, "old\n")
_ = os.WriteFile(path, []byte(original), 0o644)
_, state := applyBlockDecl(t, blockDecl(t, path, meshNames, `"at":"start"`), store.State{})
raw, err := json.Marshal(state)
if err != nil {
t.Fatal(err)
}
var back store.State
if err := json.Unmarshal(raw, &back); err != nil {
t.Fatal(err)
}
undeclare(t, back)
if got := readText(t, path); got != original {
t.Errorf("undeclaring from a saved state left %q", got)
}
}
// The mesh's old whole hosts file, as the controller composed it before issue 128.
const oldWholeHosts = "# Generated by the mesh. Do not edit — this file is replaced whenever a machine\n" +
"# joins or leaves, and an edit would survive until then and vanish.\n\n" +
"127.0.0.1\tlocalhost\n" +
"::1\t\tlocalhost ip6-localhost ip6-loopback\n" +
"127.0.1.1\tg14\n" +
"\n" +
"10.42.0.1\tace.internal\tace\n" +
"10.42.0.9\tg14.internal\tg14\t# this machine\n"
func wholeDecl(path, content string) string {
return fmt.Sprintf(`{"declaration":1,"resources":[
{"id":%q,"type":"file","path":%q,"content":%q}
]}`, namesID, path, content)
}
func applyKeepingIn(t *testing.T, raw string, known store.State, keepDir string) (Report, store.State) {
t.Helper()
report, state, err := ApplyKeeping(context.Background(), archHost(t), parse(t, raw), known,
store.OriginDeclared, (&machine{}).run, nil, nil, KeepIn(keepDir))
if err != nil {
t.Fatalf("apply failed: %v", err)
}
return report, state
}
func TestAFileTheMeshWroteWholeAndMadeItselfKeepsOnlyItsLoopbackLines(t *testing.T) {
// Written whole into a file that was not there, then declared as a block under the same id:
// the old names must not stay above the region, where a resolver would answer from them first.
path := filepath.Join(t.TempDir(), "hosts")
_, state := applyKeepingIn(t, wholeDecl(path, oldWholeHosts), store.State{}, t.TempDir())
report, state := applyKeepingIn(t, blockDecl(t, path, meshNames), state, t.TempDir())
floor := "127.0.0.1\tlocalhost\n::1\t\tlocalhost ip6-localhost ip6-loopback\n127.0.1.1\tg14\n"
if got := readText(t, path); got != floor+"\n"+marked(namesID, meshNames) {
t.Fatalf("the old whole file became:\n%q", got)
}
o := outcomeOf(report, namesID)
if o.Action != "updated" || !strings.Contains(o.Detail, "loopback lines") {
t.Errorf("the rebuild was reported as %q: %s", o.Action, o.Detail)
}
if rec, _ := state.Find(namesID); !rec.Into.Created {
t.Error("a file the mesh made itself was not recorded as the mesh's")
}
report, _ = applyKeepingIn(t, blockDecl(t, path, meshNames), state, t.TempDir())
if got := outcomeOf(report, namesID).Action; got != "unchanged" {
t.Errorf("applied again, the rebuilt file was %q", got)
}
undeclare(t, state)
if got := readText(t, path); got != floor {
t.Errorf("undeclared, the file holds %q", got)
}
}
func TestAFileTheMeshWroteWholeOverAnOriginalGetsTheOriginalBack(t *testing.T) {
path := filepath.Join(t.TempDir(), "hosts")
_ = os.WriteFile(path, []byte(workstationHosts), 0o644)
keep := t.TempDir()
_, state := applyKeepingIn(t, wholeDecl(path, oldWholeHosts), store.State{}, keep)
if rec, _ := state.Find(namesID); rec.Kept == "" {
t.Fatal("where the original was kept was not recorded")
}
report, state := applyKeepingIn(t, blockDecl(t, path, meshNames), state, keep)
if got := readText(t, path); got != workstationHosts+"\n"+marked(namesID, meshNames) {
t.Fatalf("the old whole file became:\n%q", got)
}
if d := outcomeOf(report, namesID).Detail; !strings.Contains(d, "original kept at") {
t.Errorf("the rebuild was reported as: %s", d)
}
undeclare(t, state)
if got := readText(t, path); got != workstationHosts {
t.Errorf("undeclared, the machine did not get its original back: %q", got)
}
}
func TestAFileTheMeshWroteWholeAndSomebodyChangedIsWrittenIntoAsItStands(t *testing.T) {
path := filepath.Join(t.TempDir(), "hosts")
_, state := applyKeepingIn(t, wholeDecl(path, oldWholeHosts), store.State{}, t.TempDir())
edited := oldWholeHosts + "192.168.1.20 printer\n"
_ = os.WriteFile(path, []byte(edited), 0o644)
report, _ := applyKeepingIn(t, blockDecl(t, path, meshNames), state, t.TempDir())
if got := readText(t, path); got != edited+"\n"+marked(namesID, meshNames) {
t.Fatalf("an edited whole file became:\n%q", got)
}
if d := outcomeOf(report, namesID).Detail; !strings.Contains(d, "changed since; its old lines were kept") {
t.Errorf("the outcome does not say so: %s", d)
}
}
func TestALinkedFileStaysALink(t *testing.T) {
dir := t.TempDir()
real := filepath.Join(dir, "static", "hosts")
_ = os.MkdirAll(filepath.Dir(real), 0o755)
_ = os.WriteFile(real, []byte(workstationHosts), 0o644)
link := filepath.Join(dir, "hosts")
if err := os.Symlink(real, link); err != nil {
t.Fatal(err)
}
_, state := applyBlockDecl(t, blockDecl(t, link, meshNames), store.State{})
if info, err := os.Lstat(link); err != nil || info.Mode()&os.ModeSymlink == 0 {
t.Fatalf("the link was replaced by a file")
}
if got := readText(t, real); got != workstationHosts+"\n"+marked(namesID, meshNames) {
t.Errorf("the file the link names holds %q", got)
}
undeclare(t, state)
if info, err := os.Lstat(link); err != nil || info.Mode()&os.ModeSymlink == 0 {
t.Fatalf("undeclaring replaced the link with a file")
}
if got := readText(t, real); got != workstationHosts {
t.Errorf("undeclared, the file the link names holds %q", got)
}
}
func TestALastLineWithNoEndIsGivenBackWithNone(t *testing.T) {
path := filepath.Join(t.TempDir(), "hosts")
_ = os.WriteFile(path, []byte("x"), 0o644)
_, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
if got := readText(t, path); got != "x\n\n"+marked(namesID, meshNames) {
t.Fatalf("got %q", got)
}
undeclare(t, state)
if got := readText(t, path); got != "x" {
t.Errorf("undeclaring left %q, the machine had %q", got, "x")
}
}
func TestAFailedBlockWriteKeepsItsHold(t *testing.T) {
// Held from when it was declared whole, then declared as a block into a file whose markers do
// not pair: the write is refused, and the hold — with where its original is — stays.
path := filepath.Join(t.TempDir(), "hosts")
broken := "a\n# BEGIN mesh " + namesID + "\n"
_ = os.WriteFile(path, []byte(broken), 0o644)
known := store.State{Held: []store.Held{{ID: namesID, Module: "mesh-wireguard", Kind: "file",
Target: path, Kept: "/var/lib/mesh/kept/hosts"}}}
resource := fmt.Sprintf(`{"id":%q,"type":"file","path":%q,"into":"block","content":%q}`, namesID, path, meshNames)
d := adopted(t, `{"taken":[],"untaken":{"mesh-wireguard":["`+namesID+`"]}}`, resource)
_, state, err := ApplyKeeping(context.Background(), archHost(t), d, known,
store.OriginDeclared, (&machine{}).run, nil, nil, KeepIn(t.TempDir()))
if err == nil {
t.Fatal("a write into unpaired markers was not refused")
}
h, held := state.HeldAt(namesID)
if !held || h.Kept != "/var/lib/mesh/kept/hosts" {
t.Errorf("a failed write released the hold: %+v", state.Held)
}
}
internal/apply/found_test.go
+387
View File
@@ -0,0 +1,387 @@
package apply
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0118: undeclaring removes what the mesh made, gives back what it changed,
// and leaves what was the machine's — which needs what was found kept exactly, and a unit the mesh
// made known for the mesh's whatever its record says.
func unitsMachine(units map[string]*fakeUnit) *machine {
return &machine{containers: map[string]*fakeContainer{}, units: units}
}
// nothingButA is a declaration of one unrelated file, so everything recorded is undeclared.
func nothingButA(t *testing.T) string {
return `{"declaration":1,"resources":[
{"id":"other","type":"file","path":"` + filepath.Join(t.TempDir(), "a") + `","content":"a\n"}]}`
}
// copyOf is a state as a later load of it would be: sharing nothing with the one it came from.
func copyOf(t *testing.T, s store.State) store.State {
t.Helper()
raw, err := json.Marshal(s)
if err != nil {
t.Fatal(err)
}
var out store.State
if err := json.Unmarshal(raw, &out); err != nil {
t.Fatal(err)
}
return out
}
func applyOn(t *testing.T, raw string, known store.State, m *machine) (Report, store.State, error) {
t.Helper()
return Apply(context.Background(), archHost(t), parse(t, raw), known, store.OriginDeclared, m.run, nil, nil)
}
func TestAUnitWhoseFileTheMeshWroteIsStoppedWhateverItsRecordSays(t *testing.T) {
// The adoption guard's unit file is the mesh's own file resource, written where there was none.
// Its service recorded before the host kept what it found has no Found, and forgetting it left
// the guard's table loaded on a converged node and its unit file deleted from under it.
units := t.TempDir()
was := unitDir
unitDir = units
t.Cleanup(func() { unitDir = was })
unitFile := filepath.Join(units, "mesh-guard.service")
if err := os.WriteFile(unitFile, []byte("[Unit]\n"), 0o644); err != nil {
t.Fatal(err)
}
m := unitsMachine(map[string]*fakeUnit{"mesh-guard.service": {active: "active", enabled: "enabled"}})
fileThereAtStop := false
run := func(ctx context.Context, name string, args ...string) (string, error) {
if name == "systemctl" && args[0] == "stop" {
_, err := os.Stat(unitFile)
fileThereAtStop = err == nil
}
return m.run(ctx, name, args...)
}
// As a host before this change recorded them: the unit file first, then the service it
// starts, and no Found on the service.
known := store.State{Resources: []store.Applied{
{ID: "adoption.guard-unit", Type: "file", Target: unitFile, Origin: store.OriginDeclared},
{ID: "adoption.guard-running", Type: "service", Target: "mesh-guard.service", Origin: store.OriginDeclared},
}}
report, after, err := applyWith(t, parse(t, nothingButA(t)), known, run)
if err != nil {
t.Fatal(err)
}
if u := m.units["mesh-guard.service"]; u.active != "inactive" || u.enabled != "disabled" {
t.Errorf("the mesh's own unit was left %s and %s: %v", u.active, u.enabled, m.asked)
}
if !fileThereAtStop {
t.Error("the unit was stopped after its file was deleted, or not at all")
}
if o := outcomeOf(report, "adoption.guard-running"); o.Action != "removed" || !strings.Contains(o.Detail, "unit file") {
t.Errorf("outcome %+v", o)
}
if _, err := os.Stat(unitFile); !os.IsNotExist(err) {
t.Error("the unit file outlived its record")
}
if len(after.Resources) != 1 {
t.Errorf("still recorded: %v", after.IDs())
}
// A unit file the host wrote OVER — its original kept — is the machine's unit, and a record
// with no Found leaves it as it is.
if err := os.WriteFile(unitFile, []byte("[Unit]\n"), 0o644); err != nil {
t.Fatal(err)
}
m = unitsMachine(map[string]*fakeUnit{"mesh-guard.service": {active: "active", enabled: "enabled"}})
known.Resources[0].Kept = filepath.Join(t.TempDir(), "original")
if err := os.WriteFile(known.Resources[0].Kept, []byte("[Unit]\n"), 0o600); err != nil {
t.Fatal(err)
}
if _, _, err := applyWith(t, parse(t, nothingButA(t)), known, m.run); err != nil {
t.Fatal(err)
}
if m.did("systemctl stop") || m.did("systemctl disable") {
t.Errorf("a unit whose file the mesh only wrote over was stopped: %v", m.asked)
}
}
func TestOnlyAUnitsOwnFileTheMeshCreatedMakesItTheMeshs(t *testing.T) {
known := store.State{Resources: []store.Applied{
{ID: "a", Type: "file", Target: "/etc/systemd/system/made.service"},
{ID: "b", Type: "file", Target: "/run/systemd/system/runtime.service"},
{ID: "c", Type: "file", Target: "/etc/systemd/system/kept.service", Kept: "/var/lib/mesh-host/kept/x"},
{ID: "d", Type: "file", Target: "/etc/systemd/system/into.service", Into: &store.Into{Format: "block"}},
{ID: "e", Type: "file", Target: "/etc/systemd/system/docker.service.d/mesh.conf"},
{ID: "f", Type: "file", Target: "/etc/mesh/elsewhere.service"},
{ID: "g", Type: "directory", Target: "/etc/systemd/system/dir.service"},
}}
made := meshMadeUnits(known)
for unit, want := range map[string]bool{"made.service": true, "runtime.service": true, "kept.service": false,
"into.service": false, "mesh.conf": false, "docker.service.d": false, "elsewhere.service": false, "dir.service": false} {
if made[unit] != want {
t.Errorf("%s: made %v, want %v", unit, made[unit], want)
}
}
}
func TestWhatWasFoundOutlivesAFirstApplyThatFailed(t *testing.T) {
// Enabled, then it would not start: no record. The next apply must not read the enable as
// the machine's — or undeclaring leaves enabled a unit the mesh enabled.
m := unitsMachine(map[string]*fakeUnit{"filter.service": {active: "inactive", enabled: "disabled", wontStart: true}})
declared := `{"declaration":1,"resources":[
{"id":"s","type":"service","unit":"filter.service","state":"running","boot":"enabled"}]}`
_, first, err := applyOn(t, declared, store.State{}, m)
if err == nil || !m.did("systemctl enable filter.service") {
t.Fatalf("the fixture did not enable and then fail: %v, %v", err, m.asked)
}
if _, ok := first.Find("s"); ok {
t.Fatal("a failed apply was recorded")
}
if p := first.FoundFirst["s"]; p.State != "stopped" || p.Boot != "disabled" || p.Unit != "filter.service" {
t.Fatalf("what was found was not kept through the failure: %+v", first.FoundFirst)
}
failed := copyOf(t, first)
m.units["filter.service"].wontStart = false
_, second, err := applyOn(t, declared, first, m)
if err != nil {
t.Fatal(err)
}
if rec, _ := second.Find("s"); rec.Found == nil || rec.Found.State != "stopped" || rec.Found.Boot != "disabled" {
t.Errorf("the record carries the mesh's own effect as found: %+v", rec.Found)
}
if second.FoundFirst != nil {
t.Errorf("kept apart after the record carried it: %+v", second.FoundFirst)
}
if _, _, err := applyOn(t, nothingButA(t), second, m); err != nil {
t.Fatal(err)
}
if u := m.units["filter.service"]; u.active != "inactive" || u.enabled != "disabled" {
t.Errorf("undeclared, the unit was left %s and %s", u.active, u.enabled)
}
// Undeclared before it was ever recorded, what was found goes with it — only for its origin.
if _, kept, _ := Apply(context.Background(), archHost(t), parse(t, nothingButA(t)), copyOf(t, failed),
store.OriginCarried, m.run, nil, nil); kept.FoundFirst["s"].State == "" {
t.Error("a carried apply dropped what the mesh's declaration found")
}
if _, dropped, err := applyOn(t, nothingButA(t), copyOf(t, failed), m); err != nil || dropped.FoundFirst != nil {
t.Errorf("kept for a service no longer declared: %+v, %v", dropped.FoundFirst, err)
}
}
func TestBootIsFoundTheFirstTimeTheMeshSetsIt(t *testing.T) {
// The declaration said nothing about boot at first, so the mesh never touched it: what is
// there when a declaration first does is still the machine's.
m := unitsMachine(map[string]*fakeUnit{"web.service": {active: "active", enabled: "disabled"}})
known := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "web.service",
Origin: store.OriginDeclared, Found: &store.FoundUnit{Unit: "web.service", State: "running"}}}}
_, after, err := applyOn(t, `{"declaration":1,"resources":[
{"id":"s","type":"service","unit":"web.service","state":"running","boot":"enabled"}]}`, known, m)
if err != nil {
t.Fatal(err)
}
rec, _ := after.Find("s")
if rec.Found == nil || rec.Found.State != "running" || rec.Found.Boot != "disabled" {
t.Fatalf("found %+v, want running and disabled", rec.Found)
}
report, _, err := applyOn(t, nothingButA(t), after, m)
if err != nil {
t.Fatal(err)
}
if u := m.units["web.service"]; u.active != "active" || u.enabled != "disabled" {
t.Errorf("undeclared, the unit is %s and %s; want running, and disabled again", u.active, u.enabled)
}
if o := outcomeOf(report, "s"); o.Action != "restored" || o.Detail != "disabled at boot, as the host found it" {
t.Errorf("outcome %+v", o)
}
}
func TestAServiceOnceDeclaredWithNoStateIsFoundWhenFirstGivenOne(t *testing.T) {
// Declared with no state (novox/hq ADR 0117), the mesh never started or stopped it — so what
// is there when a declaration first gives it one is what the machine had.
m := unitsMachine(map[string]*fakeUnit{"net.service": {active: "inactive", enabled: "disabled"}})
known := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "net.service",
Origin: store.OriginDeclared, Stateless: true}}}
_, after, err := applyOn(t, `{"declaration":1,"resources":[
{"id":"s","type":"service","unit":"net.service","state":"running"}]}`, known, m)
if err != nil {
t.Fatal(err)
}
if rec, _ := after.Find("s"); rec.Found == nil || rec.Found.State != "stopped" {
t.Fatalf("found %+v, want stopped", rec.Found)
}
if _, _, err := applyOn(t, nothingButA(t), after, m); err != nil {
t.Fatal(err)
}
if m.units["net.service"].active != "inactive" {
t.Error("the unit the mesh started outlived its declaration")
}
}
func TestAUnitWrittenInTheSameApplyIsLoadedBeforeItIsRead(t *testing.T) {
// Read before the service manager is told about its new file, the unit is not there to find.
dir := t.TempDir()
m := unitsMachine(map[string]*fakeUnit{"fresh.service": {active: "inactive", enabled: "disabled"}})
_, _, err := applyOn(t, `{"declaration":1,"resources":[
{"id":"unit","type":"file","path":"`+filepath.Join(dir, "fresh.service")+`","content":"[Unit]\n"},
{"id":"s","type":"service","unit":"fresh.service","state":"running","restart-on":["unit"]}]}`,
store.State{}, m)
if err != nil {
t.Fatal(err)
}
reload, show := -1, -1
for i, a := range m.asked {
if a == "systemctl daemon-reload" && reload < 0 {
reload = i
}
if strings.HasPrefix(a, "systemctl show fresh.service") && show < 0 {
show = i
}
}
if reload < 0 || show < 0 || reload > show {
t.Errorf("the unit was read before its file was loaded: %v", m.asked)
}
}
func TestAServiceMovedToAnotherUnitGivesTheOldOneBackAndFindsTheNewOne(t *testing.T) {
m := unitsMachine(map[string]*fakeUnit{
"old.service": {active: "active", enabled: "enabled"},
"new.service": {active: "inactive", enabled: "disabled"},
})
known := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "old.service",
Origin: store.OriginDeclared, Found: &store.FoundUnit{Unit: "old.service", State: "stopped"}}}}
report, after, err := applyOn(t, `{"declaration":1,"resources":[
{"id":"s","type":"service","unit":"new.service","state":"running"}]}`, known, m)
if err != nil {
t.Fatal(err)
}
if m.units["old.service"].active != "inactive" {
t.Error("the unit the mesh started is still running though nothing declares it")
}
if m.units["new.service"].active != "active" {
t.Error("the unit now declared was not started")
}
rec, _ := after.Find("s")
if rec.Found == nil || rec.Found.Unit != "new.service" || rec.Found.State != "stopped" {
t.Errorf("what was found about the old unit was carried to the new one: %+v", rec.Found)
}
if o := outcomeOf(report, "s"); !strings.Contains(o.Detail, "old.service stopped, as the host found it") {
t.Errorf("giving the old unit back went unsaid: %+v", o)
}
}
func TestARemovalSaysWhatItDid(t *testing.T) {
cases := []struct {
name string
found *store.FoundUnit
unit *fakeUnit
action, detail string
}{
{"found stopped and still stopped", &store.FoundUnit{State: "stopped"},
&fakeUnit{active: "inactive", enabled: "disabled"}, "forgotten", "already as the host found it (stopped)"},
{"started by the mesh", &store.FoundUnit{State: "stopped"},
&fakeUnit{active: "active", enabled: "disabled"}, "restored", "stopped, as the host found it"},
{"started and enabled by the mesh", &store.FoundUnit{State: "stopped", Boot: "disabled"},
&fakeUnit{active: "active", enabled: "enabled"}, "restored", "stopped, disabled at boot, as the host found it"},
{"found running, stopped by the mesh", &store.FoundUnit{State: "running"},
&fakeUnit{active: "inactive", enabled: "disabled"}, "forgotten",
"left as it is; the mesh stopped it and does not start anything on the way out"},
{"found running and enabled, disabled by the mesh", &store.FoundUnit{State: "running", Boot: "enabled"},
&fakeUnit{active: "active", enabled: "disabled"}, "forgotten",
"left as it is; the mesh disabled it at boot and does not start anything on the way out"},
{"found running, still running", &store.FoundUnit{State: "running"},
&fakeUnit{active: "active", enabled: "enabled"}, "forgotten", "it was running before the mesh; left as it is"},
// Found says to stop it, so the machine is asked about it — and it is gone.
{"started by the mesh, since uninstalled", &store.FoundUnit{State: "stopped"},
nil, "forgotten", "the unit no longer exists"},
{"recorded before the host kept what it found", nil,
&fakeUnit{active: "active", enabled: "enabled"}, "forgotten", "recorded before the host kept what it found; left as it is"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
units := map[string]*fakeUnit{}
if c.unit != nil {
units["unit.service"] = c.unit
}
m := unitsMachine(units)
known := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "unit.service",
Origin: store.OriginDeclared, Found: c.found}}}
report, after, err := applyOn(t, nothingButA(t), known, m)
if err != nil {
t.Fatal(err)
}
if o := outcomeOf(report, "s"); o.Action != c.action || o.Detail != c.detail {
t.Errorf("said %s: %s; want %s: %s", o.Action, o.Detail, c.action, c.detail)
}
if c.unit == nil && !m.did("systemctl show unit.service") {
t.Errorf("the machine was never asked whether the unit is there: %v", m.asked)
}
if m.did("systemctl start") || m.did("systemctl enable") {
t.Errorf("something was started on the way out: %v", m.asked)
}
if _, still := after.Find("s"); still {
t.Error("still recorded")
}
})
}
}
func TestAUnitTheMeshStartedIsStoppedWhenUndeclaredAndOneFoundRunningIsNot(t *testing.T) {
// End to end: found by the first apply, carried, given back.
m := unitsMachine(map[string]*fakeUnit{
"filter.service": {active: "inactive", enabled: "disabled"},
"runtime.service": {active: "active", enabled: "enabled"},
})
_, state, err := applyOn(t, `{"declaration":1,"resources":[
{"id":"filter","type":"service","unit":"filter.service","state":"running","boot":"enabled"},
{"id":"runtime","type":"service","unit":"runtime.service","state":"running","boot":"enabled"}]}`,
store.State{}, m)
if err != nil {
t.Fatal(err)
}
if m.units["filter.service"].active != "active" {
t.Fatal("the fixture did not start the filter")
}
if _, _, err := applyOn(t, nothingButA(t), state, m); err != nil {
t.Fatal(err)
}
if u := m.units["filter.service"]; u.active != "inactive" || u.enabled != "disabled" {
t.Errorf("the filter the mesh started and enabled is %s and %s", u.active, u.enabled)
}
if u := m.units["runtime.service"]; u.active != "active" || u.enabled != "enabled" {
t.Errorf("the runtime that was running before the mesh is %s and %s", u.active, u.enabled)
}
}
func TestAUnitHeldAndThenTakenIsFoundAsThePredecessorLeftIt(t *testing.T) {
// Held while its module was untaken, nothing was applied and nothing found; taken, the first
// apply finds the predecessor's unit — stopped, but started at boot — before starting it.
dir := t.TempDir()
m := unitsMachine(map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "enabled"}})
service := `{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled"}`
_, held := applyAdopted(t, adopted(t, untaken("hello-web.unit"), service), store.State{}, m, dir)
if _, ok := held.HeldAt("hello-web.unit"); !ok {
t.Fatal("the fixture's unit was not held")
}
_, taken := applyAdopted(t, adopted(t, `{"taken":["hello-web"]}`, service), held, m, dir)
rec, _ := taken.Find("hello-web.unit")
if rec.Found == nil || rec.Found.State != "stopped" || rec.Found.Boot != "enabled" {
t.Fatalf("found %+v, want the predecessor's stopped and enabled", rec.Found)
}
if m.units["hello.service"].active != "active" {
t.Fatal("the taken unit was not started")
}
if _, _, err := applyOn(t, nothingButA(t), taken, m); err != nil {
t.Fatal(err)
}
if u := m.units["hello.service"]; u.active != "inactive" || u.enabled != "enabled" {
t.Errorf("given back as %s and %s; the predecessor left it stopped and enabled", u.active, u.enabled)
}
}
internal/apply/hold.go
+19 -6
View File
@@ -108,7 +108,7 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati
}
}
case *declaration.Service:
if known.Recorded(string(declaration.TypeService), res.Unit) {
if res.Stateless() || known.Recorded(string(declaration.TypeService), res.Unit) {
continue
}
// **Found is a unit somebody put on this machine, or one the machine uses.**
@@ -261,6 +261,19 @@ func heldContainer(known store.State, name string) (store.Held, bool) {
return store.Held{}, false
}
// replacesNothing is a resource that takes nothing found on the machine from it, so on an adopted
// node it is never held and never previewed as replacing what was found: a file written into
// (novox/hq ADR 0102), and a service whose unit's lifecycle is the machine's (novox/hq ADR 0117).
func replacesNothing(r declaration.Resource) bool {
switch res := r.(type) {
case *declaration.File:
return res.Into != ""
case *declaration.Service:
return res.Stateless()
}
return false
}
// holdOnAdopted decides whether a resource of an adopted node is held rather than applied, and
// holds it (novox/hq ADR 0100, ADR 0103). For a module not yet taken, what is present with no
// record is kept as it is: a file or a container under its name, a directory, a service's unit,
@@ -295,11 +308,11 @@ func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resourc
}
// A file written into replaces nothing that was found, so it is never held (novox/hq ADR
// 0102) — and a hold from when it was declared whole must not keep the mesh's keys out.
if f, ok := r.(*declaration.File); ok && f.Into != "" {
if already {
known.Release(r.Identity())
}
// 0102) — and a hold from when it was declared whole must not keep the mesh's keys out. That
// hold is released by the apply once the write has worked, not here: a write that fails keeps
// it, and with it where the original was kept. A service whose lifecycle is the machine's
// replaces nothing either (novox/hq ADR 0117).
if replacesNothing(r) {
return false, false, out, nil
}
internal/apply/hold_test.go
+7
View File
@@ -21,6 +21,10 @@ type machine struct {
asked []string
// wgUp is what `wg show interfaces` answers: the tunnels up on the machine.
wgUp string
// handshakes is what `wg show <interface> latest-handshakes` answers, and handshakesFail the
// error it fails with instead — a machine with no `wg`, say (novox/hq ADR 0119).
handshakes string
handshakesFail error
// units are service units by name, as systemd would report them; volumes are the runtime's
// named volumes.
@@ -101,6 +105,9 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
return m.systemctl(args)
}
if name == "wg" {
if len(args) > 0 && args[len(args)-1] == "latest-handshakes" {
return m.handshakes, m.handshakesFail
}
return m.wgUp, nil
}
if name == "getent" {
internal/apply/opening_test.go
+12
View File
@@ -339,8 +339,18 @@ func TestReturningToAdoptedLoadsTheGuardBeforeRemovingTheFilter(t *testing.T) {
dir := t.TempDir()
guard := filepath.Join(dir, "guard.nft")
guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}`
// The filter's unit file is the mesh's own, written where there was none — which is what makes
// its unit the mesh's to stop, with or without a record of what was found (novox/hq ADR 0118).
units := t.TempDir()
was := unitDir
unitDir = units
t.Cleanup(func() { unitDir = was })
filterUnit := filepath.Join(units, "mesh-filter.service")
for _, stopFails := range []bool{false, true} {
_ = os.Remove(guard)
if err := os.WriteFile(filterUnit, []byte("[Unit]\n"), 0o644); err != nil {
t.Fatal(err)
}
guardUpAtStop := false
run := func(_ context.Context, name string, args ...string) (string, error) {
if name != "systemctl" {
@@ -358,7 +368,9 @@ func TestReturningToAdoptedLoadsTheGuardBeforeRemovingTheFilter(t *testing.T) {
}
return "", nil
}
// As a host recorded them before it kept what it found: no Found on the service.
converged := store.State{Resources: []store.Applied{
{ID: "nftables.unit", Type: "file", Target: filterUnit, Origin: store.OriginDeclared},
{ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}}
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run)
if !guardUpAtStop {
internal/apply/plan.go
+80 -7
View File
@@ -19,7 +19,7 @@ import (
// Step is one thing an apply would do to this machine.
type Step struct {
// Verb is create · update · check · hold · run · remove · forget · disable · enable.
// Verb is create · update · check · hold · run · remove · forget · restore · disable · enable.
Verb string `json:"verb"`
Type string `json:"type,omitempty"`
ID string `json:"id,omitempty"`
@@ -56,6 +56,12 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step {
for _, r := range d.Resources {
declared[r.Identity()] = true
}
// The found tunnel's configuration is held under an id of its own, declared for as long as the
// service taking it over is — as ApplyKeeping counts it, or a plan would forget a hold the
// apply keeps (novox/hq ADR 0105).
if svc := takesOver(d); svc != nil {
declared[takeOverID(svc)] = true
}
rec := known.Firewall
ufw := rec != nil && rec.Kind == string(firewall.UFW)
@@ -77,9 +83,42 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step {
}
var protecting, orphans []Step
made := meshMadeUnits(known)
for _, orphan := range known.Orphans(declared, origin) {
step := Step{Verb: "remove", Type: orphan.Type, ID: orphan.ID, Target: orphan.Target,
Why: "recorded here and no longer declared"}
switch {
case orphan.Stateless:
step.Verb, step.Why = "forget", "no longer declared; its unit's state was never the mesh's and is left as it is"
case orphan.Type == string(declaration.TypeService):
// What removal will do, said before it does it (novox/hq ADR 0118), in removeService's
// words. "restore" only where it may stop or disable something — the record cannot say
// whether the unit is still as the mesh left it, so "may" is as far as a preview goes —
// and a unit whose file the mesh wrote is named as the mesh's, since that one is
// stopped whatever was found.
f := orphan.Found
switch {
case made[orphan.Target]:
step.Verb, step.Why = "remove", "no longer declared; the mesh wrote its unit file, so it is "+
"stopped and disabled at boot before that file goes"
case f == nil:
step.Verb, step.Why = "forget", "no longer declared; recorded before the host kept what it "+
"found, so it is left as it is"
case f.State == "stopped" || f.Boot == "disabled":
var back []string
if f.State == "stopped" {
back = append(back, "stopped")
}
if f.Boot == "disabled" {
back = append(back, "disabled at boot")
}
step.Verb, step.Why = "restore", "no longer declared; the host found it "+
strings.Join(back, " and ")+", and it goes back to that if the mesh changed it"
default:
step.Verb, step.Why = "forget", "no longer declared; it was running before the mesh and is "+
"left as it is — nothing is started or stopped on the way out"
}
}
if d.Adoption == nil && strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) {
step.Why = "what protected this node while adopted; removed last, once everything else applied"
protecting = append(protecting, step)
@@ -103,7 +142,12 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step {
}
steps = append(steps, orphans...)
for _, r := range rest {
steps = append(steps, planned(r, d, known))
step := planned(r, d, known)
if svc, ok := r.(*declaration.Service); ok && svc.TakesOver != nil && d.Adoption != nil && step.Verb != "hold" {
// The take comes before the service that replaces the tunnel, as it does in the apply.
steps = append(steps, plannedTake(svc, known))
}
steps = append(steps, step)
}
// Only a declaration from the mesh converges a node; a bundle or a file never retires the
@@ -139,11 +183,9 @@ func planned(r declaration.Resource, d *declaration.Declaration, known store.Sta
}
}
}
// A file written into replaces nothing that was found, so it is never held (ADR 0102).
into := false
if f, ok := r.(*declaration.File); ok && f.Into != "" {
into = true
}
// A file written into, or a service whose lifecycle is the machine's, replaces nothing that
// was found, so it is never held (ADR 0102, ADR 0117).
into := replacesNothing(r)
h, held := known.HeldAt(r.Identity())
module, untaken := d.Adoption.UntakenModuleOf(r.Identity())
switch {
@@ -182,6 +224,12 @@ func planned(r declaration.Resource, d *declaration.Declaration, known store.Sta
return step
}
if svc, ok := r.(*declaration.Service); ok && svc.Stateless() {
// Nothing is created: the unit and whether it runs are the machine's (novox/hq ADR 0117).
step.Verb, step.Why = "check", "its lifecycle is the machine's; reloaded or restarted only if "+
"running when what it reflects changes"
return step
}
was, recorded := known.Find(r.Identity())
if !recorded {
step.Verb, step.Why = "create", "no record of it on this node"
@@ -202,6 +250,31 @@ func planned(r declaration.Resource, d *declaration.Declaration, known store.Sta
return step
}
// plannedTake is what the take of a found tunnel would do to its configuration (novox/hq ADR 0105,
// ADR 0119): kept as found while the take is not proven, and retired — removed from where its unit
// reads it, its original staying kept — by the first apply that finds the mesh's interface up in
// its place with a peer handshaken. Whether that is this apply is read from the machine, which a
// plan does not do, so it says when rather than whether. One the mesh retired already is said as
// retired: nothing brings it back.
func plannedTake(svc *declaration.Service, known store.State) Step {
t := svc.TakesOver
step := Step{Verb: "hold", Type: string(declaration.TypeFile), ID: takeOverID(svc), Target: t.Config}
if r, ok := known.RetiredAt(t.Config); ok {
step.Verb = "check"
step.Why = "retired once the take of " + t.Interface + " was proven; its original stays at " + r.Kept +
" and the mesh never brings it back"
return step
}
step.Why = "the configuration of the tunnel " + t.Interface + ", kept as found while " + svc.Unit +
" takes it over (" + t.Unit + " stopped and disabled, never flushed); retired — removed from " +
t.Config + ", its original staying kept — once the take is proven by a peer handshaking on " +
strings.TrimPrefix(svc.Unit, "wg-quick@")
if h, ok := known.HeldAt(takeOverID(svc)); ok && h.Kept != "" {
step.Why += "; the original is at " + h.Kept
}
return step
}
// readsChanged is which of the files a container was created reading the apply will hand it
// changed — the same comparison applyContainer makes (novox/hq 04-ISSUES/103), settled from the
// declaration and the record alone.
internal/apply/plan_test.go
+39
View File
@@ -261,3 +261,42 @@ func TestAPlanSaysAContainerIsRecreatedWhenAFileItReadsChanged(t *testing.T) {
t.Errorf("a container with no record of what it read is planned as %q", got)
}
}
func TestAPlanSaysWhichUnitsAnUndeclareGivesBackAndWhichItLeaves(t *testing.T) {
// novox/hq ADR 0118, said before it is done: "restore" only where removal may stop or disable
// something, and a unit whose file the mesh wrote named as the mesh's.
known := store.State{}
known.Record(store.Applied{ID: "guard-unit", Type: "file", Target: "/etc/systemd/system/mesh-guard.service"})
known.Record(store.Applied{ID: "guard", Type: "service", Target: "mesh-guard.service"})
known.Record(store.Applied{ID: "filter", Type: "service", Target: "filter.service",
Found: &store.FoundUnit{State: "stopped"}})
known.Record(store.Applied{ID: "boot", Type: "service", Target: "boot.service",
Found: &store.FoundUnit{State: "running", Boot: "disabled"}})
known.Record(store.Applied{ID: "runtime", Type: "service", Target: "docker.service",
Found: &store.FoundUnit{State: "running", Boot: "enabled"}})
known.Record(store.Applied{ID: "old", Type: "service", Target: "sshd.service"})
known.Record(store.Applied{ID: "nm", Type: "service", Target: "NetworkManager.service", Stateless: true})
steps := Plan(parse(t, nothingButA(t)), known, store.OriginCarried)
got := verbs(steps)
want := "forget nm, forget old, forget runtime, restore boot, restore filter, remove guard, remove guard-unit, create other"
if got != want {
t.Fatalf("planned %s\nwant %s", got, want)
}
for _, s := range steps {
switch s.ID {
case "guard":
if !strings.Contains(s.Why, "unit file") {
t.Errorf("the mesh's own unit was not named as the mesh's: %q", s.Why)
}
case "filter":
if !strings.Contains(s.Why, "found it stopped") {
t.Errorf("what the unit goes back to went unsaid: %q", s.Why)
}
case "old":
if !strings.Contains(s.Why, "left as it is") {
t.Errorf("a unit with nothing found was not said to be left: %q", s.Why)
}
}
}
}
internal/apply/process.go
+52 -1
View File
@@ -30,7 +30,7 @@ import (
// Under the mesh's own directory rather than somewhere a distribution owns: these are files the
// mesh puts there and replaces, and putting them where a package manager also writes is how two
// owners end up disagreeing about one path.
const daemonRoot = "/var/lib/mesh/daemons"
var daemonRoot = "/var/lib/mesh/daemons"
// unitDir is where the mesh writes the units it owns. A variable only so a test can point it at a
// directory of its own.
@@ -290,3 +290,54 @@ func unitValue(key, value string) string {
).Replace(value)
return `"` + key + "=" + escaped + `"`
}
// removeProcess takes away a process the mesh no longer declares: its timer and unit stopped and
// disabled, their files removed, the supervisor told, and the unpacked bundle deleted.
//
// **All of it is the host's**, which is why all of it goes (novox/hq ADR 0118). Before this there
// was no way to remove a process at all, and one left undeclared failed every apply on its node
// until someone edited the host's state by hand — unassigning any module that ran its own code
// stranded the machine.
//
// Idempotent, like every removal: a unit already gone is not an error, and a record whose files
// have all vanished is forgotten rather than reported as removed.
func removeProcess(ctx context.Context, a store.Applied, run Runner) (string, string, error) {
name := a.Target
if problem := declaration.ProcessNameProblem(name); problem != "" {
// The name is a unit name and a directory under the mesh's own, and what goes is that
// directory, whole. One that could climb out of either — ".." is the mesh's own directory's
// parent — is refused rather than acted on, whatever wrote it into the record.
return "", "", fmt.Errorf("a process recorded under %q cannot be removed by name: %s", name, problem)
}
found := false
for _, unit := range []string{name + ".timer", name + ".service"} {
path := filepath.Join(unitDir, unit)
if _, err := os.Stat(path); err != nil {
continue
}
found = true
// The timer first, so a scheduled run cannot start the service between the two.
if _, err := run(ctx, "systemctl", "disable", "--now", unit); err != nil {
return "", "", fmt.Errorf("stopping %s: %w", unit, err)
}
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return "", "", err
}
}
if found {
if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil {
return "", "", err
}
}
bundle := filepath.Join(daemonRoot, name)
if _, err := os.Stat(bundle); err == nil {
found = true
if err := os.RemoveAll(bundle); err != nil {
return "", "", err
}
}
if !found {
return "forgotten", "no longer there", nil
}
return "removed", "stopped; its unit and its bundle removed — the mesh's own code", nil
}
internal/apply/process_test.go
+93
View File
@@ -1,10 +1,14 @@
package apply
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
func aProcess() *declaration.Process {
@@ -163,3 +167,92 @@ func TestAnEnvironmentValueMeansWhatTheDeclarationSaid(t *testing.T) {
t.Fatalf("a quote was not escaped, so the value ends early: %s", line)
}
}
func TestAnUndeclaredProcessIsRemovedWithItsUnitAndBundle(t *testing.T) {
// novox/hq ADR 0118: a process's unit and bundle are the host's own, so they go with the
// declaration. Before, there was no way to remove a process at all, and one left undeclared
// failed every apply on its node.
units, bundles := t.TempDir(), t.TempDir()
wasUnits, wasBundles := unitDir, daemonRoot
unitDir, daemonRoot = units, bundles
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
for _, f := range []string{"mesh-job.service", "mesh-job.timer"} {
if err := os.WriteFile(filepath.Join(units, f), []byte("[Unit]\n"), 0o644); err != nil {
t.Fatal(err)
}
}
if err := os.MkdirAll(filepath.Join(bundles, "mesh-job", "bin"), 0o755); err != nil {
t.Fatal(err)
}
var commands []string
run := func(ctx context.Context, name string, args ...string) (string, error) {
commands = append(commands, strings.Join(args, " "))
return "", nil
}
known := store.State{Resources: []store.Applied{{ID: "p", Type: "process", Target: "mesh-job"}}}
d := parse(t, `{"declaration":1,"resources":[
{"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
]}`)
report, after, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
if err != nil {
t.Fatalf("an undeclared process failed the apply: %v", err)
}
joined := strings.Join(commands, "; ")
timer, service := strings.Index(joined, "disable --now mesh-job.timer"), strings.Index(joined, "disable --now mesh-job.service")
if timer < 0 || service < 0 || timer > service {
t.Errorf("the timer and the unit were not stopped, timer first: %s", joined)
}
if !strings.Contains(joined, "daemon-reload") {
t.Errorf("the service manager was not told its units changed: %s", joined)
}
for _, gone := range []string{filepath.Join(units, "mesh-job.service"), filepath.Join(units, "mesh-job.timer"), filepath.Join(bundles, "mesh-job")} {
if _, err := os.Stat(gone); !os.IsNotExist(err) {
t.Errorf("%s is still there", gone)
}
}
if o := outcomeOf(report, "p"); o.Action != "removed" {
t.Errorf("outcome %+v, want removed", o)
}
if _, still := after.Find("p"); still {
t.Error("the host still believes it owns the process")
}
// Again, with everything already gone: forgotten, not an error.
_, _, err = Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
if err != nil {
t.Errorf("removing a process that is already gone failed: %v", err)
}
}
func TestAProcessRecordedUnderAPathlikeNameIsRefusedNotRemoved(t *testing.T) {
// filepath.Join(daemonRoot, "..") is the mesh's own directory, and removal deletes what that
// names, whole. A record is what some host wrote, perhaps under looser rules than today's, so
// the removal holds the name to the declaration's rule again (novox/hq ADR 0118).
root := t.TempDir()
bundles := filepath.Join(root, "daemons")
wasUnits, wasBundles := unitDir, daemonRoot
unitDir, daemonRoot = t.TempDir(), bundles
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
precious := filepath.Join(root, "state.json")
if err := os.MkdirAll(filepath.Join(bundles, "other"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(precious, []byte("{}"), 0o600); err != nil {
t.Fatal(err)
}
for _, name := range []string{"..", ".", "", "-x"} {
known := store.State{Resources: []store.Applied{{ID: "p", Type: "process", Target: name}}}
d := parse(t, `{"declaration":1,"resources":[
{"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
]}`)
if _, _, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, noServices, nil, nil); err == nil {
t.Errorf("a process recorded as %q was removed by name", name)
}
for _, still := range []string{precious, filepath.Join(bundles, "other")} {
if _, err := os.Stat(still); err != nil {
t.Fatalf("removing a process recorded as %q took %s with it", name, still)
}
}
}
}
internal/apply/stateless_test.go
+191
View File
@@ -0,0 +1,191 @@
package apply
import (
"context"
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0117: a service that omits its state leaves the unit's lifecycle to the
// machine. The mesh reflects its triggers on a unit already running and does nothing else to it —
// never starts, stops, enables or disables it, and forgets it when undeclared.
// unitIn is a service manager whose one unit is active or not, recording what it is asked.
func unitIn(active bool, commands *[]string) Runner {
return func(_ context.Context, name string, args ...string) (string, error) {
line := name + " " + strings.Join(args, " ")
*commands = append(*commands, line)
switch {
case strings.Contains(line, "is-enabled"):
return "enabled", nil
case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
if active {
return "LoadState=loaded\nActiveState=active\nSubState=running", nil
}
return "LoadState=loaded\nActiveState=inactive\nSubState=dead", nil
}
return "", nil
}
}
func statelessDecl(path, content, triggers string) string {
return fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"uplink.conf","type":"file","path":%q,"into":"block","content":%q},
{"id":"uplink.manager","type":"service","unit":"NetworkManager.service",%s}
]}`, path, content, triggers)
}
// touched is whether any command would change the unit's lifecycle.
func touched(commands []string) []string {
var changing []string
for _, c := range commands {
for _, verb := range []string{" start ", " stop ", " restart ", " enable ", " disable ", " reload "} {
if strings.Contains(c+" ", verb) {
changing = append(changing, c)
}
}
}
return changing
}
func TestAStatelessServiceRunningIsReloadedForItsTrigger(t *testing.T) {
path := filepath.Join(t.TempDir(), "mesh.conf")
var commands []string
report, _, err := Apply(context.Background(), archHost(t),
parse(t, statelessDecl(path, "[main]\ndns=none\n", `"reload-on":["uplink.conf"]`)),
store.State{}, store.OriginDeclared, unitIn(true, &commands), nil, nil)
if err != nil {
t.Fatal(err)
}
joined := strings.Join(commands, "\n")
if !strings.Contains(joined, "systemctl reload NetworkManager.service") {
t.Errorf("the running manager was not reloaded; commands were %v", commands)
}
for _, c := range touched(commands) {
if !strings.Contains(c, "reload") {
t.Errorf("the manager's lifecycle was touched: %s", c)
}
}
if o := outcomeOf(report, "uplink.manager"); o.Action != "updated" || !strings.Contains(o.Detail, "reloaded for uplink.conf") {
t.Errorf("reported as %q: %s", o.Action, o.Detail)
}
}
func TestAStatelessServiceNotRunningIsLeftSo(t *testing.T) {
path := filepath.Join(t.TempDir(), "mesh.conf")
for _, triggers := range []string{`"reload-on":["uplink.conf"]`, `"restart-on":["uplink.conf"]`} {
var commands []string
report, _, err := Apply(context.Background(), archHost(t),
parse(t, statelessDecl(path, fmt.Sprintf("# %s\n", triggers), triggers)),
store.State{}, store.OriginDeclared, unitIn(false, &commands), nil, nil)
if err != nil {
t.Fatal(err)
}
if changing := touched(commands); len(changing) > 0 {
t.Errorf("%s: an inactive unit was acted on: %v", triggers, changing)
}
if o := outcomeOf(report, "uplink.manager"); o.Action != "unchanged" ||
o.Detail != "not running; the change applies at its next start" {
t.Errorf("%s: reported as %q: %s", triggers, o.Action, o.Detail)
}
}
}
func TestAStatelessServiceIsRestartedOnlyForItsRestartTrigger(t *testing.T) {
path := filepath.Join(t.TempDir(), "mesh.conf")
var commands []string
d := parse(t, statelessDecl(path, "x\n", `"restart-on":["uplink.conf"]`))
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
unitIn(true, &commands), nil, nil)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(strings.Join(commands, "\n"), "stop NetworkManager.service") {
t.Errorf("not restarted for its restart trigger; commands were %v", commands)
}
// Nothing it reflects changed: nothing is asked of the unit at all.
commands = nil
report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared,
unitIn(true, &commands), nil, nil)
if err != nil {
t.Fatal(err)
}
if changing := touched(commands); len(changing) > 0 {
t.Errorf("with nothing changed the unit was acted on: %v", changing)
}
if got := outcomeOf(report, "uplink.manager").Action; got != "unchanged" {
t.Errorf("with nothing changed it was %q", got)
}
}
func TestAStatelessServiceUndeclaredIsForgottenNotStopped(t *testing.T) {
path := filepath.Join(t.TempDir(), "mesh.conf")
var commands []string
_, state, err := Apply(context.Background(), archHost(t),
parse(t, statelessDecl(path, "x\n", `"reload-on":["uplink.conf"]`)),
store.State{}, store.OriginDeclared, unitIn(true, &commands), nil, nil)
if err != nil {
t.Fatal(err)
}
if rec, _ := state.Find("uplink.manager"); !rec.Stateless {
t.Fatal("the record does not say the service was stateless")
}
if steps := Plan(somethingElse(t), state, store.OriginDeclared); !hasStep(steps, "forget", "uplink.manager") {
t.Errorf("the preview does not forget it: %v", steps)
}
commands = nil
report, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared,
unitIn(true, &commands), nil, nil)
if err != nil {
t.Fatal(err)
}
if changing := touched(commands); len(changing) > 0 {
t.Errorf("undeclaring acted on the unit: %v", changing)
}
o := outcomeOf(report, "uplink.manager")
if o.Action != "forgotten" || o.Detail != "its state was never the mesh's" {
t.Errorf("undeclaring was %q: %s", o.Action, o.Detail)
}
}
func TestAStatelessServiceIsNeverHeldOnAnAdoptedNode(t *testing.T) {
path := filepath.Join(t.TempDir(), "mesh.conf")
resources := fmt.Sprintf(`{"id":"uplink.conf","type":"file","path":%q,"into":"block","content":"x\n"},
{"id":"uplink.manager","type":"service","unit":"NetworkManager.service","reload-on":["uplink.conf"]}`, path)
d := adopted(t, `{"taken":[],"untaken":{"uplink":["uplink.conf","uplink.manager"]}}`, resources)
for _, s := range Plan(d, store.State{}, store.OriginDeclared) {
if s.ID == "uplink.manager" && (s.Verb == "hold" || s.Verb == "create" || strings.Contains(s.Why, "replaces")) {
t.Errorf("the preview holds or replaces a stateless service: %+v", s)
}
}
var commands []string
report, state, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{},
store.OriginDeclared, unitIn(true, &commands), nil, nil, KeepIn(t.TempDir()))
if err != nil {
t.Fatal(err)
}
if got := outcomeOf(report, "uplink.manager").Action; got == "held" {
t.Fatal("a stateless service was held, though it replaces nothing that was found")
}
if len(state.Held) != 0 {
t.Errorf("something was held: %+v", state.Held)
}
for _, c := range touched(commands) {
if !strings.Contains(c, "reload") {
t.Errorf("the adopted node's manager lifecycle was touched: %s", c)
}
}
}
func hasStep(steps []Step, verb, id string) bool {
for _, s := range steps {
if s.Verb == verb && s.ID == id {
return true
}
}
return false
}
internal/apply/takeover.go
+231 -3
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
@@ -27,6 +28,16 @@ import (
// Every apply, not once: a found unit somebody starts again would take the port back from the
// mesh's interface, so it is stopped again and said so. That is the one place an adopted node
// undoes something done by hand, and it is because the tunnel is the mesh's now.
//
// **Until the take is proven, and then the found configuration is retired** (novox/hq ADR 0119).
// Keeping it on disk was the caution the take needed: if the mesh's interface does not come up,
// the found unit is started again and the peers never notice. That caution is spent once the
// tunnel is taken — the found unit down and disabled, the mesh's interface up — and a peer has
// handshaken with the mesh's interface. From then on a configuration nothing maintains, one
// command away from raising a second way onto the network, is not a rollback path but a door
// nobody watches. So it is removed from where its unit reads it; its original, kept before
// anything happened to it (ADR 0100), stays kept; and the hold on it ends. A take never proven
// keeps it, and says so — a broken take is visible, not silently retired.
// TakenTunnel is what an apply says about a tunnel it took over, for the node's report.
type TakenTunnel struct {
@@ -36,7 +47,9 @@ type TakenTunnel struct {
Peers int
// State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one
// down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's
// not up: the peers reach nothing). Note is what this apply did about it.
// not up: the peers reach nothing). Note is what this apply did about it — and, for a taken
// tunnel, whether the take is proven and its found configuration retired (novox/hq ADR 0119).
// Kept is where the found configuration's original is, retired or not.
State string
Note string
Kept string
@@ -84,14 +97,53 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service,
// 1. The configuration, kept like any held file. A synthetic file resource stands for it, so
// the same code keeps its original, digests it and notices it changing.
//
// **Unless it was retired** (novox/hq ADR 0119): the take was proven and the mesh removed
// it, so there is nothing to hold and nothing missing — only where its original is, which
// the retirement recorded. A hold still standing is let go: that is what retiring it meant.
//
// **One that comes back is held again on its FIRST original** — the one kept before anything
// happened to it (ADR 0100), never whatever was put back — and retired again by the first
// apply that finds the take still proven, keeping what came back only if it differs from
// what is already kept. Put back by hand while the private network is assigned, it is not a
// rollback: that means unassigning the private network first, and the note says so.
file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config}
var held store.Held
retired, wasRetired := known.RetiredAt(t.Config)
cameBack := wasRetired && present(t.Config)
switch {
case wasRetired && !cameBack:
known.Release(id)
held = store.Held{Kept: retired.Kept}
out = begin(file)
out.Action = "unchanged"
facts.Note = "the found configuration " + t.Config + " was retired once the take was proven; " +
"its original is kept at " + retired.Kept + " and the mesh never brings it back"
default:
was, already := known.HeldAt(id)
out, held, err := hold(ctx, sys, file, module, was, already,
"the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now)
why := "the configuration of the tunnel " + t.Interface + ", taken over by " + svc.Unit
if cameBack && !already {
digest := retired.Digest
if digest == "" {
if raw, err := os.ReadFile(retired.Kept); err == nil {
digest = digestOf(string(raw))
}
}
was = store.Held{ID: id, Module: module, Kind: string(declaration.TypeFile), Target: t.Config,
Since: now, Why: why, Kept: retired.Kept, Digest: digest}
already = true
}
out, held, err = hold(ctx, sys, file, module, was, already, why, run, keep, now)
if err != nil {
return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err)
}
known.RecordHeld(held)
if cameBack {
facts.Note = "the found configuration " + t.Config + " came back after it was retired; while the " +
"private network is assigned the mesh retires it again, so rolling back to the found tunnel " +
"means unassigning the private network first"
}
}
facts.Kept = held.Kept
// What the file says, for the report: from the machine, or from the kept original when the
// machine's copy is gone. The private key stays in the file; nothing here keeps it.
@@ -185,6 +237,13 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service,
}
out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found"
switch {
case cameBack:
out.Detail = "the tunnel " + t.Interface + "'s configuration, back after it was retired; held until " +
"it is retired again"
case wasRetired:
out.Detail = "the tunnel " + t.Interface + "'s configuration, retired once the take was proven"
}
if held.Kept != "" {
out.Detail += " (original at " + held.Kept + ")"
}
@@ -335,6 +394,175 @@ func restoreFound(ctx context.Context, sys system.System, unit string, run Runne
facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had"
}
// wireguardDir is where a found tunnel's configuration may be retired from: wg-quick's own, and
// nowhere else. A variable so a test can hand in a directory.
var wireguardDir = tunnel.ConfigDir
// retireFound removes the found tunnel's configuration from where its unit reads it, once the take
// is proven, and ends the hold on it (novox/hq ADR 0119). Asked after the mesh's service applied
// and the tunnel reads as taken; retired says whether this apply retired it, and out is then what
// replaces the take's outcome for the configuration.
//
// **Proven is taken and a handshake.** Taken alone — the found unit down and disabled, the mesh's
// interface up — says the mesh's interface exists, not that any peer reaches it: an interface up
// with the wrong key is taken and carries nothing. A peer that has completed a handshake with it
// has checked its key, so that is the proof, asked of the kernel through `wg`. Any handshake counts,
// however old: a change to the mesh's configuration restarts its unit, which recreates the
// interface and resets its counters, so a time that is there at all was made by this interface.
// Anything short of one — no peer yet, every time zero, `wg` missing or failing — keeps the file,
// and the account says which: a take that never proves itself is visible rather than silently
// retired.
//
// **Only what the take names, and only wg-quick's own file.** Nothing is removed unless the path
// is exactly `<wireguard dir>/<found interface>.conf`, is not a path the mesh itself writes, and is
// a file rather than a link: removing a link would leave the key-bearing file it points at where it
// is, a retirement in name only, so that one is said and left to a person.
//
// **The original must still be kept.** It is the record of what the predecessor was and a
// person's only way back (ADR 0100); a kept copy that has gone missing is said, and the file is
// not removed, since removing it then would lose the only copy. What is on disk now, if it differs
// from the first original and from what was kept at the last retirement, is kept too before it
// goes — by content, so the first original is never overwritten and a file that keeps coming back
// the same keeps nothing more.
//
// The found unit is left disabled; without its configuration it cannot raise the interface, so
// every later apply's check of it finds nothing to do. Nothing here ever writes the file back.
func retireFound(ctx context.Context, svc *declaration.Service, d *declaration.Declaration, known *store.State,
run Runner, keep Keep, facts *TakenTunnel, now time.Time) (out Outcome, retired bool) {
t := svc.TakesOver
id := takeOverID(svc)
if facts.State != Taken {
return out, false
}
held, isHeld := known.HeldAt(id)
if !isHeld {
// Retired already (takeOver let any hold go and said so), or never held: nothing to do.
return out, false
}
say := func(note string) {
if facts.Note != "" {
facts.Note += "; "
}
facts.Note += note
}
notRetired := func(why string) (Outcome, bool) {
say("the found configuration " + t.Config + " is not retired: " + why)
return Outcome{}, false
}
mesh := strings.TrimPrefix(svc.Unit, "wg-quick@")
peers, err := tunnel.Handshaken(ctx, tunnel.Runner(run), mesh)
if err != nil {
say("taken, not yet proven: " + err.Error() + "; the found configuration " + t.Config + " is kept")
return out, false
}
if peers == 0 {
say("taken, not yet proven: no peer has handshaken on " + mesh + "; the found configuration " +
t.Config + " is kept")
return out, false
}
proven := fmt.Sprintf("proven: %d peer(s) handshaken on %s", peers, mesh)
say(proven)
// What may be removed at all.
if want := filepath.Join(wireguardDir, t.Interface+".conf"); t.Config != want {
return notRetired("only " + want + ", the found interface's own wg-quick configuration, is ever " +
"retired by the mesh, and the take names " + t.Config)
}
if known.Recorded(string(declaration.TypeFile), t.Config) || declaresFile(d, t.Config) {
return notRetired("it is a path the mesh itself writes")
}
if info, err := os.Lstat(t.Config); err == nil && info.Mode()&os.ModeSymlink != 0 {
target, _ := os.Readlink(t.Config)
return notRetired("it is a link to " + target + "; removing the link would leave the key-bearing file " +
"it points at, so it must be retired by hand — both are kept")
}
// The kept original, read back — not just named in a record.
if held.Kept == "" {
return notRetired("no original of it was kept, so removing it would leave no record of what the " +
"predecessor was")
}
original, err := os.ReadFile(held.Kept)
if err != nil || (held.Digest != "" && digestOf(string(original)) != held.Digest) {
why := "is missing"
if err == nil {
why = "no longer holds what was found"
} else if !errors.Is(err, os.ErrNotExist) {
why = "cannot be read (" + err.Error() + ")"
}
return notRetired("its kept original " + held.Kept + " " + why + ", so removing it would lose the only copy")
}
before, cameBack := known.RetiredAt(t.Config)
record := store.Retired{ID: id, Path: t.Config, Kept: held.Kept, Digest: digestOf(string(original)), At: now}
if cameBack {
// The first original stays the record's, and so does what the last retirement kept.
record.Extra, record.ExtraDigest, record.Again = before.Extra, before.ExtraDigest, before.Again+1
}
newCopy := ""
gone := !present(t.Config)
if !gone {
current, err := os.ReadFile(t.Config)
if err != nil {
return notRetired("it cannot be read (" + err.Error() + ")")
}
if sum := digestOf(string(current)); sum != record.Digest && sum != record.ExtraDigest {
if keep == nil {
return notRetired("it holds something other than its kept original and this host has nowhere " +
"to keep it")
}
where, err := keep(t.Config, current, 0o600)
if err != nil {
return notRetired("keeping what it holds now failed (" + err.Error() + ")")
}
record.Extra, record.ExtraDigest, newCopy = where, sum, where
}
if err := os.Remove(t.Config); err != nil && !errors.Is(err, os.ErrNotExist) {
return notRetired("removing it failed (" + err.Error() + ")")
}
if present(t.Config) {
return notRetired("it is still there after it was removed")
}
}
known.RecordRetired(record)
known.Release(id)
facts.Kept = held.Kept
copied := ""
if newCopy != "" {
copied = "; what it held, which differed from the original, is kept at " + newCopy
}
out = Outcome{ID: id, Type: string(declaration.TypeFile), Target: t.Config, Action: "removed"}
switch {
case cameBack:
say("the found configuration came back and was retired again — its original still kept at " +
held.Kept + copied + "; rolling back to the found tunnel means unassigning the private network first")
out.Detail = "the found configuration came back and was retired again; original kept at " + held.Kept + copied
default:
say("the found configuration " + t.Config + " is retired — its original kept at " + held.Kept + copied +
", " + t.Unit + " left disabled, and the mesh never brings it back")
out.Detail = "retired: the take of " + t.Interface + " is " + proven + "; original kept at " + held.Kept + copied
}
if gone {
// Already gone — removed by something other than the mesh, or by an apply whose record was
// never saved. Nothing removed here; the hold ends all the same.
out.Action = "unchanged"
out.Detail = "retired: the take of " + t.Interface + " is " + proven + " and " + t.Config +
" was already gone; original kept at " + held.Kept
}
return out, true
}
// declaresFile is whether a declaration writes a file at a path.
func declaresFile(d *declaration.Declaration, path string) bool {
for _, r := range d.Resources {
if f, ok := r.(*declaration.File); ok && filepath.Clean(f.Path) == filepath.Clean(path) {
return true
}
}
return false
}
// takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since
// a machine has one private network.
func takesOver(d *declaration.Declaration) *declaration.Service {
internal/apply/takeover_test.go
+460 -1
View File
@@ -4,6 +4,7 @@ import (
"crypto/ecdh"
"crypto/rand"
"encoding/base64"
"errors"
"os"
"path/filepath"
"strings"
@@ -65,6 +66,10 @@ func aHubInUse(t *testing.T) (dir, config, mesh, keyFile string, m *machine) {
"wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"},
}}
takeoverRecheck = 0
// The found configuration lives in this test's own wireguard directory (novox/hq ADR 0119).
was := wireguardDir
wireguardDir = dir
t.Cleanup(func() { wireguardDir = was })
return dir, config, mesh, keyFile, m
}
@@ -77,7 +82,10 @@ func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T
t.Fatalf("the found unit was not stopped and disabled: %+v", u)
}
for _, asked := range m.asked {
if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") {
// Only ever asked about: which interfaces are up, and whether a peer has handshaken with
// the mesh's own (novox/hq ADR 0119).
if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") &&
asked != "wg show mesh0 latest-handshakes" {
t.Errorf("the found interface was touched with %q; it is stopped, never flushed", asked)
}
if strings.HasPrefix(asked, "wg-quick") || strings.Contains(asked, "peer remove") {
@@ -254,3 +262,454 @@ func TestATakeoverIsRefusedOnAConvergedDeclaration(t *testing.T) {
t.Fatalf("a takeover on a converged node was accepted: %v", err)
}
}
// novox/hq ADR 0119: once the take is proven — taken, and a peer handshaken on the mesh's
// interface — the found configuration is removed from where its unit reads it, its original stays
// kept and the hold on it ends. Never before, and never brought back.
const takesOverID = "mesh-wireguard.overlay-up.takes-over"
// handshaken is `wg show mesh0 latest-handshakes` with one of the two peers through.
const handshaken = "PEER-A=\t1790000000\nPEER-B=\t0\n"
func TestAProvenTakeRetiresTheFoundConfiguration(t *testing.T) {
dir, config, mesh, keyFile, m := aHubInUse(t)
m.handshakes = handshaken
report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)
if _, err := os.Lstat(config); !os.IsNotExist(err) {
t.Fatalf("a proven take left the found configuration where its unit reads it: %v", err)
}
retired, ok := state.RetiredAt(config)
if !ok || retired.Kept == "" || retired.ID != takesOverID {
t.Fatalf("the retirement was not recorded: %+v", state.Retired)
}
if kept, _ := os.ReadFile(retired.Kept); string(kept) != foundConf {
t.Fatalf("the kept original did not survive the retirement: %q", kept)
}
if _, held := state.HeldAt(takesOverID); held {
t.Error("the hold on the found configuration did not end with its retirement")
}
if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" {
t.Errorf("the found unit is not left down and disabled: %+v", u)
}
if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Kept != retired.Kept ||
!strings.Contains(report.Tunnel.Note, "proven: 1 peer(s) handshaken on mesh0") ||
!strings.Contains(report.Tunnel.Note, "is retired") {
t.Fatalf("the account does not say the take is proven and the configuration retired: %+v", report.Tunnel)
}
if o := outcomeOf(report, takesOverID); o.Action != "removed" || !strings.Contains(o.Detail, "retired") {
t.Errorf("the retirement is not what the apply says it did to the file: %+v", o)
}
// And the account still carries what was found, read from the kept original.
if report.Tunnel.Port != 51900 || report.Tunnel.Peers != 2 {
t.Errorf("the account lost what the tunnel was: %+v", report.Tunnel)
}
}
func TestATakeNotProvenKeepsTheFoundConfigurationAndSaysSo(t *testing.T) {
cases := map[string]struct {
handshakes string
fail error
says string
}{
"no peer at all": {"", nil, "no peer has handshaken on mesh0"},
"every handshake at zero": {"PEER-A=\t0\nPEER-B=\t0\n", nil, "no peer has handshaken on mesh0"},
"wg is not there": {"", errors.New(`exec: "wg": executable file not found in $PATH`), "executable file not found"},
"the answer is nonsense": {"unable to access interface\n", nil, "not a peer and a time"},
}
for name, c := range cases {
dir, config, mesh, keyFile, m := aHubInUse(t)
m.handshakes, m.handshakesFail = c.handshakes, c.fail
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
report, state := applyAdopted(t, d, store.State{}, m, dir)
if got, _ := os.ReadFile(config); string(got) != foundConf {
t.Fatalf("%s: a take not proven lost the found configuration", name)
}
if _, held := state.HeldAt(takesOverID); !held {
t.Errorf("%s: the hold ended although the take is not proven", name)
}
if _, retired := state.RetiredAt(config); retired {
t.Errorf("%s: recorded as retired", name)
}
if report.Tunnel == nil || report.Tunnel.State != Taken ||
!strings.Contains(report.Tunnel.Note, "taken, not yet proven") ||
!strings.Contains(report.Tunnel.Note, c.says) || !strings.Contains(report.Tunnel.Note, "is kept") {
t.Errorf("%s: the account does not say the take is not proven and why: %+v", name, report.Tunnel)
}
// A later apply that finds a peer through retires it: the take itself need not be the one.
m.handshakes, m.handshakesFail = handshaken, nil
_, state = applyAdopted(t, d, state, m, dir)
if _, err := os.Lstat(config); !os.IsNotExist(err) {
t.Errorf("%s: the apply after the take was proven kept the found configuration", name)
}
if _, retired := state.RetiredAt(config); !retired {
t.Errorf("%s: the later retirement was not recorded", name)
}
}
}
func TestAFoundConfigurationWhoseKeptOriginalIsMissingIsNotRetired(t *testing.T) {
dir, config, mesh, keyFile, m := aHubInUse(t)
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
_, state := applyAdopted(t, d, store.State{}, m, dir)
held, _ := state.HeldAt(takesOverID)
if err := os.Remove(held.Kept); err != nil {
t.Fatal(err)
}
m.handshakes = handshaken
report, state := applyAdopted(t, d, state, m, dir)
if got, _ := os.ReadFile(config); string(got) != foundConf {
t.Fatal("the found configuration was removed with no kept original left of it")
}
if _, still := state.HeldAt(takesOverID); !still {
t.Error("the hold ended although nothing was retired")
}
if _, retired := state.RetiredAt(config); retired {
t.Error("recorded as retired")
}
if report.Tunnel == nil || !strings.Contains(report.Tunnel.Note, "is not retired") ||
!strings.Contains(report.Tunnel.Note, held.Kept+" is missing") {
t.Errorf("the account does not say the kept original is missing: %+v", report.Tunnel)
}
}
func TestAFoundConfigurationRewrittenSinceItWasFoundIsKeptAgainBeforeItGoes(t *testing.T) {
dir, config, mesh, keyFile, m := aHubInUse(t)
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
_, state := applyAdopted(t, d, store.State{}, m, dir)
rewritten := foundConf + "\n[Peer]\nPublicKey = PEER-C=\nAllowedIPs = 192.0.2.4/32\n"
if err := os.WriteFile(config, []byte(rewritten), 0o600); err != nil {
t.Fatal(err)
}
m.handshakes = handshaken
_, state = applyAdopted(t, d, state, m, dir)
if _, err := os.Lstat(config); !os.IsNotExist(err) {
t.Fatal("a proven take kept a rewritten configuration")
}
retired, _ := state.RetiredAt(config)
if first, _ := os.ReadFile(retired.Kept); string(first) != foundConf {
t.Errorf("the first original was overwritten: %q", first)
}
kept, _ := filepath.Glob(filepath.Join(dir, "kept", "*-wg0.conf"))
var found bool
for _, k := range kept {
if got, _ := os.ReadFile(k); string(got) == rewritten {
found = true
}
}
if !found {
t.Errorf("what the file held when it was retired was not kept: %v", kept)
}
}
func TestARetiredTakeIsSteadyAndItsFoundUnitFindsNothingToDo(t *testing.T) {
dir, config, mesh, keyFile, m := aHubInUse(t)
m.handshakes = handshaken
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
_, state := applyAdopted(t, d, store.State{}, m, dir)
retired, _ := state.RetiredAt(config)
// wg-quick@wg0 with no configuration: inactive, and disabled — the check of it every apply
// makes must find nothing to do and fail on nothing.
m.asked = nil
report, again := applyAdopted(t, d, state, m, dir)
if report.Changed() {
t.Errorf("an apply after the retirement moved the machine: %+v", report.Outcomes)
}
if m.did("systemctl stop wg-quick@wg0") || m.did("systemctl start wg-quick@wg0") {
t.Errorf("the retired tunnel's unit was acted on: %v", m.asked)
}
if _, err := os.Lstat(config); !os.IsNotExist(err) {
t.Error("the found configuration came back")
}
if _, held := again.HeldAt(takesOverID); held {
t.Error("a retired configuration is held again")
}
if r, ok := again.RetiredAt(config); !ok || r != retired {
t.Errorf("the retirement was not kept as it was: %+v", again.Retired)
}
if o := outcomeOf(report, takesOverID); o.Action != "unchanged" || !strings.Contains(o.Detail, "retired") {
t.Errorf("the retired configuration is not said as retired: %+v", o)
}
if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Kept != retired.Kept ||
!strings.Contains(report.Tunnel.Note, "retired") || report.Tunnel.Port != 51900 {
t.Errorf("the account of a retired take does not say so: %+v", report.Tunnel)
}
// Enabled at boot again by a person: disabled again, as any take does, and still no error.
m.units["wg-quick@wg0"].enabled = "enabled"
_, _ = applyAdopted(t, d, again, m, dir)
if m.units["wg-quick@wg0"].enabled != "disabled" {
t.Error("the found unit enabled again by hand was left to start at boot")
}
}
func TestUndeclaringThePrivateNetworkAfterRetirementBringsNothingBack(t *testing.T) {
dir, config, mesh, keyFile, m := aHubInUse(t)
m.handshakes = handshaken
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
_, state := applyAdopted(t, d, store.State{}, m, dir)
// The private network unassigned: only something else is declared.
other := adopted(t, `{"taken":[],"untaken":{}}`,
`{"id":"other.file","type":"file","path":"`+filepath.Join(dir, "other.conf")+`","content":"x\n"}`)
m.asked = nil
_, after := applyAdopted(t, other, state, m, dir)
if _, err := os.Lstat(config); !os.IsNotExist(err) {
t.Fatal("undeclaring the private network brought the found configuration back")
}
if m.did("systemctl start wg-quick@wg0") || m.did("systemctl enable wg-quick@wg0") {
t.Errorf("undeclaring the private network started the found tunnel: %v", m.asked)
}
if _, ok := after.RetiredAt(config); !ok {
t.Error("the retirement was forgotten with the private network")
}
// Assigned again, it finds the configuration retired rather than missing, and raises the
// mesh's interface.
report, _ := applyAdopted(t, d, after, m, dir)
if report.Tunnel == nil || report.Tunnel.State != Taken {
t.Errorf("the private network assigned again did not take the tunnel: %+v", report.Tunnel)
}
if _, err := os.Lstat(config); !os.IsNotExist(err) {
t.Error("assigning the private network again brought the found configuration back")
}
}
func TestAPlanSaysTheFoundConfigurationIsRetiredWhenTheTakeIsProven(t *testing.T) {
dir, config, mesh, keyFile, m := aHubInUse(t)
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
plan := Plan(d, store.State{}, store.OriginDeclared)
report, state := applyAdopted(t, d, store.State{}, m, dir)
if got, want := strings.Join(ids(plan), " "), strings.Join(outcomeIDs(report), " "); got != want {
t.Errorf("the plan said %q and the apply did %q", got, want)
}
var take Step
for _, s := range plan {
if s.ID == takesOverID {
take = s
}
}
if take.Verb != "hold" || take.Target != config || !strings.Contains(take.Why, "retired — removed from "+config) ||
!strings.Contains(take.Why, "handshaking on mesh0") {
t.Errorf("the plan does not say the found configuration is retired once proven: %+v", take)
}
// Held and still declared: never planned as forgotten.
if strings.Contains(verbs(Plan(d, state, store.OriginDeclared)), "forget "+takesOverID) {
t.Error("the plan forgets a hold the apply keeps")
}
m.handshakes = handshaken
_, state = applyAdopted(t, d, state, m, dir)
for _, s := range Plan(d, state, store.OriginDeclared) {
if s.ID == takesOverID && (s.Verb != "check" || !strings.Contains(s.Why, "retired once the take")) {
t.Errorf("a retired configuration is planned as %+v", s)
}
}
}
// keptCopies is every copy kept of the found configuration.
func keptCopies(t *testing.T, dir string) []string {
t.Helper()
kept, err := filepath.Glob(filepath.Join(dir, "kept", "*-wg0.conf"))
if err != nil {
t.Fatal(err)
}
return kept
}
func TestAConfigurationPutBackIsRetiredAgainOnItsFirstOriginalAndSettles(t *testing.T) {
dir, config, mesh, keyFile, m := aHubInUse(t)
m.handshakes = handshaken
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
_, state := applyAdopted(t, d, store.State{}, m, dir)
first, _ := state.RetiredAt(config)
// Put back by hand with the original, while no peer is through yet: held on the first
// original, and the account says what a rollback takes.
write(t, config, foundConf)
m.handshakes = ""
report, state := applyAdopted(t, d, state, m, dir)
if held, ok := state.HeldAt(takesOverID); !ok || held.Kept != first.Kept {
t.Fatalf("what came back is not held on the first original: %+v", held)
}
if report.Tunnel == nil || !strings.Contains(report.Tunnel.Note, "came back after it was retired") ||
!strings.Contains(report.Tunnel.Note, "unassigning the private network first") {
t.Errorf("the account does not say a rollback means unassigning the private network: %+v", report.Tunnel)
}
// Proven: retired again, nothing more kept, said once.
m.handshakes = handshaken
report, state = applyAdopted(t, d, state, m, dir)
again, _ := state.RetiredAt(config)
if _, err := os.Lstat(config); !os.IsNotExist(err) || again.Kept != first.Kept || again.Extra != "" {
t.Fatalf("put back as it was, it was not retired again on the first original: %+v", again)
}
if o := outcomeOf(report, takesOverID); o.Action != "removed" ||
!strings.HasPrefix(o.Detail, "the found configuration came back and was retired again") ||
strings.Contains(o.Detail, "differed") {
t.Errorf("the second retirement is not said as one: %+v", o)
}
if !strings.Contains(report.Tunnel.Note, "unassigning the private network first") {
t.Errorf("the account does not say what a rollback takes: %q", report.Tunnel.Note)
}
if n := len(keptCopies(t, dir)); n != 1 {
t.Errorf("%d copies kept of one content", n)
}
if report, _ := applyAdopted(t, d, state, m, dir); report.Changed() {
t.Errorf("a steady machine moved after the second retirement: %+v", report.Outcomes)
}
// Put back with something else: that is kept beside the first original, which stays the record's.
other := strings.Replace(foundConf, "PEER-B=", "PEER-Z=", 1)
write(t, config, other)
report, state = applyAdopted(t, d, state, m, dir)
third, _ := state.RetiredAt(config)
if third.Kept != first.Kept || third.Extra == "" || third.Extra == first.Kept {
t.Fatalf("other content was not kept apart from the first original: %+v", third)
}
if got, _ := os.ReadFile(third.Extra); string(got) != other {
t.Errorf("the extra copy does not hold what was put back: %q", got)
}
if o := outcomeOf(report, takesOverID); !strings.Contains(o.Detail, third.Extra) ||
!strings.Contains(report.Tunnel.Note, third.Extra) {
t.Errorf("where the extra copy is was not said: %+v / %q", o, report.Tunnel.Note)
}
// And the same other content again: nothing more kept, the record as it was.
write(t, config, other)
report, state = applyAdopted(t, d, state, m, dir)
fourth, _ := state.RetiredAt(config)
if fourth.Kept != first.Kept || fourth.Extra != third.Extra || len(keptCopies(t, dir)) != 2 {
t.Errorf("the same content put back again grew the copies: %+v, %v", fourth, keptCopies(t, dir))
}
if o := outcomeOf(report, takesOverID); strings.Contains(o.Detail, "differed") {
t.Errorf("a copy already kept was said as new: %+v", o)
}
if report, _ := applyAdopted(t, d, state, m, dir); report.Changed() {
t.Errorf("a steady machine moved: %+v", report.Outcomes)
}
}
func TestOnlyWgQuicksOwnConfigurationIsRetired(t *testing.T) {
// Not under the wireguard directory.
dir, config, mesh, keyFile, m := aHubInUse(t)
wireguardDir = filepath.Join(dir, "elsewhere")
m.handshakes = handshaken
report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)
if got, _ := os.ReadFile(config); string(got) != foundConf {
t.Fatal("a configuration outside wg-quick's directory was removed")
}
if _, held := state.HeldAt(takesOverID); !held || !strings.Contains(report.Tunnel.Note, "is not retired: only ") {
t.Errorf("the refusal is not said, or the hold ended: %+v", report.Tunnel)
}
// A path the mesh itself writes.
dir, config, mesh, keyFile, m = aHubInUse(t)
m.handshakes = handshaken
known := store.State{}
known.Record(store.Applied{ID: "bundle.wg0", Type: "file", Target: config, Origin: store.OriginCarried})
report, _ = applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), known, m, dir)
if got, _ := os.ReadFile(config); string(got) != foundConf {
t.Fatal("a path the mesh writes was retired")
}
if !strings.Contains(report.Tunnel.Note, "a path the mesh itself writes") {
t.Errorf("the refusal is not said: %+v", report.Tunnel)
}
}
func TestAFoundConfigurationThatIsALinkIsKeptAndLeftToAPerson(t *testing.T) {
dir, config, mesh, keyFile, m := aHubInUse(t)
target := filepath.Join(dir, "predecessor", "hub.conf")
if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil {
t.Fatal(err)
}
write(t, target, foundConf)
if err := os.Remove(config); err != nil {
t.Fatal(err)
}
if err := os.Symlink(target, config); err != nil {
t.Fatal(err)
}
m.handshakes = handshaken
report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)
if _, err := os.Lstat(config); err != nil {
t.Fatal("the link was removed, leaving the key-bearing file it points at")
}
if got, _ := os.ReadFile(target); string(got) != foundConf {
t.Fatal("the file the link points at was touched")
}
held, ok := state.HeldAt(takesOverID)
if !ok {
t.Fatal("the hold ended")
}
if kept, _ := os.ReadFile(held.Kept); string(kept) != foundConf {
t.Errorf("what was kept is not what the link points at: %q", kept)
}
if !strings.Contains(report.Tunnel.Note, "is a link to "+target) || !strings.Contains(report.Tunnel.Note, "by hand") {
t.Errorf("the account does not say the link must be retired by hand: %q", report.Tunnel.Note)
}
}
func TestARetirementWhoseRecordWasNeverSavedIsRecordedByTheNextApply(t *testing.T) {
dir, config, mesh, keyFile, m := aHubInUse(t)
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
_, state := applyAdopted(t, d, store.State{}, m, dir)
held, _ := state.HeldAt(takesOverID)
// An apply removed the file and stopped before its state was saved.
if err := os.Remove(config); err != nil {
t.Fatal(err)
}
m.handshakes = handshaken
report, state := applyAdopted(t, d, state, m, dir)
if r, ok := state.RetiredAt(config); !ok || r.Kept != held.Kept {
t.Fatalf("the retirement was not recorded: %+v", state.Retired)
}
if _, still := state.HeldAt(takesOverID); still {
t.Error("the hold did not end")
}
if o := outcomeOf(report, takesOverID); o.Action != "unchanged" || !strings.Contains(o.Detail, "already gone") {
t.Errorf("a file already gone is not said as such: %+v", o)
}
}
func TestARemovalThatFailsKeepsTheFileAndTheHold(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("root removes from a directory it may not write to")
}
dir, _, mesh, keyFile, m := aHubInUse(t)
wg := filepath.Join(dir, "wireguard")
if err := os.MkdirAll(wg, 0o700); err != nil {
t.Fatal(err)
}
config := filepath.Join(wg, "wg0.conf")
write(t, config, foundConf)
wireguardDir = wg
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
_, state := applyAdopted(t, d, store.State{}, m, dir)
if err := os.Chmod(wg, 0o500); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(wg, 0o700) })
m.handshakes = handshaken
report, state := applyAdopted(t, d, state, m, dir)
if got, _ := os.ReadFile(config); string(got) != foundConf {
t.Fatal("the found configuration is gone although it could not be removed")
}
if _, held := state.HeldAt(takesOverID); !held {
t.Error("the hold ended although nothing was retired")
}
if _, retired := state.RetiredAt(config); retired {
t.Error("recorded as retired")
}
if !strings.Contains(report.Tunnel.Note, "removing it failed") || !strings.Contains(report.Tunnel.Note, "permission denied") {
t.Errorf("the failed removal is not said: %q", report.Tunnel.Note)
}
}
internal/declaration/block_test.go
+52
View File
@@ -0,0 +1,52 @@
package declaration
import (
"strings"
"testing"
)
// Defends novox/hq issue 128: a file written into a block carries only its lines, in content,
// never a line the host keeps as its own marker, and says where a new region goes only as a block.
func TestAFileWrittenIntoABlockIsRefusedUnlessItIsOnlyItsLines(t *testing.T) {
for name, c := range map[string]struct{ resource, refusal string }{
"a begin marker in content": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a\n# BEGIN mesh f\nb\n"}`, "# BEGIN mesh f"},
"an end marker in content": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a\n# END mesh other\n"}`, "# END mesh other"},
"a marker with a CR": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"# BEGIN mesh x\r\n"}`, "marker"},
"sealed": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","sealed":"abc"}`, "not sealed"},
"bytes": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","bytes":"YQ=="}`, "not sealed, bytes"},
"secrets": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"${secret:s}","secrets":{"s":"abc"}}`, "secrets"},
"create-once": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a","create-once":true}`, "create-once"},
"at on a whole file": {`{"id":"f","type":"file","path":"/etc/hosts","content":"a","at":"start"}`, `at "start"`},
"at on a JSON file": {`{"id":"f","type":"file","path":"/etc/x.json","into":"json","content":"{}","at":"end"}`, `at "end"`},
"at somewhere else": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a","at":"middle"}`, `"start" or "end"`},
"an id ending in a space": {`{"id":"f ","type":"file","path":"/etc/hosts","into":"block","content":"a"}`, "whitespace"},
"an unknown format": {`{"id":"f","type":"file","path":"/etc/hosts","into":"lines","content":"a"}`, `"json" or "block"`},
} {
_, err := Parse([]byte(`{"declaration":1,"resources":[` + c.resource + `]}`))
if err == nil || !strings.Contains(err.Error(), c.refusal) {
t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err)
}
}
}
func TestAFileWrittenIntoABlockIsRead(t *testing.T) {
d, err := Parse([]byte(`{"declaration":1,"resources":[
{"id":"mesh-wireguard.fact-node-names","type":"file","path":"/etc/hosts","into":"block","content":"10.42.0.1 ace\n# BEGIN devtool x\n"},
{"id":"dhcpcd.options","type":"file","path":"/etc/dhcpcd.conf","into":"block","content":"nohook resolv.conf\n","at":"start"},
{"id":"empty","type":"file","path":"/etc/x","into":"block","content":"","at":"end"}
]}`))
if err != nil {
t.Fatal(err)
}
if f := d.Resources[0].(*File); f.Into != IntoBlock || f.At != "" {
t.Errorf("read as into %q at %q", f.Into, f.At)
}
if f := d.Resources[1].(*File); f.At != AtStart {
t.Errorf("at was read as %q", f.At)
}
if begin, end := BlockMarkers("mesh-wireguard.fact-node-names"); begin != "# BEGIN mesh mesh-wireguard.fact-node-names" ||
end != "# END mesh mesh-wireguard.fact-node-names" {
t.Errorf("the markers are %q and %q", begin, end)
}
}
internal/declaration/declaration.go
+150 -22
View File
@@ -161,8 +161,30 @@ type File struct {
// "json" is spoken — the content is a JSON object whose keys the host sets in the file's
// object, keeping every other key as it found it and recording what each of its keys held
// before, so undeclaring the file gives those back.
//
// "block" is the same idea for a file that is not structured (novox/hq issue 128): the
// content is the mesh's lines, and the host owns only the region between `# BEGIN mesh <id>`
// and `# END mesh <id>`, keeping every line outside it byte for byte. The machine's hosts file
// is the case that needed it — on a workstation the distribution, a local development tool and
// the operator all write into it, and the mesh writing it whole took their lines away at the
// next change to the mesh's names, silently. Marked blocks are the shape the other tools in
// that file already use, and `#` is the comment character of every file this serves.
//
// Written into, in either format, the file's mode and owner are the machine's: a declared mode
// and owner apply only to a file the host creates, and a file that was there keeps its own.
Into string `json:"into,omitempty"`
// At is where a file written into a block has its region added when the file does not hold
// one yet: "end", the default, or "start". A region already there stays where it is, whatever
// this says — moving it would move the lines around it, and those are the machine's.
//
// **Some files give a line its meaning by what stands above it.** dhcpcd's configuration scopes
// every line after `interface X` to that interface, and a real one ends with exactly that — an
// interface and its static address. A region added at the end would make the mesh's global
// options (`nohook resolv.conf`, `denyinterfaces mesh0`) options of one interface, and dhcpcd
// would read them without complaint. At the start, nothing stands above them.
At string `json:"at,omitempty"`
// Sealed is content encrypted to this node's sealing key, for a file the mesh must deliver
// without being able to read.
//
@@ -241,16 +263,50 @@ func (f *File) validate(where string, _ bool) []string {
problems = append(problems, where+
": a file written into JSON carries a JSON object of the keys it sets")
}
if f.Sealed != "" || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce {
case IntoBlock:
// The markers are how the host finds its region again. A marker in the content would be
// a second region, or the end of this one, the next time the file is read — and the host
// would then rewrite, or on undeclare take out, lines that were never the mesh's.
for _, line := range strings.Split(f.Content, "\n") {
if strings.HasPrefix(line, BlockBegin) || strings.HasPrefix(line, BlockEnd) {
problems = append(problems, fmt.Sprintf(
"%s: a file written into a block carries the mesh's lines, and %q is a marker the "+
"host keeps for itself", where, strings.TrimRight(line, "\r")))
break
}
}
// The id is written into the markers, so it has to stay on one line — and whitespace at
// either end of it is whitespace the host would have to match exactly in a line some
// editor may trim.
if strings.ContainsAny(f.ID, "\r\n") {
problems = append(problems, where+
": a file written into says only its keys, in content — not sealed, bytes, "+
"secrets or create-once")
": a file written into a block names its region by its id, and this id spans lines")
} else if strings.TrimSpace(f.ID) != f.ID {
problems = append(problems, where+
": a file written into a block names its region by its id, and this id begins or ends in whitespace")
}
default:
problems = append(problems, fmt.Sprintf(
"%s: into %q; a file is written into \"json\", or omits it to be written whole",
"%s: into %q; a file is written into \"json\" or \"block\", or omits it to be written whole",
where, f.Into))
}
switch {
case f.At == "":
case f.Into != IntoBlock:
problems = append(problems, fmt.Sprintf(
"%s: at %q; only a file written into a block has a place its region is added", where, f.At))
case f.At != AtStart && f.At != AtEnd:
problems = append(problems, fmt.Sprintf(
"%s: at %q; a block is added at \"start\" or \"end\", or omits it to be added at the end",
where, f.At))
}
if f.Into == IntoJSON || f.Into == IntoBlock {
if f.Sealed != "" || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce {
problems = append(problems, where+
": a file written into says only its part, in content — not sealed, bytes, "+
"secrets or create-once")
}
}
var said []string
for name, value := range map[string]string{
"content": f.Content, "sealed": f.Sealed, "bytes": f.Bytes,
@@ -519,16 +575,34 @@ func (d *Process) Identity() string { return d.ID }
func (d *Process) Kind() Type { return TypeProcess }
func (d *Process) Target() string { return d.Name }
// ProcessNameProblem says what is wrong with a process name, or nothing.
//
// The name becomes a unit name, a file under the unit directory and a directory under the mesh's
// own — the one removing the process deletes, whole (novox/hq ADR 0118). So a name that is not one
// plain path element is refused: with a separator it writes somewhere nobody meant, and "." or
// ".." IS the mesh's directory or its parent — removing a process named ".." would delete every
// bundle the mesh has, and more. One with a leading dash is read by the service manager as an
// option, not a unit. Exported because the removal checks the recorded name again: a record is
// what the host wrote, and a host of an older version wrote it under looser rules.
func ProcessNameProblem(name string) string {
switch {
case name == "":
return "a process needs a name, which is what its unit is called"
case strings.ContainsAny(name, "/ \t"):
return "a process name becomes a unit name, so it cannot contain a path separator or a space"
case name == "." || name == "..":
return fmt.Sprintf("a process name becomes a directory under the mesh's own, and %q would be "+
"that directory or its parent", name)
case strings.HasPrefix(name, "-"):
return "a process name cannot begin with a dash: the service manager would read it as an option"
}
return ""
}
func (d *Process) validate(where string, _ bool) []string {
var problems []string
if d.Name == "" {
problems = append(problems, where+": a process needs a name, which is what its unit is called")
}
if strings.ContainsAny(d.Name, "/ \t") {
// It becomes a unit name and a file on disk. A name with a separator in it would write
// somewhere nobody meant.
problems = append(problems, where+": a process name becomes a unit name, so it cannot "+
"contain a path separator or a space")
if problem := ProcessNameProblem(d.Name); problem != "" {
problems = append(problems, where+": "+problem)
}
if d.Source == "" {
problems = append(problems, where+": a process needs somewhere to fetch its bundle from")
@@ -592,7 +666,17 @@ type Service struct {
ID string `json:"id"`
Type Type `json:"type"`
Unit string `json:"unit"`
State string `json:"state"`
// State is "running" or "stopped" — or absent, and then **the unit's lifecycle is the
// machine's; the mesh only reflects its triggers** (novox/hq ADR 0117). The uplink modules
// declare the machine's own network manager this way: the mesh writes into its configuration
// and needs it to read that again, and nothing more. Stated, the host would start the manager
// on a machine that uses another one — two managers fighting over the same links — and, when
// the module was unassigned, stop it: the machine's network, the channel the mesh itself
// arrives on, gone at the moment of a routine change. So a service without a state is never
// started, stopped, enabled or disabled, is reloaded or restarted only when a trigger changed
// and it is already running, and undeclared is simply forgotten. It says nothing unless it
// names a trigger, and it may not say boot or takes-over, which are both lifecycle.
State string `json:"state,omitempty"`
// Boot is "enabled" or "disabled" — whether the unit starts at boot. Optional: absent means
// the host asserts nothing about it and leaves whatever is there.
//
@@ -622,9 +706,10 @@ type Service struct {
// TakesOver names the found tunnel this service replaces (novox/hq ADR 0105): before this unit
// is started, the named unit is stopped and disabled — never flushed — and its configuration
// file is kept like any held file. Only on an adopted node, and only said by the controller,
// which knows the found tunnel's key is this node's own: without that, starting this unit on
// the found one's port would drop every peer's packets.
// file is kept like any held file — until the take is proven by a peer's handshake, and then
// retired, its original staying kept (novox/hq ADR 0119). Only on an adopted node, and only
// said by the controller, which knows the found tunnel's key is this node's own: without that,
// starting this unit on the found one's port would drop every peer's packets.
TakesOver *TakeOver `json:"takes-over,omitempty"`
}
@@ -636,6 +721,10 @@ type TakeOver struct {
Config string `json:"config"`
}
// Stateless reports whether the unit's lifecycle is the machine's, and the mesh only reflects the
// service's triggers (novox/hq ADR 0117).
func (s *Service) Stateless() bool { return s.State == "" }
func (s *Service) Identity() string { return s.ID }
func (s *Service) Kind() Type { return TypeService }
func (s *Service) Target() string { return s.Unit }
@@ -645,9 +734,18 @@ func (s *Service) validate(where string, _ bool) []string {
if s.Unit == "" {
problems = append(problems, where+": a service needs a unit")
}
if s.State != "running" && s.State != "stopped" {
switch {
case s.State == "running" || s.State == "stopped":
case s.State != "":
problems = append(problems, fmt.Sprintf(
"%s: state %q; a service is \"running\" or \"stopped\"", where, s.State))
"%s: state %q; a service is \"running\" or \"stopped\", or omits state to leave the "+
"unit's lifecycle to the machine", where, s.State))
case s.Boot != "" || s.TakesOver != nil:
problems = append(problems, where+": a service that omits state leaves the unit's lifecycle "+
"to the machine, and boot and takes-over are both its lifecycle")
case len(s.RestartOn) == 0 && len(s.ReloadOn) == 0:
problems = append(problems, where+": a service that omits state leaves the unit's lifecycle "+
"to the machine, and names no restart-on or reload-on — it declares nothing")
}
if s.Boot != "" && s.Boot != "enabled" && s.Boot != "disabled" {
problems = append(problems, fmt.Sprintf(
@@ -662,7 +760,7 @@ func (s *Service) validate(where string, _ bool) []string {
case t.Unit == s.Unit:
problems = append(problems, fmt.Sprintf("%s: takes-over names %s, which is this service's own unit",
where, t.Unit))
case s.State != "running":
case s.State != "running" && s.State != "":
problems = append(problems, where+": a service that takes over a tunnel is running — stopping "+
"the found one for a service that will not run would leave the peers with nothing")
}
@@ -670,8 +768,31 @@ func (s *Service) validate(where string, _ bool) []string {
return problems
}
// IntoJSON is the one structured format a file is written into.
const IntoJSON = "json"
// What a file is written into (novox/hq ADR 0102): a JSON object whose keys the mesh sets, or a
// text file in which the mesh owns one marked block of lines (novox/hq issue 128).
const (
IntoJSON = "json"
IntoBlock = "block"
)
// The lines that delimit the mesh's region in a file written into a block, each followed by the
// resource's id. Exact lines, never patterns: another tool's `# BEGIN …` block in the same file is
// that tool's, and a marker that merely resembled the mesh's must not be taken for it.
const (
BlockBegin = "# BEGIN mesh "
BlockEnd = "# END mesh "
)
// Where a file written into a block has its region added, when it has none yet.
const (
AtStart = "start"
AtEnd = "end"
)
// BlockMarkers are the two lines, without their line ends, that delimit a resource's region.
func BlockMarkers(id string) (begin, end string) {
return BlockBegin + id, BlockEnd + id
}
// Opening is a port reachable on an adopted node, from where, and on which path.
//
@@ -1145,6 +1266,13 @@ type envelope struct {
Version int `json:"declaration"`
For string `json:"for,omitempty"`
Adoption *Adoption `json:"adoption,omitempty"`
// OwnsNothing is the control plane saying, explicitly, that this node's declaration is empty
// on purpose — it owns nothing the mesh put there (novox/hq issue 127). Without it an empty
// resources list is refused as a likely mistake; with it the node applies the empty
// declaration and drops what it last held. The two are distinguished because a truncated or
// mis-composed body arrives as empty too, and a host that could not tell them apart would let
// a bug quietly strip a machine.
OwnsNothing bool `json:"owns_nothing,omitempty"`
Resources []json.RawMessage `json:"resources"`
}
@@ -1165,7 +1293,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption}
var problems []string
if len(env.Resources) == 0 {
if len(env.Resources) == 0 && !env.OwnsNothing {
problems = append(problems, "no resources. An empty declaration is a mistake, not a "+
"machine with nothing on it — say so with an explicit empty list if that is meant")
}
internal/declaration/declaration_test.go
+13
View File
@@ -451,3 +451,16 @@ func TestAContainersResolverAndAddressAreAddressesOrRefused(t *testing.T) {
t.Errorf("a well-formed resolver and address were refused: %v", p)
}
}
func TestAnExplicitlyEmptyDeclarationIsAccepted(t *testing.T) {
// A deliberately-empty declaration (novox/hq issue 127) says owns_nothing, and is applied so
// the node drops what it last held — distinct from an accidental empty body, which is refused.
if _, err := Parse([]byte(`{"declaration":1,"owns_nothing":true,"resources":[]}`)); err != nil {
t.Fatalf("an explicitly-empty declaration must be accepted: %v", err)
}
// Without the marker, an empty declaration is still refused as a likely mistake.
_, err := Parse([]byte(`{"declaration":1,"resources":[]}`))
if err == nil || !strings.Contains(err.Error(), "no resources") {
t.Fatalf("an unmarked empty declaration must still be refused; got %v", err)
}
}
internal/declaration/process_test.go
+4 -1
View File
@@ -55,7 +55,10 @@ func TestAProcessMustSayWhatToRun(t *testing.T) {
// Its name becomes a unit name and a path, so a separator in it would write somewhere nobody meant.
func TestAProcesssNameCannotEscapeItsUnit(t *testing.T) {
for _, bad := range []string{"", "../escape", "two words", "a/b"} {
// "." and ".." are one path element each, and the mesh's own bundle directory and its parent:
// removing a process named ".." would delete every bundle the mesh has, and more (novox/hq ADR
// 0118). A leading dash is an option to the service manager, not a unit.
for _, bad := range []string{"", "../escape", "two words", "a/b", ".", "..", "-", "--now"} {
d := aProcess()
d.Name = bad
if problems := d.validate("a process", false); len(problems) == 0 {
internal/declaration/stateless_test.go
+30
View File
@@ -0,0 +1,30 @@
package declaration
import (
"strings"
"testing"
)
// Defends novox/hq ADR 0117: a service may leave its unit's lifecycle to the machine, and then
// says nothing but its triggers.
func TestAServiceWithoutAStateSaysOnlyItsTriggers(t *testing.T) {
for name, c := range map[string]struct{ resource, refusal string }{
"no trigger": {`{"id":"s","type":"service","unit":"NetworkManager.service"}`, "declares nothing"},
"with boot": {`{"id":"s","type":"service","unit":"NetworkManager.service","boot":"enabled","reload-on":["f"]}`, "boot and takes-over"},
"with takes-over": {`{"id":"s","type":"service","unit":"a.service","reload-on":["f"],"takes-over":{"interface":"wg0","unit":"b.service","config":"/etc/x"}}`, "boot and takes-over"},
"an unknown state": {`{"id":"s","type":"service","unit":"a.service","state":"paused"}`, "omits state to leave the unit's lifecycle to the machine"},
} {
_, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"f","type":"file","path":"/etc/x","content":"x"},` + c.resource + `]}`))
if err == nil || !strings.Contains(err.Error(), c.refusal) {
t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err)
}
}
d, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"f","type":"file","path":"/etc/x","content":"x"},
{"id":"s","type":"service","unit":"NetworkManager.service","restart-on":["f"]}]}`))
if err != nil {
t.Fatal(err)
}
if s := d.Resources[1].(*Service); !s.Stateless() {
t.Error("a service without a state was not read as stateless")
}
}
internal/link/enrol.go
+1
View File
@@ -75,6 +75,7 @@ type Tunnel struct {
Port int `json:"port"`
Address string `json:"address"`
Range string `json:"range"`
MTU int `json:"mtu,omitempty"`
PublicKey string `json:"public_key"`
Peers []TunnelPeer `json:"peers,omitempty"`
}
internal/store/store.go
+150 -1
View File
@@ -70,6 +70,25 @@ type Applied struct {
// anywhere saying why.
Wrote string `json:"wrote,omitempty"`
// Kept is where the original of a file this host wrote over was kept (novox/hq ADR 0100):
// by the keep on its first write, or by the hold that was released when its module was taken.
// Recorded rather than only reported, because a file once written whole and now written into
// (novox/hq issue 128) is given back its original with the mesh's region in it — and a path
// said once in a log line is not a path the host can find again.
Kept string `json:"kept,omitempty"`
// Stateless is, for a service, that its unit's lifecycle was never the mesh's (novox/hq ADR
// 0117) — kept here because removal happens once the declaration that said so is gone, and a
// service removed as if it had a state is stopped: the machine's network manager, for one.
Stateless bool `json:"stateless,omitempty"`
// Found is, for a service, the state its unit was in when this host first applied it — before
// the mesh started, stopped, enabled or disabled anything. Removal gives that back and nothing
// more (novox/hq ADR 0118): a unit that was running before the mesh arrived keeps running
// when its declaration goes; one the mesh started is stopped again. Absent on a record written
// before the host kept it, and then the unit is left exactly as it is.
Found *FoundUnit `json:"found,omitempty"`
// Into is set for a file written into rather than over (novox/hq ADR 0102): the format, what
// each of the mesh's keys held before it set them, which of them were absent, and whether the
// file itself was — so undeclaring it gives the machine back exactly what it had.
@@ -84,7 +103,7 @@ type Applied struct {
Reads map[string]string `json:"reads,omitempty"`
}
// Into is what a file written into held before the mesh's keys.
// Into is what a file written into held before the mesh's keys, or before the mesh's block.
type Into struct {
Format string `json:"format"`
Before map[string]json.RawMessage `json:"before,omitempty"`
@@ -94,6 +113,22 @@ type Into struct {
// to the machine's list — never a member that was already there. Undeclared, only these go,
// and drift is judged on these alone (novox/hq ADR 0102).
Added map[string][]json.RawMessage `json:"added,omitempty"`
// Region is, for a file written into a block (novox/hq issue 128), what the lines between the
// mesh's markers held before the mesh wrote them — nil when there was no region, which is
// different from a region that was there and empty. Undeclared, a recorded region is put back
// and an unrecorded one is taken out, markers and all. It is the block's "what each key held
// before": the one thing the host needs to give the file back.
Region *string `json:"region,omitempty"`
// Separated says the host put a blank line between the region and the machine's lines when it
// added the region — before it at the end, after it at the start, as At says — so taking the
// region out takes that line with it and nothing of the operator's.
Separated bool `json:"separated,omitempty"`
// At is where the host added the region: "start", or empty for the end.
At string `json:"at,omitempty"`
// Ended says the machine's last line had no line end and the host gave it one to add the
// region after it, so taking the region out takes that line end too.
Ended bool `json:"ended,omitempty"`
}
// State is the whole of what a node knows about what it has done.
@@ -119,10 +154,56 @@ type State struct {
// other mode can be refused before it is applied (novox/hq issue 104).
Mode string `json:"mode,omitempty"`
// FoundFirst is, by resource id, what a service's unit was found as by an apply of it that did
// not finish — kept apart from Resources, because a record follows the fact and this apply's
// fact never came (novox/hq ADR 0118).
//
// **The capture is the one reading that cannot be taken again.** A first apply that enabled a
// unit and then failed to start it leaves no record; without this, the next apply would find
// the unit enabled, take that for what the machine had, and undeclaring would leave enabled a
// unit the mesh enabled. So what is found is written the moment it is read, whatever the apply
// of the resource then does, and a later apply reads it here before it reads the machine.
// Dropped once a record carrying it is written, and when its resource is no longer declared.
FoundFirst map[string]PendingFound `json:"found_first,omitempty"`
// Genesis is the bundle this host consumed raising the foundation, if it has. Once recorded,
// the bundle carried in the binary is not applied again: what genesis applied was rewritten
// for this machine, and the mesh has said more since (novox/hq issue 104).
Genesis *Genesis `json:"genesis,omitempty"`
// Retired is each found tunnel configuration the mesh removed from where its unit reads it,
// once the private network's take of that tunnel was proven (novox/hq ADR 0119).
//
// **Not a hold, and never released with one.** The hold on the found configuration ends at the
// retirement — what it held for has been replaced, and the node stops reporting it — so without
// this the next apply would find no hold and no file and read the take as one whose
// configuration vanished before it could be kept. It is kept whether or not the private
// network stays declared: undeclaring brings nothing back (ADR 0118), and a private network
// assigned again finds the tunnel's configuration retired rather than missing.
Retired []Retired `json:"retired,omitempty"`
}
// Retired is a found configuration the mesh removed once what replaced it was proven (novox/hq
// ADR 0119): where it was, under which hold it had been kept, and where its original still is.
type Retired struct {
ID string `json:"id"`
Path string `json:"path"`
// Kept is the original as found (novox/hq ADR 0100) — the record of what the predecessor was,
// and a person's way back if one is ever wanted. The mesh never copies it back. It is the FIRST
// original, and stays so however often the file comes back: a retirement repeated never moves
// it. Digest is what it holds.
Kept string `json:"kept"`
Digest string `json:"digest,omitempty"`
// Extra is where what was at the path when it was last retired is kept, when that differed from
// the first original — rewritten since it was found, or put back with other content — and
// ExtraDigest what it holds. One copy per distinct content: a file that comes back as it was
// last retired keeps nothing more.
Extra string `json:"extra,omitempty"`
ExtraDigest string `json:"extra_digest,omitempty"`
At time.Time `json:"at"`
// Again is how many times the configuration came back after it was retired, and was retired
// again (novox/hq ADR 0119).
Again int `json:"again,omitempty"`
}
// Modes a node can be in (novox/hq ADR 0100).
@@ -265,6 +346,27 @@ func (s *State) Release(id string) {
}
}
// RetiredAt returns the retirement of the found configuration at a path, if the mesh retired one.
func (s State) RetiredAt(path string) (Retired, bool) {
for _, r := range s.Retired {
if r.Path == path {
return r, true
}
}
return Retired{}, false
}
// RecordRetired adds or replaces the retirement of the configuration at one path.
func (s *State) RecordRetired(r Retired) {
for i, existing := range s.Retired {
if existing.Path == r.Path {
s.Retired[i] = r
return
}
}
s.Retired = append(s.Retired, r)
}
// Find returns what was applied under an identity.
func (s State) Find(id string) (Applied, bool) {
for _, r := range s.Resources {
@@ -419,3 +521,50 @@ func originOf(r Applied) string {
}
return r.Origin
}
// FoundUnit is a service's unit as the host first found it.
type FoundUnit struct {
// Unit is which unit this was read from. What was found about one unit says nothing about
// another, so a service whose declaration moves to a different unit is read again for that one
// (novox/hq ADR 0118). Empty on what was kept before this was: the record's Target then says.
Unit string `json:"unit,omitempty"`
// State is "running" or "stopped".
State string `json:"state"`
// Boot is "enabled" or "disabled" — or empty when the declaration never set it, and the host
// never touched it.
Boot string `json:"boot,omitempty"`
}
// PendingFound is a unit as found by an apply of its service that has not yet been recorded, and
// who asked for that apply — so only a declaration from the same origin can say it is gone.
type PendingFound struct {
FoundUnit
Origin string `json:"origin,omitempty"`
}
// KeepFound writes down what an unfinished apply found a service's unit as.
func (s *State) KeepFound(id, origin string, f FoundUnit) {
if s.FoundFirst == nil {
s.FoundFirst = map[string]PendingFound{}
}
s.FoundFirst[id] = PendingFound{FoundUnit: f, Origin: origin}
}
// DropFound forgets what was found for one resource: its record now carries it, or it is gone.
func (s *State) DropFound(id string) {
delete(s.FoundFirst, id)
if len(s.FoundFirst) == 0 {
s.FoundFirst = nil
}
}
// DropFoundUndeclared forgets what was found for every resource of this origin the declaration no
// longer names. Only this origin's, for the reason Orphans gives: a mesh declaration's silence says
// nothing about what the bundle applies, nor the other way round.
func (s *State) DropFoundUndeclared(declared map[string]bool, origin string) {
for id, p := range s.FoundFirst {
if !declared[id] && originOf(Applied{Origin: p.Origin}) == origin {
s.DropFound(id)
}
}
}
internal/tunnel/tunnel.go
+55 -1
View File
@@ -3,7 +3,9 @@
//
// On an adopted node that is the hub, the mesh's interface is raised with the found interface's
// private key, on its port, with its address and range, and every peer it had. The found interface
// is stopped, never flushed; its configuration stays on disk. What this package does is the
// is stopped, never flushed; its configuration stays on disk until the take is proven — a peer
// has handshaken with the mesh's interface — and is then retired (novox/hq ADR 0119). What this
// package does is the
// reading: which interface is there, what its file says, and what of that travels to the mesh —
// everything but the private key, which becomes the node's own overlay key and is stored the way
// that key is stored.
@@ -44,6 +46,11 @@ type Found struct {
Port int `json:"port"`
Address string `json:"address"`
Range string `json:"range"`
// MTU is the interface's, when the found config set one. Kept because a tuned tunnel (a path
// that needs 1380, say) breaks silently if the mesh's interface comes up at the 1420 default:
// no ping fails, but TLS handshakes stall and transfers hang (novox/hq: a taken tunnel carries
// its MTU). Zero when the config named none, and the mesh sets no MTU line then.
MTU int `json:"mtu,omitempty"`
// PublicKey is what every peer knows this tunnel by — derived here from the private key, so
// it is the key the file actually holds and not a comment beside it.
PublicKey string `json:"public_key"`
@@ -146,6 +153,47 @@ func Find(ctx context.Context, run Runner, named string) (Found, error) {
return found, nil
}
// Handshaken is how many peers of an interface have completed a handshake with it: the proof that
// the interface carries the tunnel, rather than merely being up (novox/hq ADR 0119).
//
// Asked of the running interface, since a handshake is a fact about the kernel's tunnel that no
// file records. A question that cannot be asked — no `wg` on the machine, no such interface, a
// permission refused — is an error and never a zero: "no peer has handshaken" retires nothing
// either, but it is a different thing to tell a person.
func Handshaken(ctx context.Context, run Runner, iface string) (int, error) {
out, err := run(ctx, "wg", "show", iface, "latest-handshakes")
if err != nil {
return 0, fmt.Errorf("cannot ask %s which peers have handshaken: %w", iface, err)
}
return ParseHandshakes(out)
}
// ParseHandshakes reads `wg show <interface> latest-handshakes`: one line per peer, its public key
// and the Unix time of its latest handshake, tab-separated — zero for a peer that never has. What
// is counted is the peers with a time. A line that is not a key and a time is refused rather than
// skipped: output this does not understand is not evidence of anything.
func ParseHandshakes(out string) (int, error) {
n := 0
for i, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
fields := strings.Fields(line)
if len(fields) != 2 {
return 0, fmt.Errorf("line %d of the handshakes is not a peer and a time: %q", i+1, line)
}
at, err := strconv.ParseInt(fields[1], 10, 64)
if err != nil || at < 0 {
return 0, fmt.Errorf("line %d of the handshakes does not end in a time: %q", i+1, line)
}
if at > 0 {
n++
}
}
return n, nil
}
func orNone(names []string) string {
if len(names) == 0 {
return "none"
@@ -209,6 +257,12 @@ func Parse(raw []byte) (Found, error) {
return Found{}, fmt.Errorf("ListenPort %q is not a port", value)
}
f.Port = port
case "mtu":
mtu, err := strconv.Atoi(value)
if err != nil || mtu < 576 || mtu > 65535 {
return Found{}, fmt.Errorf("MTU %q is not a plausible MTU", value)
}
f.MTU = mtu
case "address":
// The first address is the interface's; a second family would be a second
// tunnel's worth of addressing, which this does not carry.
internal/tunnel/tunnel_test.go
+70
View File
@@ -170,3 +170,73 @@ func TestNoneUpIsAnOrdinaryAnswerAndSeveralIsAQuestion(t *testing.T) {
t.Errorf("naming a tunnel that is not up was not refused: %v", err)
}
}
func TestAFoundTunnelReadsItsMTU(t *testing.T) {
// A tuned path sets MTU in [Interface]; the mesh must carry it or the tunnel regresses to the
// default silently (novox/hq: a taken tunnel carries its MTU).
private, public := aKey(t)
withMTU := "# tuned\n[Interface]\nPrivateKey = " + private +
"\nListenPort = 51820\nAddress = 10.10.0.3/24\nMTU = 1380\n" +
"[Peer]\nPublicKey = " + public + "\nAllowedIPs = 10.10.0.1/32\n"
f, err := Parse([]byte(withMTU))
if err != nil {
t.Fatal(err)
}
if f.MTU != 1380 {
t.Fatalf("MTU 1380 was not read; got %d", f.MTU)
}
// And a config with none leaves MTU zero, so the mesh writes no MTU line.
f2, err := Parse([]byte(aConfig(private, public)))
if err != nil {
t.Fatal(err)
}
if f2.MTU != 0 {
t.Fatalf("a config with no MTU must leave it zero; got %d", f2.MTU)
}
}
// novox/hq ADR 0119: a take is proven by a handshake on the mesh's interface, read from `wg show
// <interface> latest-handshakes` — as wg prints it, a key and a Unix time per peer, zero for never.
func TestAHandshakeIsAPeerWithATime(t *testing.T) {
cases := map[string]struct {
out string
want int
}{
"two peers, one handshaken": {"PEER-A=\t1790000000\nPEER-B=\t0\n", 1},
"every peer handshaken": {"PEER-A=\t1790000000\nPEER-B=\t1790000042\n", 2},
"no peer ever": {"PEER-A=\t0\nPEER-B=\t0\n", 0},
"an interface with no peer": {"", 0},
"spaces, a trailing line": {"PEER-A= 1790000000\n\n", 1},
}
for name, c := range cases {
got, err := ParseHandshakes(c.out)
if err != nil || got != c.want {
t.Errorf("%s: %d peer(s) handshaken (%v), want %d", name, got, err, c.want)
}
}
// Output that is not a key and a time is not evidence of anything, and not a zero either.
for _, nonsense := range []string{"PEER-A=\n", "PEER-A=\tyesterday\n", "PEER-A=\t-1\n", "a b c\n"} {
if _, err := ParseHandshakes(nonsense); err == nil {
t.Errorf("%q was read as handshakes", nonsense)
}
}
}
func TestHandshakesThatCannotBeAskedAreAnErrorNotAZero(t *testing.T) {
var asked string
ok := func(_ context.Context, name string, args ...string) (string, error) {
asked = name + " " + strings.Join(args, " ")
return "PEER-A=\t1790000000\n", nil
}
if n, err := Handshaken(context.Background(), ok, "mesh0"); err != nil || n != 1 ||
asked != "wg show mesh0 latest-handshakes" {
t.Fatalf("asked %q and read %d (%v)", asked, n, err)
}
missing := func(context.Context, string, ...string) (string, error) {
return "", errors.New(`exec: "wg": executable file not found in $PATH`)
}
if _, err := Handshaken(context.Background(), missing, "mesh0"); err == nil ||
!strings.Contains(err.Error(), "mesh0") {
t.Fatalf("a machine with no wg was read as one with no handshake: %v", err)
}
}