Compare commits
41
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
14e64844df | ||
|
|
d82fc9a121 | ||
|
|
9fd3762e93 | ||
|
|
587b8aa220 | ||
|
|
48e2ff2de5 | ||
|
|
6a3435629e | ||
|
|
9072f60a30 | ||
|
|
6e208f7b3e | ||
|
|
54f6eb661a | ||
|
|
646be4fdf4 | ||
|
|
e688b3b816 | ||
|
|
50776b8613 | ||
|
|
b462f461c6 | ||
|
|
23a4436499 | ||
|
|
08c0f40ff0 | ||
|
|
3112c881e4 | ||
|
|
06aaac0820 | ||
|
|
fdc768c476 | ||
|
|
25449a31c3 | ||
|
|
1cb895346d | ||
|
|
3ae999497f | ||
|
|
32d7234637 | ||
|
|
2722e7b36e | ||
|
|
808e93a477 | ||
|
|
8e2a1e762d | ||
|
|
5dbc8e5c3c | ||
|
|
260bf0b752 | ||
|
|
93caf26aed | ||
|
|
3ac765db65 | ||
|
|
7e245dae92 | ||
|
|
5dd439df50 | ||
|
|
f68139c9d1 | ||
|
|
fc593b9dfd | ||
|
|
83306b2dba | ||
|
|
4840e21405 | ||
|
|
982b84310e | ||
|
|
c60228e719 | ||
|
|
977df39e0d | ||
|
|
f08a8ea3f7 | ||
|
|
7283924a35 | ||
|
|
27c4b765b2 |
@@ -55,12 +55,13 @@ connects to nothing and listens on nothing — what it applies comes from a file
|
|||||||
mesh-host profile what this machine can be asked to do
|
mesh-host profile what this machine can be asked to do
|
||||||
mesh-host inventory what this machine is, and what it holds
|
mesh-host inventory what this machine is, and what it holds
|
||||||
mesh-host apply FILE make this machine match a declaration from a file
|
mesh-host apply FILE make this machine match a declaration from a file
|
||||||
mesh-host reconcile make this machine match the declaration this host carries
|
mesh-host reconcile make this machine match what the mesh last told it — or, before
|
||||||
|
any mesh has, the bundle this host carries
|
||||||
mesh-host bundle show what this host carries
|
mesh-host bundle show what this host carries
|
||||||
mesh-host owned what this host has applied and still owns
|
mesh-host owned what this host has applied and still owns
|
||||||
--json machine-readable
|
--json machine-readable
|
||||||
--state where this node keeps what it knows
|
--state where this node keeps what it knows
|
||||||
--dry-run read and check the declaration, change nothing
|
--dry-run say what applying would change, and change nothing
|
||||||
```
|
```
|
||||||
|
|
||||||
```
|
```
|
||||||
@@ -114,8 +115,16 @@ A host built for a machine carries its declaration **inside the binary**:
|
|||||||
make host BUNDLE=path/to/foundation.lock
|
make host BUNDLE=path/to/foundation.lock
|
||||||
```
|
```
|
||||||
|
|
||||||
`mesh-host reconcile` then applies it. That is the first node's path — no mesh present, nothing
|
`mesh-host reconcile` then applies it, on a machine the mesh has told nothing yet. That is the
|
||||||
fetched, nothing else copied onto the machine. `copy it and run it` stops being true the moment
|
first node's path — no mesh present, nothing fetched, nothing else copied onto the machine. Once
|
||||||
|
the mesh has spoken, `reconcile` holds the machine to what it last said and never to the bundle,
|
||||||
|
which genesis consumed; a bundle or a file is refused when it says the other mode than the node
|
||||||
|
is in; and `apply FILE` is refused altogether once the mesh has spoken — a file is applied as the
|
||||||
|
bundle is, its resources recorded as the machine's own, so on an enrolled node it would plan to
|
||||||
|
remove the foundation. `apply FILE` is for a machine the mesh has not spoken to. (hq's to-be
|
||||||
|
node lifecycle describes `apply repair.json` as a rescue on an enrolled node; that line is being
|
||||||
|
amended in hq, and no rescue path exists here yet.) Both commands say what they would change
|
||||||
|
before changing anything, and `--dry-run` is that alone. `copy it and run it` stops being true the moment
|
||||||
a second file has to arrive with it, which is why the bundle is embedded rather than beside it.
|
a second file has to arrive with it, which is why the bundle is embedded rather than beside it.
|
||||||
|
|
||||||
**A default build carries nothing and refuses to reconcile**, saying so. A host that applied
|
**A default build carries nothing and refuses to reconcile**, saying so. A host that applied
|
||||||
|
|||||||
@@ -140,6 +140,10 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
|
|||||||
firewall stay as they are, the foundation's filter is not loaded and
|
firewall stay as they are, the foundation's filter is not loaded and
|
||||||
the mesh guards its own ports instead, and each module is taken on it
|
the mesh guards its own ports instead, and each module is taken on it
|
||||||
one at a time. Without it, a machine in use is refused
|
one at a time. Without it, a machine in use is refused
|
||||||
|
--tunnel adopted: the interface of the tunnel the private network takes over
|
||||||
|
(its key, port, range and peers); found by itself when one is up, and
|
||||||
|
needed only when several are. --hub-port and --overlay-range then
|
||||||
|
follow the tunnel
|
||||||
|
|
||||||
The installer carries a builder, not a control plane. What raises a mesh is therefore
|
The installer carries a builder, not a control plane. What raises a mesh is therefore
|
||||||
the same thing that will maintain it, and the control plane a mesh ends up running is
|
the same thing that will maintain it, and the control plane a mesh ends up running is
|
||||||
@@ -334,6 +338,8 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
|
|||||||
"raise this machine adopted: keep what it runs and its firewall until each module is taken")
|
"raise this machine adopted: keep what it runs and its firewall until each module is taken")
|
||||||
set.StringVar(&opts.OverlayRange, "overlay-range", opts.OverlayRange,
|
set.StringVar(&opts.OverlayRange, "overlay-range", opts.OverlayRange,
|
||||||
"the private network's address range; must not overlap a tunnel the machine already runs")
|
"the private network's address range; must not overlap a tunnel the machine already runs")
|
||||||
|
set.StringVar(&opts.Tunnel, "tunnel", "",
|
||||||
|
"adopted: the found tunnel's interface the private network takes over; found by itself when one is up")
|
||||||
if opts.Answers == nil {
|
if opts.Answers == nil {
|
||||||
opts.Answers = map[string]string{}
|
opts.Answers = map[string]string{}
|
||||||
}
|
}
|
||||||
|
|||||||
+475
-41
@@ -15,6 +15,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -36,6 +37,7 @@ import (
|
|||||||
"github.com/novox/mesh-host/internal/reachable"
|
"github.com/novox/mesh-host/internal/reachable"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
"github.com/novox/mesh-host/internal/system"
|
"github.com/novox/mesh-host/internal/system"
|
||||||
|
"github.com/novox/mesh-host/internal/tunnel"
|
||||||
"github.com/novox/mesh-host/internal/upgrade"
|
"github.com/novox/mesh-host/internal/upgrade"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -53,15 +55,21 @@ const usage = `mesh-host — the node host
|
|||||||
profile what this machine can be asked to do
|
profile what this machine can be asked to do
|
||||||
inventory what this machine is, and what it holds
|
inventory what this machine is, and what it holds
|
||||||
apply FILE make this machine match a declaration from a file
|
apply FILE make this machine match a declaration from a file
|
||||||
reconcile make this machine match the declaration this host carries
|
reconcile make this machine match what the mesh last told it — or, before any
|
||||||
|
mesh has, the bundle this host carries
|
||||||
bundle show what this host carries
|
bundle show what this host carries
|
||||||
|
overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this
|
||||||
|
node's overlay key and the mesh is told, signed; --tunnel <iface> when several are up
|
||||||
owned what this host has applied and still owns
|
owned what this host has applied and still owns
|
||||||
version
|
version
|
||||||
|
|
||||||
--json machine-readable output
|
--json machine-readable output
|
||||||
--timeout how long any single probe may take (default 10s)
|
--timeout how long any single probe may take (default 10s)
|
||||||
--state where this node keeps what it knows (default /var/lib/mesh-host/state.json)
|
--state where this node keeps what it knows (default /var/lib/mesh-host/state.json)
|
||||||
--dry-run read the declaration and refuse it if wrong, but change nothing
|
--dry-run say what applying would change, and change nothing
|
||||||
|
|
||||||
|
Both apply and reconcile say what they would change before changing anything, and refuse a
|
||||||
|
declaration for the other mode than this node is in, or one older than what the mesh last said.
|
||||||
|
|
||||||
It connects to nothing and listens on nothing. What it applies comes from a file.
|
It connects to nothing and listens on nothing. What it applies comes from a file.
|
||||||
`
|
`
|
||||||
@@ -88,8 +96,11 @@ type options struct {
|
|||||||
state string
|
state string
|
||||||
token string
|
token string
|
||||||
nodeName string
|
nodeName string
|
||||||
|
tunnel string
|
||||||
dryRun bool
|
dryRun bool
|
||||||
file string
|
file string
|
||||||
|
// out is where what a command says goes. Stdout, and a buffer under test.
|
||||||
|
out io.Writer
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseArgs takes the subcommand first, then its flags.
|
// parseArgs takes the subcommand first, then its flags.
|
||||||
@@ -99,7 +110,7 @@ type options struct {
|
|||||||
// passed, silently ignored, with a successful exit. That is the fault this whole project keeps
|
// passed, silently ignored, with a successful exit. That is the fault this whole project keeps
|
||||||
// naming, so the parser takes the subcommand off the front and parses what follows.
|
// naming, so the parser takes the subcommand off the front and parses what follows.
|
||||||
func parseArgs(args []string) (string, options, error) {
|
func parseArgs(args []string) (string, options, error) {
|
||||||
opts := options{timeout: 10 * time.Second, state: store.DefaultPath}
|
opts := options{timeout: 10 * time.Second, state: store.DefaultPath, out: os.Stdout}
|
||||||
|
|
||||||
command := ""
|
command := ""
|
||||||
if len(args) > 0 {
|
if len(args) > 0 {
|
||||||
@@ -116,6 +127,8 @@ func parseArgs(args []string) (string, options, error) {
|
|||||||
set.BoolVar(&opts.dryRun, "dry-run", false, "read and check the declaration, change nothing")
|
set.BoolVar(&opts.dryRun, "dry-run", false, "read and check the declaration, change nothing")
|
||||||
set.StringVar(&opts.token, "token", "", "enrol: the one-time token, carried here by a person")
|
set.StringVar(&opts.token, "token", "", "enrol: the one-time token, carried here by a person")
|
||||||
set.StringVar(&opts.nodeName, "name", "", "enrol: override the name the token carries")
|
set.StringVar(&opts.nodeName, "name", "", "enrol: override the name the token carries")
|
||||||
|
set.StringVar(&opts.tunnel, "tunnel", "", "enrol, adopted: the found tunnel's interface whose key "+
|
||||||
|
"this node takes as its own; found by itself when one is up")
|
||||||
|
|
||||||
// Parsed in a loop, because the standard library stops at the FIRST non-flag argument.
|
// Parsed in a loop, because the standard library stops at the FIRST non-flag argument.
|
||||||
// `mesh-host inventory --json` hit that once, and taking the subcommand off the front
|
// `mesh-host inventory --json` hit that once, and taking the subcommand off the front
|
||||||
@@ -143,6 +156,13 @@ func parseArgs(args []string) (string, options, error) {
|
|||||||
opts.file = positionals[0]
|
opts.file = positionals[0]
|
||||||
return command, opts, nil
|
return command, opts, nil
|
||||||
}
|
}
|
||||||
|
if command == "overlay" {
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return "", opts, errors.New("overlay take [--tunnel <iface>]")
|
||||||
|
}
|
||||||
|
opts.file = positionals[0]
|
||||||
|
return command, opts, nil
|
||||||
|
}
|
||||||
// Anything left over was neither the command nor a flag. Refused rather than ignored: a
|
// Anything left over was neither the command nor a flag. Refused rather than ignored: a
|
||||||
// mistyped argument that changes nothing and reports success is worse than an error.
|
// mistyped argument that changes nothing and reports success is worse than an error.
|
||||||
if len(positionals) > 0 {
|
if len(positionals) > 0 {
|
||||||
@@ -184,18 +204,14 @@ func run(ctx context.Context, command string, opts options) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return runApply(ctx, opts, d, opts.file)
|
return runApply(ctx, opts, d, raw, fromFile)
|
||||||
|
|
||||||
case "reconcile":
|
case "reconcile":
|
||||||
// The first node's path. novox/hq ADR 0004: no mesh reachable means the declaration
|
d, raw, from, err := reconcileSource(opts)
|
||||||
// comes from the bundle the host carries. There is no link yet, so this is currently
|
|
||||||
// the only source — which is a stage, not a design, and saying so beats implying the
|
|
||||||
// other source exists.
|
|
||||||
d, err := bundle.Load(builtFor)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return runApply(ctx, opts, d, "the carried bundle")
|
return runApply(ctx, opts, d, raw, from)
|
||||||
|
|
||||||
case "bundle":
|
case "bundle":
|
||||||
if bundle.IsEmpty(builtFor) {
|
if bundle.IsEmpty(builtFor) {
|
||||||
@@ -230,6 +246,8 @@ func run(ctx context.Context, command string, opts options) error {
|
|||||||
|
|
||||||
case "enrol", "enroll":
|
case "enrol", "enroll":
|
||||||
return enrol(ctx, opts)
|
return enrol(ctx, opts)
|
||||||
|
case "overlay":
|
||||||
|
return overlayCommand(ctx, opts)
|
||||||
|
|
||||||
case "run":
|
case "run":
|
||||||
return runLink(ctx, opts)
|
return runLink(ctx, opts)
|
||||||
@@ -247,8 +265,10 @@ func run(ctx context.Context, command string, opts options) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeJSON(v any) error {
|
func writeJSON(v any) error { return writeJSONTo(os.Stdout, v) }
|
||||||
enc := json.NewEncoder(os.Stdout)
|
|
||||||
|
func writeJSONTo(w io.Writer, v any) error {
|
||||||
|
enc := json.NewEncoder(w)
|
||||||
enc.SetIndent("", " ")
|
enc.SetIndent("", " ")
|
||||||
return enc.Encode(v)
|
return enc.Encode(v)
|
||||||
}
|
}
|
||||||
@@ -305,20 +325,221 @@ func writeInventory(inv inventory.Inventory) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// runApply reads a declaration and makes the machine match it.
|
// provenance is where a declaration a command applies came from. It decides the origin its
|
||||||
|
// resources are recorded under, what it is held against, and what is recorded once it applied.
|
||||||
|
type provenance int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// fromFile is `apply FILE`: handed to the host by someone already running it as root.
|
||||||
|
fromFile provenance = iota
|
||||||
|
// fromBundle is `reconcile` on a machine the mesh has told nothing yet: the bundle carried in
|
||||||
|
// the binary, the first node's path before its mesh is up (novox/hq ADR 0004).
|
||||||
|
fromBundle
|
||||||
|
// fromDeclared is `reconcile` on a node the mesh has spoken to: what it last said, kept
|
||||||
|
// signed beside the state (declared.json), verified again before it is applied.
|
||||||
|
fromDeclared
|
||||||
|
)
|
||||||
|
|
||||||
|
// reconcileSource is which declaration `reconcile` holds this machine to.
|
||||||
|
//
|
||||||
|
// **What the mesh last said, never the bundle, once the mesh has said anything** (novox/hq issue
|
||||||
|
// 104). The bundle is right at genesis and stale a minute later — genesis rewrites it for the
|
||||||
|
// machine before applying it, and every declaration since came from the controller — and on an
|
||||||
|
// adopted node it is a converged declaration for a machine that is not converged. A node that
|
||||||
|
// has been told something and cannot prove it is the mesh's is refused, with the reason; the
|
||||||
|
// bundle is not applied in its place.
|
||||||
|
func reconcileSource(opts options) (*declaration.Declaration, []byte, provenance, error) {
|
||||||
|
path := store.DeclaredPath(opts.state)
|
||||||
|
_, err := store.ReadDeclared(path)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
mine, err := identity.Load(identity.Path(opts.state))
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, 0, fmt.Errorf("this node was told a declaration by the mesh, kept at %s, "+
|
||||||
|
"and cannot prove it is the mesh's: %w\n\nIt is not applied unproven, and the bundle "+
|
||||||
|
"this host carries is not applied in its place: that was consumed at genesis", path, err)
|
||||||
|
}
|
||||||
|
raw, err := store.LoadDeclared(path, mine.Membership.Signer)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, 0, fmt.Errorf("%w\n\nThe bundle this host carries is not applied in its "+
|
||||||
|
"place: that was consumed at genesis", err)
|
||||||
|
}
|
||||||
|
d, err := declaration.Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, 0, err
|
||||||
|
}
|
||||||
|
return d, raw, fromDeclared, nil
|
||||||
|
case errors.Is(err, store.ErrNothingDeclared):
|
||||||
|
d, err := bundle.Load(builtFor)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, 0, err
|
||||||
|
}
|
||||||
|
return d, bundle.Raw(builtFor), fromBundle, nil
|
||||||
|
default:
|
||||||
|
return nil, nil, 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p provenance) origin() string {
|
||||||
|
if p == fromDeclared {
|
||||||
|
return store.OriginDeclared
|
||||||
|
}
|
||||||
|
// A file handed to the host is applied as the bundle is: what it puts here is invisible to
|
||||||
|
// the removal pass of a declaration that later arrives from the mesh (novox/hq issue 010).
|
||||||
|
return store.OriginCarried
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p provenance) name(opts options) string {
|
||||||
|
switch p {
|
||||||
|
case fromBundle:
|
||||||
|
return "the carried bundle"
|
||||||
|
case fromDeclared:
|
||||||
|
return "what the mesh last told this node (" + store.DeclaredName + ")"
|
||||||
|
}
|
||||||
|
return opts.file
|
||||||
|
}
|
||||||
|
|
||||||
|
// short is a digest as a person reads one aloud.
|
||||||
|
func short(digest string) string {
|
||||||
|
if len(digest) > 12 {
|
||||||
|
return digest[:12]
|
||||||
|
}
|
||||||
|
return digest
|
||||||
|
}
|
||||||
|
|
||||||
|
// refuseStale refuses a declaration that is not the one this node should be held to (novox/hq
|
||||||
|
// issue 104), naming both.
|
||||||
|
//
|
||||||
|
// **A declaration carries no order.** The controller signs a version — the vocabulary — the
|
||||||
|
// node it is for, the mode, and the resources: no sequence, no issued-at. What exists is
|
||||||
|
// identity: the mesh names what it sends by the digest of the bytes, the node keeps the last one
|
||||||
|
// it was sent, and genesis records the digest of what it consumed. So the bundle is held to
|
||||||
|
// genesis's digest, and a file is not applied at all once the mesh has spoken: a file is applied
|
||||||
|
// as the bundle is, its resources recorded as this machine's own — so the mesh could never remove
|
||||||
|
// them again — and everything the mesh declared read as no longer declared and removed. Even the
|
||||||
|
// very declaration the mesh last sent, applied from a file, would plan to remove the foundation.
|
||||||
|
// `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so.
|
||||||
|
func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance) error {
|
||||||
|
switch from {
|
||||||
|
case fromDeclared:
|
||||||
|
return nil
|
||||||
|
case fromBundle:
|
||||||
|
if known.Genesis == nil || known.Genesis.Digest == digest {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// By digest. Genesis applies the bundle rewritten for this machine — its foundation
|
||||||
|
// ports, root credentials, mode — and writes that lock out; a host built from the
|
||||||
|
// carried template does not match it, and one built from the written lock does.
|
||||||
|
return fmt.Errorf("the bundle this host carries (%s) is not the one genesis applied here on %s "+
|
||||||
|
"(%s), which was the bundle rewritten for this machine. It was consumed then, and nothing "+
|
||||||
|
"the mesh has said is kept on this node yet, so there is nothing to reconcile against: "+
|
||||||
|
"enrol this node, or push to it from the controller",
|
||||||
|
short(digest), known.Genesis.At.Format(time.RFC3339), short(known.Genesis.Digest))
|
||||||
|
}
|
||||||
|
// A file.
|
||||||
|
switch {
|
||||||
|
case errors.Is(keptErr, store.ErrNothingDeclared):
|
||||||
|
// The mesh has said nothing here: a machine a file is for.
|
||||||
|
return nil
|
||||||
|
case keptErr != nil:
|
||||||
|
return fmt.Errorf("%w\n\nNothing is applied over what this node cannot read back", keptErr)
|
||||||
|
}
|
||||||
|
last := apply.DigestOf(kept.Declaration)
|
||||||
|
what := fmt.Sprintf("this file (%s) is not it", short(digest))
|
||||||
|
switch {
|
||||||
|
case digest == last:
|
||||||
|
what = "this file is that declaration, and from a file it would still be applied as a bundle is"
|
||||||
|
case known.Genesis != nil && digest == known.Genesis.Digest:
|
||||||
|
what = fmt.Sprintf("this file is the bundle genesis consumed on %s (%s), older than it",
|
||||||
|
known.Genesis.At.Format(time.RFC3339), short(digest))
|
||||||
|
}
|
||||||
|
return fmt.Errorf("`apply FILE` is for a machine the mesh has not spoken to. The mesh has told this "+
|
||||||
|
"node declaration %s, kept as %s, and %s. A file is applied as the bundle is — its resources "+
|
||||||
|
"recorded as this machine's own, which the mesh could then never remove, and what the mesh "+
|
||||||
|
"declared read as no longer declared — so no file is applied here: `reconcile` applies what "+
|
||||||
|
"the mesh last said, and `push` from the controller changes it",
|
||||||
|
short(last), store.DeclaredName, what)
|
||||||
|
}
|
||||||
|
|
||||||
|
// applied is what an apply says in machine-readable form: what it planned, then what it did.
|
||||||
|
type applied struct {
|
||||||
|
Plan []apply.Step `json:"plan"`
|
||||||
|
// Report is absent on a dry run, which is the plan and nothing more.
|
||||||
|
Report *apply.Report `json:"report,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// runApply makes the machine match a declaration — after refusing one this node must not apply,
|
||||||
|
// and after saying what it would change.
|
||||||
|
//
|
||||||
|
// Refused first, and before anything is read from the machine: a declaration for the other
|
||||||
|
// mode than this node is in, or one older than what the mesh last said (novox/hq issue 104).
|
||||||
|
// Then the plan, printed whole before a single resource is touched; `--dry-run` is that and
|
||||||
|
// nothing more.
|
||||||
//
|
//
|
||||||
// The state is loaded before anything is touched and saved after, including when the apply
|
// The state is loaded before anything is touched and saved after, including when the apply
|
||||||
// fails part-way: what was applied before the failure is on the machine, and a host that did
|
// fails part-way: what was applied before the failure is on the machine, and a host that did
|
||||||
// not record it would believe it owns less than it does and leave that behind forever.
|
// not record it would believe it owns less than it does and leave that behind forever.
|
||||||
func runApply(ctx context.Context, opts options, d *declaration.Declaration, source string) error {
|
func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw []byte, from provenance) error {
|
||||||
|
out := opts.out
|
||||||
|
if out == nil {
|
||||||
|
out = os.Stdout
|
||||||
|
}
|
||||||
|
// One apply at a time on this machine, whichever process asks: the link service applies too,
|
||||||
|
// and two saves of the state interleaved lose what one of them recorded.
|
||||||
|
unlock, err := store.Lock(opts.state, func() {
|
||||||
|
fmt.Fprintln(out, "another apply holds this node's state — mesh-host run, or the installer; waiting for it to finish")
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
known, err := store.Load(opts.state)
|
known, err := store.Load(opts.state)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
source, origin, digest := from.name(opts), from.origin(), apply.DigestOf(raw)
|
||||||
|
|
||||||
|
// The mode this node is in. What the mesh last said is signed and was verified on load; the
|
||||||
|
// state's note of it is this host's own, and where they disagree the note is what is wrong
|
||||||
|
// — a genesis re-run over a node the mesh converged since, a state saved when the kept
|
||||||
|
// declaration could not be — and is repaired, not obeyed. A bundle or a file is held to the
|
||||||
|
// note, or to the kept declaration when the note predates it.
|
||||||
|
kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state))
|
||||||
|
if from == fromDeclared {
|
||||||
|
if known.Mode != "" && known.Mode != apply.ModeOf(d) {
|
||||||
|
fmt.Fprintf(out, "this node's record said %s; what the mesh last said, signed, says %s — the record is repaired\n",
|
||||||
|
known.Mode, apply.ModeOf(d))
|
||||||
|
}
|
||||||
|
known.Mode = apply.ModeOf(d)
|
||||||
|
} else if known.Mode == "" && keptErr == nil {
|
||||||
|
if last, err := declaration.Parse(kept.Declaration); err == nil {
|
||||||
|
known.Mode = apply.ModeOf(last)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := apply.CheckMode(known, d); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := refuseStale(known, kept, keptErr, digest, from); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Said before anything is done.
|
||||||
|
steps := apply.Plan(d, known, origin)
|
||||||
|
if opts.json && opts.dryRun {
|
||||||
|
return writeJSONTo(out, applied{Plan: steps})
|
||||||
|
}
|
||||||
|
mode := known.Mode
|
||||||
|
if mode == "" {
|
||||||
|
mode = "in no mode yet — nothing has said one"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "%s (%s): %d resource(s), version %d\n", source, short(digest), len(d.Resources), d.Version)
|
||||||
|
fmt.Fprintf(out, "this node is %s; the declaration says %s\n", mode, apply.ModeOf(d))
|
||||||
|
fmt.Fprintf(out, "\nwould change, in this order:\n")
|
||||||
|
for _, step := range steps {
|
||||||
|
fmt.Fprintln(out, " "+step.String())
|
||||||
|
}
|
||||||
if opts.dryRun {
|
if opts.dryRun {
|
||||||
fmt.Printf("%s: %d resource(s), version %d — accepted, nothing applied\n",
|
fmt.Fprintf(out, "\n--dry-run: nothing applied\n")
|
||||||
source, len(d.Resources), d.Version)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -338,12 +559,20 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, sou
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried,
|
fmt.Fprintf(out, "\napplying:\n")
|
||||||
|
report, updated, applyErr := apply.ApplyKeeping(ctx, sys, d, known, origin,
|
||||||
apply.ExecRunner, func(line string) {
|
apply.ExecRunner, func(line string) {
|
||||||
if !opts.json {
|
if !opts.json {
|
||||||
fmt.Println(line)
|
fmt.Fprintln(out, line)
|
||||||
}
|
}
|
||||||
}, sealOpener(opts.state))
|
}, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
|
||||||
|
|
||||||
|
// What this apply settles about the node, whichever way it went. The mode is what the
|
||||||
|
// declaration said and the check above agreed with; the bundle, once applied, is consumed.
|
||||||
|
updated.Mode = apply.ModeOf(d)
|
||||||
|
if from == fromBundle {
|
||||||
|
updated.Genesis = &store.Genesis{Digest: digest, At: time.Now().UTC()}
|
||||||
|
}
|
||||||
|
|
||||||
// Saved whichever way it went. Recording only on success would lose the footprint of a
|
// Saved whichever way it went. Recording only on success would lose the footprint of a
|
||||||
// failed apply, and that footprint is on the machine either way.
|
// failed apply, and that footprint is on the machine either way.
|
||||||
@@ -380,13 +609,13 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, sou
|
|||||||
}
|
}
|
||||||
|
|
||||||
if opts.json {
|
if opts.json {
|
||||||
return writeJSON(report)
|
return writeJSONTo(out, applied{Plan: steps, Report: &report})
|
||||||
}
|
}
|
||||||
if !report.Changed() {
|
if !report.Changed() {
|
||||||
fmt.Printf("%s: already matches — %d resource(s) checked\n", source, len(report.Outcomes))
|
fmt.Fprintf(out, "%s: already matches — %d resource(s) checked\n", source, len(report.Outcomes))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
fmt.Printf("%s: applied — %d resource(s)\n", source, len(report.Outcomes))
|
fmt.Fprintf(out, "%s: applied — %d resource(s)\n", source, len(report.Outcomes))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -478,14 +707,39 @@ func enrol(ctx context.Context, opts options) error {
|
|||||||
}
|
}
|
||||||
fmt.Printf("generated this node's identity: %s\n", mine.PublicBase64())
|
fmt.Printf("generated this node's identity: %s\n", mine.PublicBase64())
|
||||||
|
|
||||||
// Its key on the private network, generated here and now for the same reason: the private
|
// Its key on the private network. Generated here and now, for the same reason: the private
|
||||||
// half must never have been anywhere else. The mesh receives only the public half and uses it
|
// half must never have been anywhere else. The mesh receives only the public half and uses it
|
||||||
// to compute a graph it cannot impersonate.
|
// to compute a graph it cannot impersonate.
|
||||||
mine.Overlay, err = identity.GenerateOverlayKey()
|
//
|
||||||
if err != nil {
|
// **Except on an adopted node with a tunnel** (novox/hq ADR 0105): the found interface's key
|
||||||
return err
|
// becomes this node's, so the peers that know the tunnel by that key keep reaching it once
|
||||||
|
// the mesh's interface takes the tunnel over. The one case where the mesh takes a credential
|
||||||
|
// it did not mint — read from the found configuration, written where a generated one is
|
||||||
|
// written, never printed, never sent.
|
||||||
|
var found *link.Tunnel
|
||||||
|
if token.Adopted {
|
||||||
|
tun, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, tunnel.ErrNone):
|
||||||
|
fmt.Println("no tunnel is up on this machine; the private network's key is generated")
|
||||||
|
case err != nil:
|
||||||
|
return err
|
||||||
|
default:
|
||||||
|
mine.Overlay, err = identity.OverlayKeyFrom(tun.PrivateKey())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
found = carried(tun)
|
||||||
|
fmt.Printf("this node's overlay key is the found tunnel's (%s): %s\n", tun, mine.Overlay.Public)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if found == nil {
|
||||||
|
mine.Overlay, err = identity.GenerateOverlayKey()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("generated this node's overlay key: %s\n", mine.Overlay.Public)
|
||||||
}
|
}
|
||||||
fmt.Printf("generated this node's overlay key: %s\n", mine.Overlay.Public)
|
|
||||||
|
|
||||||
// And the key secrets are sealed to. Here, with the others, because the mesh cannot seal
|
// And the key secrets are sealed to. Here, with the others, because the mesh cannot seal
|
||||||
// anything to a key it has not been told about — a key made later would leave a node that
|
// anything to a key it has not been told about — a key made later would leave a node that
|
||||||
@@ -518,8 +772,14 @@ func enrol(ctx context.Context, opts options) error {
|
|||||||
// who knows its public key (novox/hq issue 083).
|
// who knows its public key (novox/hq issue 083).
|
||||||
proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
|
proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
|
||||||
sealing.Public, serving.Public))
|
sealing.Public, serving.Public))
|
||||||
reply, err := link.Enrol(ctx, token.Broker, token.Fingerprint, *name, token.Secret,
|
// The token says where to go and which certificate that address must present. It says nothing
|
||||||
mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, opts.timeout)
|
// about which bus is there, and does not need to: every token names the one the mesh runs on
|
||||||
|
// today until the rollout (novox/hq ADR 0116 step 5), and that is what an empty Transport is.
|
||||||
|
reply, err := link.Enrol(ctx,
|
||||||
|
link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint},
|
||||||
|
*name, token.Secret,
|
||||||
|
mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found,
|
||||||
|
opts.timeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -578,6 +838,91 @@ func enrol(ctx context.Context, opts options) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// overlayCommand is `mesh-host overlay take`: this node takes the tunnel found on its machine over
|
||||||
|
// after it enrolled (novox/hq ADR 0105). For a node that enrolled before the mesh knew to take a
|
||||||
|
// tunnel over — re-enrolling would rotate its identity, sealing and serving keys, and with them
|
||||||
|
// every credential the mesh sealed to it.
|
||||||
|
func overlayCommand(ctx context.Context, opts options) error {
|
||||||
|
if opts.file != "take" {
|
||||||
|
return errors.New("overlay take [--tunnel <iface>] — the one thing `overlay` does here")
|
||||||
|
}
|
||||||
|
identityPath := identity.Path(opts.state)
|
||||||
|
mine, err := identity.Load(identityPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
found, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w. Nothing was changed", err)
|
||||||
|
}
|
||||||
|
taken, rekey, err := rekeyOnto(mine, found)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("taking over %s: this node's overlay key becomes the tunnel's, %s\n", found, taken.Overlay.Public)
|
||||||
|
fmt.Printf(" identity, sealing and serving keys are untouched\n")
|
||||||
|
|
||||||
|
// Told first, then written: a mesh told and a machine not yet written is put right by running
|
||||||
|
// this again (the mesh refuses the stale second rekey and changes nothing; the files are
|
||||||
|
// rewritten the same). A machine written and a mesh not told would raise the mesh's interface
|
||||||
|
// on a key the mesh does not know at the next restart.
|
||||||
|
if err := link.Publish(ctx, link.Membership{
|
||||||
|
Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint,
|
||||||
|
Password: mine.Membership.Password, Signer: mine.Membership.Signer,
|
||||||
|
Transport: mine.Membership.Transport,
|
||||||
|
}, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil {
|
||||||
|
return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err)
|
||||||
|
}
|
||||||
|
fmt.Printf(" told the mesh signed rekey sent; `node show %s` on the controller says whether it took\n", mine.Node)
|
||||||
|
|
||||||
|
if err := os.WriteFile(identity.OverlayKeyPath(opts.state),
|
||||||
|
[]byte(taken.Overlay.Private+"\n"), 0o600); err != nil {
|
||||||
|
return fmt.Errorf("the mesh was told and this node's overlay key could not be written: %w — run this again", err)
|
||||||
|
}
|
||||||
|
if err := identity.Save(identityPath, taken); err != nil {
|
||||||
|
return fmt.Errorf("the mesh was told and this node's identity could not be saved: %w — run this again", err)
|
||||||
|
}
|
||||||
|
fmt.Printf(" written %s and the identity; the mesh's interface reads the key when the next push restarts it\n",
|
||||||
|
identity.OverlayKeyPath(opts.state))
|
||||||
|
fmt.Printf(" next on the controller: `overlay place %s --hub --endpoint <host>:%d …`, `plan %s --json`, then push\n",
|
||||||
|
mine.Node, found.Port, mine.Node)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rekeyOnto is the identity with the found tunnel's key as its overlay key, and the signed rekey
|
||||||
|
// that tells the mesh. Pure, so it can be held to: node, sealing and serving keys are the same
|
||||||
|
// bytes in and out; only the overlay key moves. Run again after a take, the previous key it names
|
||||||
|
// is the one before the take, so the mesh can tell a repeat from a replay.
|
||||||
|
func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, link.Rekey, error) {
|
||||||
|
overlay, err := identity.OverlayKeyFrom(found.PrivateKey())
|
||||||
|
if err != nil {
|
||||||
|
return identity.Identity{}, link.Rekey{}, err
|
||||||
|
}
|
||||||
|
previous := mine.Overlay.Public
|
||||||
|
if previous == overlay.Public && mine.OverlayBefore != "" {
|
||||||
|
previous = mine.OverlayBefore
|
||||||
|
}
|
||||||
|
taken := mine
|
||||||
|
taken.Overlay = overlay
|
||||||
|
if previous != overlay.Public {
|
||||||
|
taken.OverlayBefore = previous
|
||||||
|
}
|
||||||
|
presented := carried(found)
|
||||||
|
rekey := link.Rekey{Previous: previous, OverlayKey: overlay.Public, Tunnel: presented}
|
||||||
|
rekey.Proof = mine.Sign(link.RekeyProof(mine.Node, previous, overlay.Public, presented))
|
||||||
|
return taken, rekey, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// carried is a found tunnel as it is presented to the mesh: everything but its private key.
|
||||||
|
func carried(t tunnel.Found) *link.Tunnel {
|
||||||
|
out := &link.Tunnel{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port,
|
||||||
|
Address: t.Address, Range: t.Range, MTU: t.MTU, PublicKey: t.PublicKey}
|
||||||
|
for _, p := range t.Peers {
|
||||||
|
out.Peers = append(out.Peers, link.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func firstNonEmpty(values ...string) string {
|
func firstNonEmpty(values ...string) string {
|
||||||
for _, v := range values {
|
for _, v := range values {
|
||||||
if strings.TrimSpace(v) != "" {
|
if strings.TrimSpace(v) != "" {
|
||||||
@@ -629,7 +974,12 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
go sched.Run(ctx)
|
go sched.Run(ctx)
|
||||||
|
|
||||||
applier := func(ctx context.Context, raw, signature []byte) link.Report {
|
applier := func(ctx context.Context, raw, signature []byte) link.Report {
|
||||||
return applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched)
|
report := applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
|
||||||
|
// **A declaration may carry this machine's membership for another bus.** It arrives as a
|
||||||
|
// sealed file like any secret, and is read after the rest has applied so the bus it names is
|
||||||
|
// standing before this machine leaves the one it is on (novox/hq design 28, task 5.2).
|
||||||
|
adoptDeliveredMembership(identity.Path(opts.state), &mine, say)
|
||||||
|
return report
|
||||||
}
|
}
|
||||||
|
|
||||||
// Two things at once, and the second is what makes disconnection ordinary. The link brings
|
// Two things at once, and the second is what makes disconnection ordinary. The link brings
|
||||||
@@ -664,6 +1014,7 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
Broker: mine.Membership.Broker,
|
Broker: mine.Membership.Broker,
|
||||||
Fingerprint: mine.Membership.Fingerprint,
|
Fingerprint: mine.Membership.Fingerprint,
|
||||||
Password: mine.Membership.Password,
|
Password: mine.Membership.Password,
|
||||||
|
Transport: mine.Membership.Transport,
|
||||||
Signer: mine.Membership.Signer,
|
Signer: mine.Membership.Signer,
|
||||||
}, applier, say, opts.timeout, rousedBySignal(ctx), outbox)
|
}, applier, say, opts.timeout, rousedBySignal(ctx), outbox)
|
||||||
}
|
}
|
||||||
@@ -792,7 +1143,7 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
report := applyDeclared(ctx, opts, declared, sched)
|
report := applyDeclared(ctx, opts, declared, sched, say)
|
||||||
// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
|
// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
|
||||||
// its firewall changed, because that is how a predecessor still writing is caught
|
// its firewall changed, because that is how a predecessor still writing is caught
|
||||||
// (novox/hq ADR 0100); publish decides whether anything did.
|
// (novox/hq ADR 0100); publish decides whether anything did.
|
||||||
@@ -813,31 +1164,58 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
|
|||||||
// Signature checking happens before this is called, in the link. By the time anything here runs,
|
// Signature checking happens before this is called, in the link. By the time anything here runs,
|
||||||
// the question "is this from the mesh I joined" is settled — which is why this can treat the
|
// the question "is this from the mesh I joined" is settled — which is why this can treat the
|
||||||
// bytes as instructions.
|
// bytes as instructions.
|
||||||
func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.Scheduler) link.Report {
|
func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.Scheduler, say link.Announce) link.Report {
|
||||||
return applyAndKeep(ctx, opts, raw, nil, sched)
|
return applyAndKeep(ctx, opts, raw, nil, sched, say)
|
||||||
}
|
}
|
||||||
|
|
||||||
// applying serialises applies on this node.
|
// applying serialises applies within this process.
|
||||||
//
|
//
|
||||||
// **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
|
// **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
|
||||||
// acts on the machine, and writes the state back. Run at the same time they interleave, and the
|
// acts on the machine, and writes the state back. Run at the same time they interleave, and the
|
||||||
// one that saves last writes a state read before the other acted — losing what the first recorded:
|
// one that saves last writes a state read before the other acted — losing what the first recorded:
|
||||||
// a hold, the firewall found here, a resource just applied. The machine would then be one thing
|
// a hold, the firewall found here, a resource just applied. The machine would then be one thing
|
||||||
// and its record another, which is the fault every read-back in this package exists to prevent.
|
// and its record another, which is the fault every read-back in this package exists to prevent.
|
||||||
|
// Across processes — `reconcile` run by hand beside this service — the lock beside the state does
|
||||||
|
// the same (store.Lock).
|
||||||
var applying sync.Mutex
|
var applying sync.Mutex
|
||||||
|
|
||||||
// applyAndKeep applies a declaration and, when it came from the mesh, keeps it so this node can
|
// applyAndKeep applies a declaration and, when it came from the mesh, keeps it so this node can
|
||||||
// go on obeying it while disconnected. One at a time, whoever asks.
|
// go on obeying it while disconnected. One at a time, whoever asks. Given unsigned, the bytes are
|
||||||
|
// what this node kept, already verified against the mesh's key on load.
|
||||||
func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared,
|
func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared,
|
||||||
sched *apply.Scheduler) link.Report {
|
sched *apply.Scheduler, say link.Announce) link.Report {
|
||||||
applying.Lock()
|
applying.Lock()
|
||||||
defer applying.Unlock()
|
defer applying.Unlock()
|
||||||
|
unlock, err := store.Lock(opts.state, nil)
|
||||||
|
if err != nil {
|
||||||
|
return link.Report{Refused: err.Error()}
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
declared, err := declaration.Parse(raw)
|
declared, err := declaration.Parse(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return link.Report{Refused: err.Error()}
|
return link.Report{Refused: err.Error()}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
known, err := store.Load(opts.state)
|
||||||
|
if err != nil {
|
||||||
|
return link.Report{Refused: err.Error()}
|
||||||
|
}
|
||||||
|
// A declaration from the mesh is the controller's word on the node's mode — the flip arrives
|
||||||
|
// as exactly that, the first converged declaration after adopted ones — and becomes the
|
||||||
|
// record. So does what this node kept: it is signed by the mesh and verified on load, where
|
||||||
|
// the state's note of the mode is this host's own. Where they disagree the note is wrong — a
|
||||||
|
// genesis re-run over a node the mesh converged since, a state saved when the kept declaration
|
||||||
|
// could not be — and it is repaired and said, not obeyed: refusing would hold this node off what
|
||||||
|
// the mesh said until a delivery that comes only when something changes (novox/hq issue 104).
|
||||||
|
if signed == nil && known.Mode != "" && known.Mode != apply.ModeOf(declared) {
|
||||||
|
if say != nil {
|
||||||
|
say(fmt.Sprintf("this node's record said %s; what the mesh last said, signed, says %s — the record is repaired",
|
||||||
|
known.Mode, apply.ModeOf(declared)))
|
||||||
|
}
|
||||||
|
known.Mode = apply.ModeOf(declared)
|
||||||
|
}
|
||||||
|
|
||||||
built, err := system.For(builtFor)
|
built, err := system.For(builtFor)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return link.Report{Refused: err.Error()}
|
return link.Report{Refused: err.Error()}
|
||||||
@@ -846,11 +1224,6 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
|
|||||||
return link.Report{Refused: err.Error()}
|
return link.Report{Refused: err.Error()}
|
||||||
}
|
}
|
||||||
|
|
||||||
known, err := store.Load(opts.state)
|
|
||||||
if err != nil {
|
|
||||||
return link.Report{Refused: err.Error()}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := built.Confirm(ctx, apply.ExecRunner); err != nil {
|
if err := built.Confirm(ctx, apply.ExecRunner); err != nil {
|
||||||
return link.Report{Refused: err.Error()}
|
return link.Report{Refused: err.Error()}
|
||||||
}
|
}
|
||||||
@@ -860,6 +1233,10 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
|
|||||||
outcome, updated, applyErr := apply.ApplyKeeping(ctx, built, declared, known, store.OriginDeclared,
|
outcome, updated, applyErr := apply.ApplyKeeping(ctx, built, declared, known, store.OriginDeclared,
|
||||||
apply.ExecRunner, nil, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
|
apply.ExecRunner, nil, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
|
||||||
|
|
||||||
|
// The mode the mesh said, recorded whichever way the apply went: the declaration is kept
|
||||||
|
// either way, and the node is held to it from the next reconcile (novox/hq ADR 0100).
|
||||||
|
updated.Mode = apply.ModeOf(declared)
|
||||||
|
|
||||||
// Saved whichever way it went. Recording only on success would lose the footprint of a
|
// Saved whichever way it went. Recording only on success would lose the footprint of a
|
||||||
// failed apply, and that footprint is on the machine either way.
|
// failed apply, and that footprint is on the machine either way.
|
||||||
if saveErr := store.Save(opts.state, updated); saveErr != nil {
|
if saveErr := store.Save(opts.state, updated); saveErr != nil {
|
||||||
@@ -891,6 +1268,11 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
|
|||||||
if updated.Firewall != nil {
|
if updated.Firewall != nil {
|
||||||
report.Firewall = updated.Firewall.Kind
|
report.Firewall = updated.Firewall.Kind
|
||||||
}
|
}
|
||||||
|
// And the tunnel the private network took over, as this apply found it (novox/hq ADR 0105).
|
||||||
|
if t := outcome.Tunnel; t != nil {
|
||||||
|
report.Tunnel = &link.CarriedTunnel{Interface: t.Interface, Port: t.Port, Range: t.Range,
|
||||||
|
Peers: t.Peers, State: t.State, Note: t.Note, Kept: t.Kept}
|
||||||
|
}
|
||||||
reached, err := reachable.Collect(ctx, apply.ExecRunner)
|
reached, err := reachable.Collect(ctx, apply.ExecRunner)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read what is reachable here: %v\n", err)
|
fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read what is reachable here: %v\n", err)
|
||||||
@@ -1001,3 +1383,55 @@ func carriedPorts(state store.State) []int {
|
|||||||
sort.Ints(out)
|
sort.Ints(out)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MembershipNextPath is where the mesh delivers this machine's membership for the bus it is moving
|
||||||
|
// to: a sealed file in a declaration, written by the host like any secret, read here after the
|
||||||
|
// declaration has applied.
|
||||||
|
const MembershipNextPath = "/var/lib/mesh/membership-next.json"
|
||||||
|
|
||||||
|
// adoptDeliveredMembership moves this machine to the bus a delivered membership names.
|
||||||
|
//
|
||||||
|
// **Saved, then restarted — not swapped in place.** The link holds one membership for the life of
|
||||||
|
// the process, and every reconnect path assumes the bus did not change under it; a process that
|
||||||
|
// found itself half on one bus and half on another would be a new kind of state nothing was written
|
||||||
|
// for. Exiting cleanly hands the machine to the service manager's restart, and the process that
|
||||||
|
// comes back reads the identity file the way it always has and dials the bus it names. That is the
|
||||||
|
// same path a machine takes after a reboot, which is why nothing new has to be right for it to work.
|
||||||
|
//
|
||||||
|
// A membership identical to the one held is nothing: the file stays and is read again next time.
|
||||||
|
func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say func(string)) {
|
||||||
|
raw, err := os.ReadFile(MembershipNextPath)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var next identity.Membership
|
||||||
|
if err := json.Unmarshal(raw, &next); err != nil {
|
||||||
|
say(fmt.Sprintf("a membership was delivered at %s and could not be read: %v", MembershipNextPath, err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if next.Broker == "" || next.Password == "" || next.Fingerprint == "" {
|
||||||
|
say(fmt.Sprintf("a membership was delivered at %s with no broker, fingerprint or password; ignored", MembershipNextPath))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
same := next.Broker == mine.Membership.Broker && next.Fingerprint == mine.Membership.Fingerprint &&
|
||||||
|
next.Password == mine.Membership.Password && next.Transport == mine.Membership.Transport
|
||||||
|
if same {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// The signer is the mesh's, not the bus's: a delivered membership that names none keeps the one
|
||||||
|
// this machine already trusts, because a change of bus is not a change of who signs declarations.
|
||||||
|
if len(next.Signer) == 0 {
|
||||||
|
next.Signer = mine.Membership.Signer
|
||||||
|
}
|
||||||
|
mine.Membership = next
|
||||||
|
if err := identity.Save(identityPath, *mine); err != nil {
|
||||||
|
say(fmt.Sprintf("a membership for another bus was delivered and could not be saved: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
transport := next.Transport
|
||||||
|
if transport == "" {
|
||||||
|
transport = "the current"
|
||||||
|
}
|
||||||
|
say(fmt.Sprintf("moving to %s bus at %s — restarting to dial it", transport, next.Broker))
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
|||||||
+240
-1
@@ -1,14 +1,21 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/apply"
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/bundle"
|
||||||
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
"github.com/novox/mesh-host/internal/link"
|
"github.com/novox/mesh-host/internal/link"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
"github.com/novox/mesh-host/internal/system"
|
"github.com/novox/mesh-host/internal/system"
|
||||||
@@ -218,7 +225,7 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) {
|
|||||||
// Whatever else is applying — the link, while this is the reconcile — this waits for it.
|
// Whatever else is applying — the link, while this is the reconcile — this waits for it.
|
||||||
applying.Lock()
|
applying.Lock()
|
||||||
done := make(chan link.Report, 1)
|
done := make(chan link.Report, 1)
|
||||||
go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil) }()
|
go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil, nil) }()
|
||||||
select {
|
select {
|
||||||
case report := <-done:
|
case report := <-done:
|
||||||
applying.Unlock()
|
applying.Unlock()
|
||||||
@@ -262,3 +269,235 @@ func TestAChangeThatNeverReachedTheMeshIsSaidAgain(t *testing.T) {
|
|||||||
t.Error("a change the mesh was told was said again")
|
t.Error("a change the mesh was told was said again")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Defends novox/hq issue 104: a declaration for the other mode than this node is in is refused at
|
||||||
|
// the point of application, whichever command delivered it, naming both — an adopted control-node
|
||||||
|
// once applied its converged genesis bundle and closed itself for forty-five minutes.
|
||||||
|
|
||||||
|
func stateWithMode(t *testing.T, mode string) options {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
opts := options{state: filepath.Join(dir, "state.json"), out: &bytes.Buffer{}}
|
||||||
|
if err := store.Save(opts.state, store.State{Mode: mode}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return opts
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAConvergedDeclarationIsRefusedOnAnAdoptedNode(t *testing.T) {
|
||||||
|
opts := stateWithMode(t, store.ModeAdopted)
|
||||||
|
path := filepath.Join(filepath.Dir(opts.state), "filter.conf")
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[{"id":"filter","type":"file","path":"` + path +
|
||||||
|
`","content":"table inet filter { chain input { policy drop; } }\n"}]}`)
|
||||||
|
d, err := declaration.ParseFileTrusted(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, from := range []provenance{fromFile, fromBundle} {
|
||||||
|
err := runApply(context.Background(), opts, d, raw, from)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("a converged declaration was applied to an adopted node (from %d)", from)
|
||||||
|
}
|
||||||
|
want := "this node is adopted; the declaration says converged"
|
||||||
|
if !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "`converge`") {
|
||||||
|
t.Errorf("the refusal does not name both modes and the act that changes it: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Error("the refused declaration touched the machine")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheKeptDeclarationRepairsARecordThatDisagrees(t *testing.T) {
|
||||||
|
// What a node kept is signed by the mesh and verified on load; the state's note of the mode is
|
||||||
|
// the host's own. A genesis re-run after `converge`, or a state saved when the kept declaration
|
||||||
|
// could not be, leaves them apart — and a loop that refused every five minutes would hold the
|
||||||
|
// node off what the mesh said until a delivery that comes only when something changes. The
|
||||||
|
// kept declaration wins, and the repair is said.
|
||||||
|
opts := stateWithMode(t, store.ModeConverged)
|
||||||
|
raw := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"` +
|
||||||
|
filepath.Join(filepath.Dir(opts.state), "a.conf") + `","content":"x\n"}]}`)
|
||||||
|
var said []string
|
||||||
|
report := applyAndKeep(context.Background(), opts, raw, nil, nil, func(line string) { said = append(said, line) })
|
||||||
|
if strings.Contains(report.Refused, "the declaration says") {
|
||||||
|
t.Fatalf("what the node kept was refused against its own note: %s", report.Refused)
|
||||||
|
}
|
||||||
|
if len(said) != 1 || !strings.Contains(said[0], "record said converged") ||
|
||||||
|
!strings.Contains(said[0], "says adopted") || !strings.Contains(said[0], "repaired") {
|
||||||
|
t.Errorf("the repair was not said: %q", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheMeshItselfMayChangeTheMode(t *testing.T) {
|
||||||
|
// The flip is a declaration: `converge` on the controller records the mode and sends the
|
||||||
|
// first converged declaration. Delivered by the link, signed, it is not held to the record —
|
||||||
|
// it becomes it. (With no system linked in, the apply is refused later for that; what this
|
||||||
|
// checks is that the refusal is not the mode's.)
|
||||||
|
opts := stateWithMode(t, store.ModeAdopted)
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` +
|
||||||
|
filepath.Join(filepath.Dir(opts.state), "a.conf") + `","content":"x\n"}]}`)
|
||||||
|
report := applyAndKeep(context.Background(), opts, raw, &store.Declared{Declaration: raw}, nil, nil)
|
||||||
|
if strings.Contains(report.Refused, "the declaration says") {
|
||||||
|
t.Errorf("the mesh's own flip was refused for its mode: %s", report.Refused)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Defends novox/hq issue 104: a declaration older than what the mesh last said is refused, naming
|
||||||
|
// both — and `apply FILE` is refused altogether once the mesh has spoken, the last declaration
|
||||||
|
// itself included: from a file it is applied as the bundle is, which would record the mesh's
|
||||||
|
// resources as this machine's own and remove the foundation as undeclared.
|
||||||
|
func TestAnOlderDeclarationIsRefused(t *testing.T) {
|
||||||
|
opts := stateWithMode(t, "")
|
||||||
|
genesis := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"genesis\n"}]}`)
|
||||||
|
since := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"since\n"}]}`)
|
||||||
|
other := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"other\n"}]}`)
|
||||||
|
if err := store.Save(opts.state, store.State{Genesis: &store.Genesis{Digest: apply.DigestOf(genesis),
|
||||||
|
At: time.Now(), Rewritten: true}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := store.SaveDeclared(store.DeclaredPath(opts.state), store.Declared{Declaration: since,
|
||||||
|
Signature: []byte("unverified here")}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
parsed := func(raw []byte) *declaration.Declaration {
|
||||||
|
d, err := declaration.ParseFileTrusted(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
err := runApply(context.Background(), opts, parsed(genesis), genesis, fromFile)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("the bundle genesis consumed was applied over what the mesh said since")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"older", short(apply.DigestOf(genesis)), short(apply.DigestOf(since))} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = runApply(context.Background(), opts, parsed(other), other, fromFile)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a declaration that is not what the mesh last said was applied")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "for a machine the mesh has not spoken to") ||
|
||||||
|
!strings.Contains(err.Error(), short(apply.DigestOf(since))) {
|
||||||
|
t.Errorf("the refusal does not say what a file is for and what was last said: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The very declaration the mesh last sent, from a file: still a file.
|
||||||
|
err = runApply(context.Background(), opts, parsed(since), since, fromFile)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "applied as the bundle is") {
|
||||||
|
t.Errorf("the last declaration, from a file, was not refused as a file: %v", err)
|
||||||
|
}
|
||||||
|
if known, _ := store.Load(opts.state); len(known.Resources) != 0 {
|
||||||
|
t.Errorf("a refused file recorded %d resource(s)", len(known.Resources))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A bundle other than the one genesis consumed: what genesis applied was rewritten for this
|
||||||
|
// machine, so the carried bytes never are.
|
||||||
|
err = runApply(context.Background(), opts, parsed(other), other, fromBundle)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "consumed") ||
|
||||||
|
!strings.Contains(err.Error(), short(apply.DigestOf(genesis))) {
|
||||||
|
t.Errorf("a bundle other than the consumed one was not refused naming it: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Defends novox/hq issue 104: what an apply would change is said before anything is, and
|
||||||
|
// `--dry-run` is that and nothing else — an action listed as the action it is.
|
||||||
|
func TestADryRunChangesNothingAndListsTheActions(t *testing.T) {
|
||||||
|
opts := stateWithMode(t, "")
|
||||||
|
opts.dryRun = true
|
||||||
|
out := &bytes.Buffer{}
|
||||||
|
opts.out = out
|
||||||
|
path := filepath.Join(filepath.Dir(opts.state), "a.conf")
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[
|
||||||
|
{"id":"a","type":"file","path":"` + path + `","content":"x\n"},
|
||||||
|
{"id":"init","type":"action","command":["createdb","mesh"],"verify":["psql","-c","select 1"]}]}`)
|
||||||
|
d, err := declaration.ParseFileTrusted(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil {
|
||||||
|
t.Fatalf("a dry run failed: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Error("a dry run wrote the file")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"would change", "create file a", "run action init",
|
||||||
|
"`createdb mesh`", "--dry-run: nothing applied"} {
|
||||||
|
if !strings.Contains(out.String(), want) {
|
||||||
|
t.Errorf("the preview does not say %q:\n%s", want, out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(out.String(), "applying:") {
|
||||||
|
t.Errorf("a dry run went on to apply:\n%s", out.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Machine-readable, the same shape as an apply's: the plan, and no report.
|
||||||
|
out.Reset()
|
||||||
|
opts.json = true
|
||||||
|
if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var shape struct {
|
||||||
|
Plan []apply.Step `json:"plan"`
|
||||||
|
Report *json.RawMessage `json:"report"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(out.Bytes(), &shape); err != nil || len(shape.Plan) != 2 || shape.Report != nil {
|
||||||
|
t.Errorf("a json dry run is not {plan} alone: %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Defends novox/hq issue 104: once the mesh has told this node anything, `reconcile` holds it to
|
||||||
|
// that — never to the bundle the host carries, which genesis consumed.
|
||||||
|
func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.T) {
|
||||||
|
was := builtFor
|
||||||
|
builtFor = "arch"
|
||||||
|
t.Cleanup(func() { builtFor = was })
|
||||||
|
opts := stateWithMode(t, store.ModeAdopted)
|
||||||
|
|
||||||
|
controllerPublic, controllerPrivate, err := ed25519.GenerateKey(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`)
|
||||||
|
if err := store.SaveDeclared(store.DeclaredPath(opts.state), store.Declared{Declaration: said,
|
||||||
|
Signature: ed25519.Sign(controllerPrivate, said)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Told, and unable to prove by whom: refused, and the bundle is not applied in its place.
|
||||||
|
_, _, _, err = reconcileSource(opts)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "not applied in its place") {
|
||||||
|
t.Errorf("a node that cannot prove what it was told fell back to something: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mine, err := identity.Generate("workstation")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mine.Membership = identity.Membership{Broker: "198.51.100.10:5671", Fingerprint: "sha256:0",
|
||||||
|
Signer: controllerPublic, Password: "issued"}
|
||||||
|
if err := identity.Save(identity.Path(opts.state), mine); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d, raw, from, err := reconcileSource(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if from != fromDeclared || !bytes.Equal(raw, said) || d.Adoption == nil {
|
||||||
|
t.Errorf("reconcile chose %d with %d bytes, not what the mesh last said", from, len(raw))
|
||||||
|
}
|
||||||
|
|
||||||
|
// And before the mesh has said anything: the bundle, as it always was. A test binary carries
|
||||||
|
// only the placeholder, and asking for it is what proves the path.
|
||||||
|
if err := os.Remove(store.DeclaredPath(opts.state)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, _, _, err = reconcileSource(opts)
|
||||||
|
if !errors.Is(err, bundle.ErrEmpty) {
|
||||||
|
t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/ed25519"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/tunnel"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0105: `overlay take` moves this node's overlay key onto the found tunnel's and
|
||||||
|
// nothing else — identity, sealing and serving keys stay as they were — and tells the mesh with a
|
||||||
|
// proof signed by the identity key, over the previous key, the new one and the tunnel.
|
||||||
|
|
||||||
|
func anEnrolledNode(t *testing.T) identity.Identity {
|
||||||
|
t.Helper()
|
||||||
|
mine, err := identity.Generate("anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mine.Overlay, err = identity.GenerateOverlayKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mine.Membership = identity.Membership{Broker: "198.51.100.1:5671", Fingerprint: "sha256:aa",
|
||||||
|
Signer: make([]byte, ed25519.PublicKeySize), Password: "p"}
|
||||||
|
return mine
|
||||||
|
}
|
||||||
|
|
||||||
|
func aFoundTunnel(t *testing.T) tunnel.Found {
|
||||||
|
t.Helper()
|
||||||
|
private, err := identity.GenerateOverlayKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
found, err := tunnel.Parse([]byte("[Interface]\nPrivateKey = " + private.Private + "\nListenPort = 51900\n" +
|
||||||
|
"Address = 192.0.2.1/24\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
found.Interface, found.Unit, found.Config = "wg0", "wg-quick@wg0", "/etc/wireguard/wg0.conf"
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTakingATunnelMovesOnlyTheOverlayKeyAndSignsForIt(t *testing.T) {
|
||||||
|
mine := anEnrolledNode(t)
|
||||||
|
found := aFoundTunnel(t)
|
||||||
|
before := mine.Overlay.Public
|
||||||
|
|
||||||
|
taken, rekey, err := rekeyOnto(mine, found)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if taken.Overlay.Public != found.PublicKey || taken.Overlay.Private != found.PrivateKey() {
|
||||||
|
t.Fatal("the overlay key is not the tunnel's")
|
||||||
|
}
|
||||||
|
if string(taken.Public) != string(mine.Public) || string(taken.Private) != string(mine.Private) ||
|
||||||
|
taken.Node != mine.Node || taken.Membership.Password != mine.Membership.Password ||
|
||||||
|
taken.Membership.Broker != mine.Membership.Broker {
|
||||||
|
t.Fatal("something other than the overlay key moved")
|
||||||
|
}
|
||||||
|
if taken.OverlayBefore != before {
|
||||||
|
t.Errorf("the key before the take was not kept: %q", taken.OverlayBefore)
|
||||||
|
}
|
||||||
|
if rekey.Previous != before || rekey.OverlayKey != found.PublicKey || rekey.Tunnel == nil ||
|
||||||
|
rekey.Tunnel.PublicKey != found.PublicKey || len(rekey.Tunnel.Peers) != 1 {
|
||||||
|
t.Fatalf("the rekey does not say what moved: %+v", rekey)
|
||||||
|
}
|
||||||
|
if !ed25519.Verify(ed25519.PublicKey(mine.Public),
|
||||||
|
link.RekeyProof("anchor", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
|
||||||
|
t.Fatal("the rekey is not signed by this node's identity key over what it says")
|
||||||
|
}
|
||||||
|
if ed25519.Verify(ed25519.PublicKey(mine.Public),
|
||||||
|
link.RekeyProof("laptop", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
|
||||||
|
t.Fatal("the proof is not bound to the node")
|
||||||
|
}
|
||||||
|
for _, said := range []string{rekey.Previous, rekey.OverlayKey, rekey.Tunnel.Interface} {
|
||||||
|
if strings.Contains(said, found.PrivateKey()) {
|
||||||
|
t.Fatal("the private key travels")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run again after the take — the mesh not yet told, or told and refused — the previous key it
|
||||||
|
// names is still the one before the take, so the mesh can tell a repeat from a replay.
|
||||||
|
again, second, err := rekeyOnto(taken, found)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if second.Previous != before || again.OverlayBefore != before || again.Overlay.Public != found.PublicKey {
|
||||||
|
t.Fatalf("a take run again does not name the key before the first: %+v", second)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,191 @@
|
|||||||
|
// foundation-first-node-nats.lock — what a machine must be before a mesh exists, on the bus being
|
||||||
|
// built (novox/hq ADR 0106, design 25).
|
||||||
|
//
|
||||||
|
// The same twelve steps as foundation-first-node.lock, with one difference that matters: **the mesh
|
||||||
|
// writes its own user list, and at genesis there is no mesh yet to write it.** So this carries the
|
||||||
|
// first one — the controller's own account, at a well-known bootstrap password, exactly as the store
|
||||||
|
// is reached at `postgres:bootstrap` and the old bus at `guest:guest`. It is rotated with those, and
|
||||||
|
// from the controller's first composition onward the file is the controller's to write.
|
||||||
|
//
|
||||||
|
// The accounts file is its own file beside the server's configuration, because the server's own
|
||||||
|
// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in
|
||||||
|
// one directory, of necessity: an include path is resolved relative to the including file's own
|
||||||
|
// directory, so a server given an absolute one looks for it underneath that directory and refuses to
|
||||||
|
// start.
|
||||||
|
//
|
||||||
|
// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client*
|
||||||
|
// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and
|
||||||
|
// authenticates with a password (ADR 0004, design 25 §4).
|
||||||
|
|
||||||
|
{
|
||||||
|
"declaration": 1,
|
||||||
|
"resources": [
|
||||||
|
{
|
||||||
|
"id": "container-runtime",
|
||||||
|
"type": "package",
|
||||||
|
"package": "docker"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "container-runtime-running",
|
||||||
|
"type": "service",
|
||||||
|
"unit": "docker.service",
|
||||||
|
"state": "running",
|
||||||
|
"boot": "enabled"
|
||||||
|
},
|
||||||
|
// **A filter before anything listens** (novox/hq issue 054, ADR 0088). The store and the
|
||||||
|
// broker are adopted as modules later and so bind to every interface from the moment they
|
||||||
|
// start; the packet filter that governs who may reach them is a module too, installed a
|
||||||
|
// dozen steps later. Between the two, a control-node facing the network had its store and
|
||||||
|
// its bus open to anyone who could reach the machine. So the foundation carries a filter of
|
||||||
|
// its own — the same table the filter module will replace wholesale once it can derive one:
|
||||||
|
// drop by default, keep loopback, replies, ssh and the mesh's own ports (the bus a node
|
||||||
|
// enrols over, the registry a node pulls from), and let the container runtime's own
|
||||||
|
// networks through the forward chain so containers keep working. A published container port
|
||||||
|
// is forwarded, never input (issue 047), which is why the forward chain is where the store's
|
||||||
|
// and broker's ports are refused from outside — and a container on this machine dialling a
|
||||||
|
// port this machine publishes reaches it through the runtime's proxy, which IS input, which
|
||||||
|
// is why the bus and the registry are opened in both chains, exactly as the derived ruleset
|
||||||
|
// does.
|
||||||
|
{
|
||||||
|
"id": "base-filter-package",
|
||||||
|
"type": "package",
|
||||||
|
"package": "nftables"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "base-filter",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/nftables.conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"content": "#!/usr/sbin/nft -f\n# the foundation's own filter, until the mesh derives one (novox/hq issue 054)\ntable inet mesh {}\ndelete table inet mesh\n\ntable inet mesh {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\tiif lo accept\n\t\ticmp type echo-request accept\n\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n\t\t# ssh, from anywhere — never closed\n\t\ttcp dport 22 accept\n\t\t# the mesh's own, from anywhere: the bus a node enrols over and a container on this machine reaches through the proxy, the registry a node pulls from\n\t\ttcp dport 5671 accept\n\t\ttcp dport 5000 accept\n\t}\n\tchain output {\n\t\ttype filter hook output priority filter; policy accept;\n\t}\n\tchain forward {\n\t\ttype filter hook forward priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\t# the container runtime's bridge networks, and the networks its compose files are given\n\t\tip saddr 172.16.0.0/12 accept\n\t\tip saddr 192.168.128.0/17 accept\n\t\t# the mesh's own: the bus a node enrols over, the registry a node pulls from\n\t\tct original proto-dst 5671 accept\n\t\tct original proto-dst 5000 accept\n\t}\n}\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "base-filter-loaded",
|
||||||
|
"type": "service",
|
||||||
|
"unit": "nftables.service",
|
||||||
|
"state": "running",
|
||||||
|
"boot": "enabled",
|
||||||
|
"restart-on": ["base-filter"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "store",
|
||||||
|
"type": "container",
|
||||||
|
"name": "mesh-store",
|
||||||
|
"image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a",
|
||||||
|
"env": {
|
||||||
|
"POSTGRES_PASSWORD": "bootstrap",
|
||||||
|
"PGDATA": "/var/lib/postgresql/data/pgdata"
|
||||||
|
},
|
||||||
|
"ports": ["5432:5432"],
|
||||||
|
"volumes": ["mesh-store-data:/var/lib/postgresql/data"]
|
||||||
|
},
|
||||||
|
// Over TCP, not the socket. While the store initialises it runs a temporary server on the
|
||||||
|
// socket ONLY, then stops it and starts the real one — so a socket check passes, the action
|
||||||
|
// exits happy, and the verify a moment later lands in the gap and fails. The action and its
|
||||||
|
// verify must ask the same question, or the action can succeed into a state verify rejects.
|
||||||
|
{
|
||||||
|
"id": "store-ready",
|
||||||
|
"type": "action",
|
||||||
|
"in": "mesh-store",
|
||||||
|
"command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
|
||||||
|
"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "inventory-database",
|
||||||
|
"type": "action",
|
||||||
|
"in": "mesh-store",
|
||||||
|
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
|
||||||
|
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "identity-database",
|
||||||
|
"type": "action",
|
||||||
|
"in": "mesh-store",
|
||||||
|
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
|
||||||
|
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
|
||||||
|
},
|
||||||
|
// Each context owns its own database (novox/hq ADR 0008). A third one is a third database,
|
||||||
|
// created the same way and named the same way — which is the whole of adding a context to the
|
||||||
|
// bootstrap, and is why the count is not something the foundation has an opinion about.
|
||||||
|
{
|
||||||
|
"id": "licences-database",
|
||||||
|
"type": "action",
|
||||||
|
"in": "mesh-store",
|
||||||
|
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
|
||||||
|
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "context-schemas",
|
||||||
|
"type": "action",
|
||||||
|
"command": ["docker", "run", "--rm", "--network", "container:mesh-store",
|
||||||
|
"-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||||
|
"-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||||
|
"-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||||
|
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||||
|
"migrate"],
|
||||||
|
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "bus-certificate",
|
||||||
|
"type": "action",
|
||||||
|
"command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
|
||||||
|
"192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
|
||||||
|
"-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' -addext 'subjectAltName=DNS:mesh-broker,IP:127.0.0.1' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"],
|
||||||
|
"verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
|
||||||
|
"192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
|
||||||
|
"-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "bus-conf-dir",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/var/lib/mesh-bus-conf",
|
||||||
|
"mode": "0700"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "bus-conf",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/var/lib/mesh-bus-conf/nats.conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "bus-accounts",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/var/lib/mesh-bus-conf/accounts.conf",
|
||||||
|
"mode": "0600",
|
||||||
|
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.API.>\", \"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"_INBOX.enrol.>\", \"mesh.control.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "broker",
|
||||||
|
"type": "container",
|
||||||
|
"name": "mesh-broker",
|
||||||
|
"image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
|
||||||
|
"ports": ["5671:4222", "127.0.0.1:8222:8222"],
|
||||||
|
"volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"],
|
||||||
|
"args": ["-c", "/etc/nats/nats.conf", "-js"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "broker-ready",
|
||||||
|
"type": "action",
|
||||||
|
"command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
||||||
|
"verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "control-plane",
|
||||||
|
"type": "container",
|
||||||
|
"name": "mesh-controller",
|
||||||
|
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||||
|
"network": "host",
|
||||||
|
"args": ["serve"],
|
||||||
|
"volumes": ["mesh-broker-tls:/broker-tls:ro"],
|
||||||
|
"env": {
|
||||||
|
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||||
|
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||||
|
"MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||||
|
"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",
|
||||||
|
"MESH_BROKER_ADDRESS": "192.0.2.10:5671",
|
||||||
|
"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -1,9 +1,13 @@
|
|||||||
module github.com/novox/mesh-host
|
module github.com/novox/mesh-host
|
||||||
|
|
||||||
go 1.25.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/klauspost/compress v1.20.0 // indirect
|
||||||
|
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||||
|
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||||
|
github.com/nats-io/nuid v1.0.1 // indirect
|
||||||
github.com/rabbitmq/amqp091-go v1.14.0 // indirect
|
github.com/rabbitmq/amqp091-go v1.14.0 // indirect
|
||||||
golang.org/x/crypto v0.55.0 // indirect
|
golang.org/x/crypto v0.57.0 // indirect
|
||||||
golang.org/x/sys v0.47.0 // indirect
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,6 +1,18 @@
|
|||||||
|
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||||
|
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||||
|
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||||
|
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||||
|
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||||
|
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||||
|
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||||
|
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||||
github.com/rabbitmq/amqp091-go v1.14.0 h1:RSaT7aOKt/OrkVUyswPDW29lnRz9psuGmfZFBmLqLek=
|
github.com/rabbitmq/amqp091-go v1.14.0 h1:RSaT7aOKt/OrkVUyswPDW29lnRz9psuGmfZFBmLqLek=
|
||||||
github.com/rabbitmq/amqp091-go v1.14.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
github.com/rabbitmq/amqp091-go v1.14.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||||
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||||
|
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||||
|
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
|
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||||
|
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
|
|||||||
+447
-31
@@ -52,6 +52,12 @@ type Outcome struct {
|
|||||||
wrote string
|
wrote string
|
||||||
// into is what a file written into held before the mesh's keys (novox/hq ADR 0102).
|
// into is what a file written into held before the mesh's keys (novox/hq ADR 0102).
|
||||||
into *store.Into
|
into *store.Into
|
||||||
|
// kept is where this apply kept the original of a file it wrote over (novox/hq ADR 0100).
|
||||||
|
kept string
|
||||||
|
// stateless is a service whose unit's lifecycle is the machine's (novox/hq ADR 0117).
|
||||||
|
stateless bool
|
||||||
|
// found is, for a service, its unit as the host first found it (novox/hq ADR 0118).
|
||||||
|
found *store.FoundUnit
|
||||||
// reads is, for a container, the digest of each file it was created reading, by path — so
|
// reads is, for a container, the digest of each file it was created reading, by path — so
|
||||||
// the next apply can say which one changed (novox/hq 04-ISSUES/103).
|
// the next apply can say which one changed (novox/hq 04-ISSUES/103).
|
||||||
reads map[string]string
|
reads map[string]string
|
||||||
@@ -60,6 +66,9 @@ type Outcome struct {
|
|||||||
// Report is what an apply did, in the order it did it.
|
// Report is what an apply did, in the order it did it.
|
||||||
type Report struct {
|
type Report struct {
|
||||||
Outcomes []Outcome `json:"outcomes"`
|
Outcomes []Outcome `json:"outcomes"`
|
||||||
|
// Tunnel is what this apply says about the tunnel the private network took over, when the
|
||||||
|
// declaration names one (novox/hq ADR 0105).
|
||||||
|
Tunnel *TakenTunnel `json:"tunnel,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Changed reports whether anything about the machine actually moved. An apply that changed
|
// Changed reports whether anything about the machine actually moved. An apply that changed
|
||||||
@@ -156,6 +165,19 @@ func ApplyKeeping(
|
|||||||
for _, r := range d.Resources {
|
for _, r := range d.Resources {
|
||||||
declared[r.Identity()] = true
|
declared[r.Identity()] = true
|
||||||
}
|
}
|
||||||
|
if svc := takesOver(d); svc != nil {
|
||||||
|
// The found tunnel's configuration is held under an id of its own, declared for as long
|
||||||
|
// as the service that took it over is (novox/hq ADR 0105).
|
||||||
|
declared[takeOverID(svc)] = true
|
||||||
|
}
|
||||||
|
// What an unfinished apply found a unit as is kept only while its service is declared: one
|
||||||
|
// declared again later is read afresh, as any resource with no record is (novox/hq ADR 0118).
|
||||||
|
known.DropFoundUndeclared(declared, origin)
|
||||||
|
|
||||||
|
// Which units the mesh made, read before any removal can take a unit file's record away — so
|
||||||
|
// whichever order a module declared a unit and its file in, the unit is known for the mesh's
|
||||||
|
// when its service goes (novox/hq ADR 0118).
|
||||||
|
made := meshMadeUnits(known)
|
||||||
|
|
||||||
// Which firewall is found here, before anything else, since an unsupported one refuses the
|
// Which firewall is found here, before anything else, since an unsupported one refuses the
|
||||||
// whole declaration (novox/hq ADR 0100). Nothing for a converged node.
|
// whole declaration (novox/hq ADR 0100). Nothing for a converged node.
|
||||||
@@ -174,7 +196,7 @@ func ApplyKeeping(
|
|||||||
if declaration.Type(orphan.Type) == declaration.TypeOpening {
|
if declaration.Type(orphan.Type) == declaration.TypeOpening {
|
||||||
action, detail, err = removeOpening(ctx, orphan, run, known.Firewall)
|
action, detail, err = removeOpening(ctx, orphan, run, known.Firewall)
|
||||||
} else {
|
} else {
|
||||||
action, detail, err = remove(ctx, sys, orphan, run)
|
action, detail, err = remove(ctx, sys, orphan, run, made)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &Error{Resource: orphan.ID, Err: err, Done: report}
|
return &Error{Resource: orphan.ID, Err: err, Done: report}
|
||||||
@@ -337,7 +359,52 @@ func ApplyKeeping(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The private network takes over the tunnel it found, ahead of the service that replaces
|
||||||
|
// it (novox/hq ADR 0105): its configuration kept, its unit stopped and disabled, never
|
||||||
|
// flushed. A failure here fails the service too — the mesh's interface is not started on a
|
||||||
|
// port the found one still holds.
|
||||||
|
stoppedFound := false
|
||||||
|
tookAt := -1
|
||||||
|
if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil {
|
||||||
|
var outcome Outcome
|
||||||
|
var facts TakenTunnel
|
||||||
|
var err error
|
||||||
|
if d.Adoption == nil {
|
||||||
|
err = errNotAdopted
|
||||||
|
facts = TakenTunnel{Interface: svc.TakesOver.Interface, State: NotTaken}
|
||||||
|
} else {
|
||||||
|
outcome, facts, stoppedFound, err = takeOver(ctx, sys, svc, d, &known, run, keep, time.Now().UTC())
|
||||||
|
}
|
||||||
|
// Always an account, failure included: the last account standing must never be an
|
||||||
|
// older "taken" over a machine whose takeover has since gone wrong.
|
||||||
|
report.Tunnel = &facts
|
||||||
|
if err != nil {
|
||||||
|
report.Tunnel.Note = err.Error()
|
||||||
|
if stoppedFound {
|
||||||
|
// The found unit is down and the mesh's not up: the one state where the
|
||||||
|
// peers reach nothing. Started again, and said.
|
||||||
|
restoreFound(ctx, sys, svc.TakesOver.Unit, run, report.Tunnel)
|
||||||
|
}
|
||||||
|
failures = append(failures, &Error{Resource: svc.Identity(), Err: err, Done: report})
|
||||||
|
log(fmt.Sprintf(" failed %s (%s): %v", svc.Identity(), svc.Unit, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
tookAt = len(report.Outcomes)
|
||||||
|
report.Outcomes = append(report.Outcomes, outcome)
|
||||||
|
if outcome.Action == "held" {
|
||||||
|
log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
was, _ := known.Find(resource.Identity())
|
was, _ := known.Find(resource.Identity())
|
||||||
|
// What an earlier apply of this service found its unit as and could not yet record is
|
||||||
|
// newer than anything the record says — the record's own finding with what was read
|
||||||
|
// since — so it is what this apply goes on from (novox/hq ADR 0118).
|
||||||
|
previous := was
|
||||||
|
if p, ok := known.FoundFirst[resource.Identity()]; ok {
|
||||||
|
f := p.FoundUnit
|
||||||
|
previous.Found = &f
|
||||||
|
}
|
||||||
var outcome Outcome
|
var outcome Outcome
|
||||||
var err error
|
var err error
|
||||||
if o, isOpening := resource.(*declaration.Opening); isOpening {
|
if o, isOpening := resource.(*declaration.Opening); isOpening {
|
||||||
@@ -350,12 +417,29 @@ func ApplyKeeping(
|
|||||||
!known.Recorded(string(declaration.TypeFile), f.Path) {
|
!known.Recorded(string(declaration.TypeFile), f.Path) {
|
||||||
keepFound = keep
|
keepFound = keep
|
||||||
}
|
}
|
||||||
outcome, err = applyOne(ctx, sys, resource, run, changed, in, was, unseal, keepFound)
|
outcome, err = applyOne(ctx, sys, resource, run, changed, in, previous, unseal, keepFound)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
// **What was found is kept whatever the apply then did** (novox/hq ADR 0118). The
|
||||||
|
// failure may have come after the mesh enabled or started the unit, and the next apply
|
||||||
|
// would otherwise read that as the machine's own.
|
||||||
|
if outcome.found != nil {
|
||||||
|
known.KeepFound(resource.Identity(), origin, *outcome.found)
|
||||||
|
}
|
||||||
failed := &Error{Resource: resource.Identity(), Err: err, Done: report}
|
failed := &Error{Resource: resource.Identity(), Err: err, Done: report}
|
||||||
failures = append(failures, failed)
|
failures = append(failures, failed)
|
||||||
log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), outcome.Target, err))
|
log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), outcome.Target, err))
|
||||||
|
if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil {
|
||||||
|
// The mesh's interface did not come up after the found one was stopped: no
|
||||||
|
// tunnel at all. The found unit is started again — the machine goes back to
|
||||||
|
// what it had — and the account says so (novox/hq ADR 0105).
|
||||||
|
report.Tunnel.Note = "the mesh's interface did not come up: " + err.Error()
|
||||||
|
if stoppedFound {
|
||||||
|
restoreFound(ctx, sys, svc.TakesOver.Unit, run, report.Tunnel)
|
||||||
|
} else {
|
||||||
|
report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// **A failed action stops what follows. Nothing else does.**
|
// **A failed action stops what follows. Nothing else does.**
|
||||||
//
|
//
|
||||||
@@ -389,20 +473,57 @@ func ApplyKeeping(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Where the original of what this file replaced was kept, carried for as long as the
|
||||||
|
// resource is recorded: kept by this apply, by a hold its module's cutover ends, or before.
|
||||||
|
held, wasHeld := known.HeldAt(resource.Identity())
|
||||||
|
kept := outcome.kept
|
||||||
|
if kept == "" && wasHeld {
|
||||||
|
kept = held.Kept
|
||||||
|
}
|
||||||
|
if kept == "" {
|
||||||
|
kept = was.Kept
|
||||||
|
}
|
||||||
// Only now. The record follows the fact, never leads it.
|
// Only now. The record follows the fact, never leads it.
|
||||||
known.Record(store.Applied{
|
known.Record(store.Applied{
|
||||||
Origin: origin,
|
Origin: origin,
|
||||||
ID: resource.Identity(), Type: string(resource.Kind()),
|
ID: resource.Identity(), Type: string(resource.Kind()),
|
||||||
Target: outcome.Target, AppliedAt: time.Now().UTC(),
|
Target: outcome.Target, AppliedAt: time.Now().UTC(),
|
||||||
Wrote: outcome.wrote,
|
Wrote: outcome.wrote,
|
||||||
Into: outcome.into,
|
Into: outcome.into,
|
||||||
Reads: outcome.reads,
|
Kept: kept,
|
||||||
Holds: holds(resource),
|
Reads: outcome.reads,
|
||||||
|
Stateless: outcome.stateless,
|
||||||
|
Found: outcome.found,
|
||||||
|
Holds: holds(resource),
|
||||||
})
|
})
|
||||||
// Its module has been taken, and what was held for it is now the mesh's.
|
if outcome.found != nil {
|
||||||
if held, wasHeld := known.HeldAt(resource.Identity()); wasHeld {
|
known.DropFound(resource.Identity())
|
||||||
|
}
|
||||||
|
// Its module has been taken, and what was held for it is now the mesh's. A file written
|
||||||
|
// into, or a service whose lifecycle is the machine's, replaced nothing that was found, so
|
||||||
|
// its outcome says what the apply did, not that a cutover happened; a hold from when it was
|
||||||
|
// declared otherwise goes all the same — here, after the apply worked, so a failed one
|
||||||
|
// keeps the hold and where its original is.
|
||||||
|
if wasHeld {
|
||||||
known.Release(held.ID)
|
known.Release(held.ID)
|
||||||
outcome.Detail = takenDetail(held)
|
if !replacesNothing(resource) {
|
||||||
|
outcome.Detail = takenDetail(held)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil {
|
||||||
|
// The found interface is down and the mesh's is up in its place: the tunnel changed
|
||||||
|
// hands (novox/hq ADR 0105). Read from the machine, not assumed.
|
||||||
|
report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run)
|
||||||
|
// And once a peer has handshaken with it, the take is proven and the found
|
||||||
|
// configuration is retired — here, after the mesh's service applied, so in the apply
|
||||||
|
// of the take itself only if a peer is already through; otherwise a later apply
|
||||||
|
// retires it (novox/hq ADR 0119). What it did replaces what the take said of the file.
|
||||||
|
if o, did := retireFound(ctx, svc, d, &known, run, keep, report.Tunnel, time.Now().UTC()); did {
|
||||||
|
if tookAt >= 0 {
|
||||||
|
report.Outcomes[tookAt] = o
|
||||||
|
}
|
||||||
|
log(fmt.Sprintf(" %s %s (%s): %s", o.Action, o.ID, o.Target, o.Detail))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
report.Outcomes = append(report.Outcomes, outcome)
|
report.Outcomes = append(report.Outcomes, outcome)
|
||||||
if outcome.Action != "unchanged" {
|
if outcome.Action != "unchanged" {
|
||||||
@@ -466,7 +587,7 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru
|
|||||||
case *declaration.File:
|
case *declaration.File:
|
||||||
return applyFile(res, previous, unseal, keepFound)
|
return applyFile(res, previous, unseal, keepFound)
|
||||||
case *declaration.Service:
|
case *declaration.Service:
|
||||||
return applyService(ctx, sys, res, run, changed)
|
return applyService(ctx, sys, res, run, changed, previous)
|
||||||
case *declaration.Package:
|
case *declaration.Package:
|
||||||
return applyPackage(ctx, sys, res, run)
|
return applyPackage(ctx, sys, res, run)
|
||||||
case *declaration.Container:
|
case *declaration.Container:
|
||||||
@@ -612,6 +733,9 @@ func applyAccess(r *declaration.Access) (Outcome, error) {
|
|||||||
// keepFound, when not nil, is where the original of a file this host has no record of is kept
|
// keepFound, when not nil, is where the original of a file this host has no record of is kept
|
||||||
// before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome.
|
// before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome.
|
||||||
func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) {
|
func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) {
|
||||||
|
if r.Into == declaration.IntoBlock {
|
||||||
|
return applyBlock(r, previous)
|
||||||
|
}
|
||||||
if r.Into != "" {
|
if r.Into != "" {
|
||||||
return applyInto(r, previous)
|
return applyInto(r, previous)
|
||||||
}
|
}
|
||||||
@@ -780,6 +904,7 @@ func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepF
|
|||||||
default:
|
default:
|
||||||
out.Action = "unchanged"
|
out.Action = "unchanged"
|
||||||
}
|
}
|
||||||
|
out.kept = kept
|
||||||
if kept != "" {
|
if kept != "" {
|
||||||
if out.Detail != "" {
|
if out.Detail != "" {
|
||||||
out.Detail += "; "
|
out.Detail += "; "
|
||||||
@@ -839,7 +964,10 @@ type unitReloader interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
|
func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
|
||||||
changed map[string]bool) (Outcome, error) {
|
changed map[string]bool, previous store.Applied) (Outcome, error) {
|
||||||
|
if r.Stateless() {
|
||||||
|
return reflectOnly(ctx, sys, r, run, changed)
|
||||||
|
}
|
||||||
out := begin(r)
|
out := begin(r)
|
||||||
var changes []string
|
var changes []string
|
||||||
|
|
||||||
@@ -854,6 +982,21 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **What the unit was before the mesh touched it**, read once and carried in the record from
|
||||||
|
// then on (novox/hq ADR 0118). Undeclared, the unit is given back to exactly this: it is the one
|
||||||
|
// fact that separates the container runtime, running before the mesh arrived and to be left
|
||||||
|
// running, from the mesh's packet filter, stopped until the mesh started it and to be stopped
|
||||||
|
// again. Read after the reload above, never before it: a unit whose file this same apply wrote
|
||||||
|
// is not a unit the service manager knows until then, and reading it first would find nothing.
|
||||||
|
found, gaveBack, err := foundAs(ctx, sys, r, run, previous)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
out.found = found
|
||||||
|
if gaveBack != "" {
|
||||||
|
changes = append(changes, gaveBack)
|
||||||
|
}
|
||||||
|
|
||||||
// Boot first. A unit asked to be running and enabled should survive this apply failing
|
// Boot first. A unit asked to be running and enabled should survive this apply failing
|
||||||
// half way in the more useful direction: enabled-and-stopped comes back at the next boot,
|
// half way in the more useful direction: enabled-and-stopped comes back at the next boot,
|
||||||
// where running-and-disabled does not.
|
// where running-and-disabled does not.
|
||||||
@@ -862,6 +1005,15 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
|
// Whether it started at boot is found the first time the mesh is about to change that —
|
||||||
|
// which is not always the first apply: a declaration that said nothing about boot never
|
||||||
|
// touched it, so what is there when one first does is still the machine's (novox/hq ADR
|
||||||
|
// 0118). Kept before the change, so a failure after it still has it.
|
||||||
|
if out.found != nil && out.found.Boot == "" {
|
||||||
|
f := *out.found
|
||||||
|
f.Boot = bootBefore
|
||||||
|
out.found = &f
|
||||||
|
}
|
||||||
if bootBefore != r.Boot {
|
if bootBefore != r.Boot {
|
||||||
if err := sys.SetServiceBoot(ctx, run, r.Unit, r.Boot); err != nil {
|
if err := sys.SetServiceBoot(ctx, run, r.Unit, r.Boot); err != nil {
|
||||||
return out, fmt.Errorf("setting %s to %s at boot: %w", r.Unit, r.Boot, err)
|
return out, fmt.Errorf("setting %s to %s at boot: %w", r.Unit, r.Boot, err)
|
||||||
@@ -950,6 +1102,73 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reflectOnly is a service whose unit's lifecycle is the machine's (novox/hq ADR 0117): nothing is
|
||||||
|
// started, stopped, enabled or disabled, and a changed trigger is acted on only where the unit is
|
||||||
|
// already running. An inactive unit is left so — started, it would be a second network manager on
|
||||||
|
// a machine that uses another — and it reads the change when whatever starts it does.
|
||||||
|
func reflectOnly(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
|
||||||
|
changed map[string]bool) (Outcome, error) {
|
||||||
|
out := begin(r)
|
||||||
|
out.stateless = true
|
||||||
|
restart := reflected(r, changed)
|
||||||
|
reload := restartedBy(r.ReloadOn, changed)
|
||||||
|
if len(restart) == 0 && len(reload) == 0 {
|
||||||
|
out.Action = "unchanged"
|
||||||
|
out.Detail = "its lifecycle is the machine's; nothing it reflects changed"
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
state, err := sys.ServiceState(ctx, run, r.Unit)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if state != "running" {
|
||||||
|
out.Action = "unchanged"
|
||||||
|
out.Detail = "not running; the change applies at its next start"
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if len(restart) > 0 {
|
||||||
|
// The same as a stated service: the unit's own file may be what changed, and the manager
|
||||||
|
// reads that again only when told to.
|
||||||
|
if u, ok := sys.(unitReloader); ok {
|
||||||
|
if err := u.ReloadUnits(ctx, run); err != nil {
|
||||||
|
return out, fmt.Errorf("reloading the service manager's units for %s: %w", r.Unit, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := sys.SetServiceState(ctx, run, r.Unit, "stopped"); err != nil {
|
||||||
|
return out, fmt.Errorf("restarting %s: stopping it: %w", r.Unit, err)
|
||||||
|
}
|
||||||
|
if err := sys.SetServiceState(ctx, run, r.Unit, "running"); err != nil {
|
||||||
|
return out, fmt.Errorf("restarting %s: starting it again: %w", r.Unit, err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
reloader, ok := sys.(serviceReloader)
|
||||||
|
if !ok {
|
||||||
|
return out, fmt.Errorf("%s must be reloaded for %s and this machine's service manager "+
|
||||||
|
"cannot reload a unit", r.Unit, strings.Join(reload, ", "))
|
||||||
|
}
|
||||||
|
if err := reloader.ReloadService(ctx, run, r.Unit); err != nil {
|
||||||
|
return out, fmt.Errorf("reloading %s: %w", r.Unit, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Read back: it was running, and a restart or reload that left it otherwise is a failure —
|
||||||
|
// the machine's network manager down is not a change to report and move past.
|
||||||
|
after, err := sys.ServiceState(ctx, run, r.Unit)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
out.Action = "updated"
|
||||||
|
if len(restart) > 0 {
|
||||||
|
out.Detail = "restarted for " + strings.Join(restart, ", ")
|
||||||
|
} else {
|
||||||
|
out.Detail = "reloaded for " + strings.Join(reload, ", ")
|
||||||
|
}
|
||||||
|
if after != "running" {
|
||||||
|
return out, fmt.Errorf("%s was %s to pick up a change and is %s", r.Unit,
|
||||||
|
strings.Fields(out.Detail)[0], after)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
// remove undoes one resource the host applied and the declaration no longer names, and reports
|
// remove undoes one resource the host applied and the declaration no longer names, and reports
|
||||||
// what it actually did.
|
// what it actually did.
|
||||||
//
|
//
|
||||||
@@ -959,7 +1178,10 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
|
|||||||
// It returns the action rather than assuming "removed", because for half the vocabulary the
|
// It returns the action rather than assuming "removed", because for half the vocabulary the
|
||||||
// honest word is "forgotten". A host that reported a package removed when it left the package
|
// honest word is "forgotten". A host that reported a package removed when it left the package
|
||||||
// installed would be describing an effect it declined to have.
|
// installed would be describing an effect it declined to have.
|
||||||
func remove(ctx context.Context, sys system.System, a store.Applied, run Runner) (string, string, error) {
|
//
|
||||||
|
// made is the units whose unit file this host wrote where there was none (meshMadeUnits).
|
||||||
|
func remove(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
||||||
|
made map[string]bool) (string, string, error) {
|
||||||
switch declaration.Type(a.Type) {
|
switch declaration.Type(a.Type) {
|
||||||
case declaration.TypeDirectory:
|
case declaration.TypeDirectory:
|
||||||
// **A directory with anything left in it is kept, and that is the rule that protects
|
// **A directory with anything left in it is kept, and that is the rule that protects
|
||||||
@@ -992,6 +1214,9 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner)
|
|||||||
return "removed", "no longer declared, and empty", nil
|
return "removed", "no longer declared, and empty", nil
|
||||||
|
|
||||||
case declaration.TypeFile:
|
case declaration.TypeFile:
|
||||||
|
if a.Into != nil && a.Into.Format == declaration.IntoBlock {
|
||||||
|
return removeBlock(a)
|
||||||
|
}
|
||||||
if a.Into != nil {
|
if a.Into != nil {
|
||||||
return removeInto(a)
|
return removeInto(a)
|
||||||
}
|
}
|
||||||
@@ -1004,24 +1229,12 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner)
|
|||||||
return "removed", "no longer declared", nil
|
return "removed", "no longer declared", nil
|
||||||
|
|
||||||
case declaration.TypeService:
|
case declaration.TypeService:
|
||||||
// A unit that is no longer declared is stopped, not deleted. The host did not install
|
return removeService(ctx, sys, a, run, made[a.Target])
|
||||||
// it and does not own the unit file — only the state it put the unit into.
|
|
||||||
//
|
case declaration.TypeProcess:
|
||||||
// A unit that no longer EXISTS is already in the state removal is trying to reach, and
|
// The other side of the same line: a process's unit is the host's own — it wrote the unit
|
||||||
// saying so matters: stopping it fails, and a failure here fails the whole apply. A
|
// file and unpacked the bundle — so it goes with its declaration (novox/hq ADR 0118).
|
||||||
// host holding a record of an uninstalled unit would then be unable to apply anything,
|
return removeProcess(ctx, a, run)
|
||||||
// ever, with no way out but editing its state by hand. Removal is idempotent for the
|
|
||||||
// same reason `os.RemoveAll` is.
|
|
||||||
if _, err := sys.ServiceState(ctx, run, a.Target); err != nil {
|
|
||||||
if strings.Contains(err.Error(), "does not exist on this machine") {
|
|
||||||
return "forgotten", "the unit no longer exists", nil
|
|
||||||
}
|
|
||||||
return "", "", err
|
|
||||||
}
|
|
||||||
if err := sys.SetServiceState(ctx, run, a.Target, "stopped"); err != nil {
|
|
||||||
return "", "", fmt.Errorf("stopping %s: %w", a.Target, err)
|
|
||||||
}
|
|
||||||
return "removed", "stopped; the unit file is not the host's to delete", nil
|
|
||||||
|
|
||||||
case declaration.TypeContainer:
|
case declaration.TypeContainer:
|
||||||
// The host CREATED this one, so the host removes it. That is the line: it removes what
|
// The host CREATED this one, so the host removes it. That is the line: it removes what
|
||||||
@@ -1257,6 +1470,15 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
|
|||||||
for _, a := range r.Args {
|
for _, a := range r.Args {
|
||||||
b.WriteString("arg " + a + "\n")
|
b.WriteString("arg " + a + "\n")
|
||||||
}
|
}
|
||||||
|
// The resolver and address are part of what was declared: a container whose dns or ip moved
|
||||||
|
// is a different container, or the fields could never reach one that already ran — which is
|
||||||
|
// exactly how their first deployment silently changed nothing.
|
||||||
|
for _, d := range r.Dns {
|
||||||
|
b.WriteString("dns " + d + "\n")
|
||||||
|
}
|
||||||
|
if r.IP != "" {
|
||||||
|
b.WriteString("ip " + r.IP + "\n")
|
||||||
|
}
|
||||||
// The cadence is part of what was declared, so a changed schedule is a changed spec — the marker
|
// The cadence is part of what was declared, so a changed schedule is a changed spec — the marker
|
||||||
// moves and the install is reported "updated" and re-established. Added only when present, so no
|
// moves and the install is reported "updated" and re-established. Added only when present, so no
|
||||||
// ordinary container's or run-once step's digest moves for a field it does not set.
|
// ordinary container's or run-once step's digest moves for a field it does not set.
|
||||||
@@ -1290,11 +1512,15 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
|
|||||||
|
|
||||||
// containerState reports whether a container is running and which spec made it.
|
// containerState reports whether a container is running and which spec made it.
|
||||||
// The error means the container does not exist.
|
// The error means the container does not exist.
|
||||||
|
//
|
||||||
|
// `container inspect`, not the bare form: a name is not unique across object kinds (a network and
|
||||||
|
// its container are routinely named alike), and the bare form can resolve to the wrong kind
|
||||||
|
// instead of reporting absence — see inspectFound in hold.go for the failure this produces.
|
||||||
func containerState(ctx context.Context, name string, run Runner) (state struct {
|
func containerState(ctx context.Context, name string, run Runner) (state struct {
|
||||||
Running bool
|
Running bool
|
||||||
Spec string
|
Spec string
|
||||||
}, err error) {
|
}, err error) {
|
||||||
out, err := run(ctx, "docker", "inspect", "--format",
|
out, err := run(ctx, "docker", "container", "inspect", "--format",
|
||||||
"{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
|
"{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return state, fmt.Errorf("no container named %s", name)
|
return state, fmt.Errorf("no container named %s", name)
|
||||||
@@ -1447,6 +1673,12 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
|
|||||||
if r.Network != "" {
|
if r.Network != "" {
|
||||||
args = append(args, "--network", r.Network)
|
args = append(args, "--network", r.Network)
|
||||||
}
|
}
|
||||||
|
for _, d := range r.Dns {
|
||||||
|
args = append(args, "--dns", d)
|
||||||
|
}
|
||||||
|
if r.IP != "" {
|
||||||
|
args = append(args, "--ip", r.IP)
|
||||||
|
}
|
||||||
args = append(args,
|
args = append(args,
|
||||||
"--label", specLabel+"="+want, "--label", idLabel+"="+r.ID)
|
"--label", specLabel+"="+want, "--label", idLabel+"="+r.ID)
|
||||||
for _, k := range sortedKeys(r.Env) {
|
for _, k := range sortedKeys(r.Env) {
|
||||||
@@ -1782,6 +2014,13 @@ func digestOf(content string) string {
|
|||||||
return hex.EncodeToString(sum[:])
|
return hex.EncodeToString(sum[:])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DigestOf names a declaration by its bytes, exactly as the mesh names what it sends: sha256 of
|
||||||
|
// the raw bytes, hex. The two sides never digest different things.
|
||||||
|
func DigestOf(raw []byte) string {
|
||||||
|
sum := sha256.Sum256(raw)
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
// holds is the machine's own ports a resource occupies.
|
// holds is the machine's own ports a resource occupies.
|
||||||
//
|
//
|
||||||
// **What the declaration binds, not what is open.** A machine's open ports are a moving target —
|
// **What the declaration binds, not what is open.** A machine's open ports are a moving target —
|
||||||
@@ -1831,3 +2070,180 @@ func declaredDigest(r declaration.Resource) string {
|
|||||||
}
|
}
|
||||||
return fmt.Sprintf("%x", sha256.Sum256([]byte(material)))
|
return fmt.Sprintf("%x", sha256.Sum256([]byte(material)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// removeService gives a unit back the state the host first found it in, and nothing more.
|
||||||
|
//
|
||||||
|
// **Removes what it made, gives back what it changed, leaves what was the machine's**
|
||||||
|
// (novox/hq ADR 0118). A service resource never installs a unit; it puts one that already existed
|
||||||
|
// into a state. So undeclaring it cannot mean stopping it — that is how unassigning the private
|
||||||
|
// network stopped the container runtime and every container with it, how unassigning sshd would
|
||||||
|
// have stopped ssh, and how an uplink module would have taken a machine off its only link
|
||||||
|
// (novox/hq issue 130). It means undoing what the mesh did to it: a unit found running is left
|
||||||
|
// running; a unit the mesh started is stopped again, and disabled again if the mesh enabled it.
|
||||||
|
//
|
||||||
|
// **A unit whose file the mesh wrote is the mesh's, whatever was found** — made is that. The
|
||||||
|
// adoption guard and the converge filter are units of exactly this kind: their unit files are the
|
||||||
|
// mesh's own `file` resources, written where there was none, and records written before the host
|
||||||
|
// kept what it found say nothing about them. Forgetting one would leave its table loaded — the
|
||||||
|
// guard beside a converged node's own filter, or the filter beside the predecessor's firewall
|
||||||
|
// re-enabled — and its unit file then deleted from under a running unit. So it is stopped and
|
||||||
|
// disabled at boot, as a process's unit is, before its file goes: orphans are removed newest
|
||||||
|
// first, and a unit's file is declared before the service that starts it.
|
||||||
|
//
|
||||||
|
// **Never started on the way out.** A unit the mesh stopped is not started again when its
|
||||||
|
// declaration goes: starting something is a decision, and the operator makes it. The report says
|
||||||
|
// what was actually done — a unit already as it was found is forgotten, not "restored".
|
||||||
|
func removeService(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
||||||
|
made bool) (string, string, error) {
|
||||||
|
if a.Stateless {
|
||||||
|
// Declared with no state (novox/hq ADR 0117): its lifecycle was never the mesh's, and the
|
||||||
|
// declaration that said so is the operator's word to hold to, a file of the mesh's or not.
|
||||||
|
return "forgotten", "its state was never the mesh's", nil
|
||||||
|
}
|
||||||
|
if !made && a.Found == nil {
|
||||||
|
// Recorded before the host kept what it found. Not knowing, it leaves the unit as it is:
|
||||||
|
// a unit left running can be stopped by the operator, and one stopped by mistake may be
|
||||||
|
// the link the operator would use to do it.
|
||||||
|
return "forgotten", "recorded before the host kept what it found; left as it is", nil
|
||||||
|
}
|
||||||
|
stop := made || a.Found.State == "stopped"
|
||||||
|
disable := made || a.Found.Boot == "disabled"
|
||||||
|
|
||||||
|
if !stop && !disable {
|
||||||
|
// Found running, and not disabled by anyone but the mesh: nothing to give back. What the
|
||||||
|
// mesh did since is said, so a unit left stopped is not reported as the machine's doing —
|
||||||
|
// read as well as the machine answers, since nothing here acts on the answer.
|
||||||
|
state, err := sys.ServiceState(ctx, run, a.Target)
|
||||||
|
if err != nil && strings.Contains(err.Error(), "does not exist on this machine") {
|
||||||
|
return "forgotten", "the unit no longer exists", nil
|
||||||
|
}
|
||||||
|
var did []string
|
||||||
|
if err == nil && state == "stopped" {
|
||||||
|
did = append(did, "stopped it")
|
||||||
|
}
|
||||||
|
if a.Found.Boot == "enabled" {
|
||||||
|
if boot, err := sys.ServiceBoot(ctx, run, a.Target); err == nil && boot == "disabled" {
|
||||||
|
did = append(did, "disabled it at boot")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(did) > 0 {
|
||||||
|
return "forgotten", "left as it is; the mesh " + strings.Join(did, " and ") +
|
||||||
|
" and does not start anything on the way out", nil
|
||||||
|
}
|
||||||
|
return "forgotten", "it was running before the mesh; left as it is", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Something may be given back, so the unit must still be there to give it to. One since
|
||||||
|
// uninstalled is already as far from the mesh as it can be, and saying so keeps a record of it
|
||||||
|
// from failing every apply.
|
||||||
|
state, err := sys.ServiceState(ctx, run, a.Target)
|
||||||
|
if err != nil {
|
||||||
|
if strings.Contains(err.Error(), "does not exist on this machine") {
|
||||||
|
return "forgotten", "the unit no longer exists", nil
|
||||||
|
}
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
var did, already []string
|
||||||
|
if stop {
|
||||||
|
if state != "stopped" {
|
||||||
|
if err := sys.SetServiceState(ctx, run, a.Target, "stopped"); err != nil {
|
||||||
|
return "", "", fmt.Errorf("stopping %s, which the mesh started: %w", a.Target, err)
|
||||||
|
}
|
||||||
|
did = append(did, "stopped")
|
||||||
|
} else {
|
||||||
|
already = append(already, "stopped")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if disable {
|
||||||
|
// Disabled unless it reads as disabled: a unit the service manager cannot say a boot state
|
||||||
|
// for — one with no install section — takes a disable as a no-op, and asking is how the
|
||||||
|
// host stays sure the mesh's enable did not outlive it.
|
||||||
|
if boot, err := sys.ServiceBoot(ctx, run, a.Target); err == nil && boot == "disabled" {
|
||||||
|
already = append(already, "disabled at boot")
|
||||||
|
} else {
|
||||||
|
if err := sys.SetServiceBoot(ctx, run, a.Target, "disabled"); err != nil {
|
||||||
|
return "", "", fmt.Errorf("disabling %s, which the mesh enabled: %w", a.Target, err)
|
||||||
|
}
|
||||||
|
did = append(did, "disabled at boot")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if made {
|
||||||
|
if len(did) == 0 {
|
||||||
|
return "forgotten", "already stopped and disabled at boot; the mesh wrote its unit file", nil
|
||||||
|
}
|
||||||
|
return "removed", strings.Join(did, ", ") + "; the mesh wrote its unit file, so the unit is the mesh's own", nil
|
||||||
|
}
|
||||||
|
if len(did) == 0 {
|
||||||
|
return "forgotten", "already as the host found it (" + strings.Join(already, ", ") + ")", nil
|
||||||
|
}
|
||||||
|
detail := strings.Join(did, ", ") + ", as the host found it"
|
||||||
|
if len(already) > 0 {
|
||||||
|
detail += "; already " + strings.Join(already, ", ")
|
||||||
|
}
|
||||||
|
return "restored", detail, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// foundAs is what a service's unit was before the mesh touched it, as far as this host can know:
|
||||||
|
// what an earlier apply recorded, or — the first time the mesh is about to act on the unit — what
|
||||||
|
// is there now (novox/hq ADR 0118). Nil when it cannot be known: a record written before the host
|
||||||
|
// kept this has had the mesh acting on the unit since, and a reading now would be the mesh's doing.
|
||||||
|
//
|
||||||
|
// Read now as well, besides on a first apply, where the mesh has never acted on this unit's state
|
||||||
|
// although a record exists: the record is of a service declared with no state (novox/hq ADR 0117),
|
||||||
|
// which the mesh never starts or stops, or of another unit altogether. What was found about one
|
||||||
|
// unit says nothing about another, so a service moved to a new unit gives the old one back, just as
|
||||||
|
// if it had been undeclared, and is read afresh for the new one; gave says what that gave back.
|
||||||
|
func foundAs(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
|
||||||
|
previous store.Applied) (found *store.FoundUnit, gave string, err error) {
|
||||||
|
if f := previous.Found; f != nil {
|
||||||
|
unit := f.Unit
|
||||||
|
if unit == "" {
|
||||||
|
unit = previous.Target
|
||||||
|
}
|
||||||
|
if unit == "" || unit == r.Unit {
|
||||||
|
return f, "", nil
|
||||||
|
}
|
||||||
|
// Given back as if undeclared, never as the mesh's own: whether the mesh wrote the old
|
||||||
|
// unit's file is known to the removal of orphans, and that file's own record goes with it.
|
||||||
|
action, detail, err := removeService(ctx, sys,
|
||||||
|
store.Applied{Type: string(declaration.TypeService), Target: unit, Found: f}, run, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("giving %s back as the host found it, now that %s is declared instead: %w",
|
||||||
|
unit, r.Unit, err)
|
||||||
|
}
|
||||||
|
if action == "restored" {
|
||||||
|
gave = unit + " " + detail
|
||||||
|
}
|
||||||
|
} else if previous.ID != "" && !previous.Stateless && previous.Target == r.Unit {
|
||||||
|
return nil, "", nil
|
||||||
|
}
|
||||||
|
state, err := sys.ServiceState(ctx, run, r.Unit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, gave, err
|
||||||
|
}
|
||||||
|
return &store.FoundUnit{Unit: r.Unit, State: state}, gave, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// meshMadeUnits is every unit whose unit file this host wrote whole where there was none: a `file`
|
||||||
|
// record with no kept original and not written into, at a unit's own path under a directory the
|
||||||
|
// service manager loads administrators' units from — or the one the host writes a process's unit
|
||||||
|
// in. Such a unit is the mesh's, whatever was found (novox/hq ADR 0118); see removeService.
|
||||||
|
//
|
||||||
|
// A drop-in is not the unit's own file, and a file the host wrote over is the machine's unit with
|
||||||
|
// the mesh's text in it: its original is kept, and put back when the file's record goes.
|
||||||
|
func meshMadeUnits(known store.State) map[string]bool {
|
||||||
|
made := map[string]bool{}
|
||||||
|
dirs := map[string]bool{"/etc/systemd/system": true, "/run/systemd/system": true,
|
||||||
|
filepath.Clean(unitDir): true}
|
||||||
|
for _, r := range known.Resources {
|
||||||
|
if r.Type != string(declaration.TypeFile) || r.Kept != "" || r.Into != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
path := filepath.Clean(r.Target)
|
||||||
|
if dirs[filepath.Dir(path)] {
|
||||||
|
made[filepath.Base(path)] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return made
|
||||||
|
}
|
||||||
|
|||||||
+104
-26
@@ -348,33 +348,111 @@ func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestADroppedServiceIsStoppedNotDeleted(t *testing.T) {
|
func TestADroppedServiceIsGivenBackTheStateItWasFoundIn(t *testing.T) {
|
||||||
// The host did not install the unit and does not own the unit file — only the state it put
|
// novox/hq ADR 0118: undeclaring removes what the mesh made, gives back what it changed, and
|
||||||
// the unit into.
|
// leaves what was the machine's. A service resource never installs a unit — so undeclaring it
|
||||||
var commands []string
|
// undoes what the mesh did to the unit, and nothing more. Stopping every undeclared unit is
|
||||||
|
// how unassigning the private network stopped the container runtime (novox/hq issue 130).
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
found *store.FoundUnit
|
||||||
|
action string
|
||||||
|
stop bool
|
||||||
|
disable bool
|
||||||
|
}{
|
||||||
|
{"recorded before the host kept what it found", nil, "forgotten", false, false},
|
||||||
|
{"running before the mesh", &store.FoundUnit{State: "running"}, "forgotten", false, false},
|
||||||
|
{"running and enabled before the mesh", &store.FoundUnit{State: "running", Boot: "enabled"}, "forgotten", false, false},
|
||||||
|
{"started by the mesh", &store.FoundUnit{State: "stopped"}, "restored", true, false},
|
||||||
|
{"started and enabled by the mesh", &store.FoundUnit{State: "stopped", Boot: "disabled"}, "restored", true, true},
|
||||||
|
{"enabled by the mesh, running before it", &store.FoundUnit{State: "running", Boot: "disabled"}, "restored", false, true},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
var commands []string
|
||||||
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||||
|
commands = append(commands, strings.Join(args, " "))
|
||||||
|
if args[0] == "show" {
|
||||||
|
return "LoadState=loaded\nActiveState=active\n", nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
state := store.State{Resources: []store.Applied{
|
||||||
|
{ID: "s", Type: "service", Target: "unit.service", Found: c.found},
|
||||||
|
}}
|
||||||
|
d := parse(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"other","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
|
||||||
|
]}`)
|
||||||
|
report, after, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
joined := strings.Join(commands, "; ")
|
||||||
|
if stopped := strings.Contains(joined, "stop unit.service"); stopped != c.stop {
|
||||||
|
t.Errorf("stopped %v, want %v: %s", stopped, c.stop, joined)
|
||||||
|
}
|
||||||
|
if disabled := strings.Contains(joined, "disable unit.service"); disabled != c.disable {
|
||||||
|
t.Errorf("disabled %v, want %v: %s", disabled, c.disable, joined)
|
||||||
|
}
|
||||||
|
if strings.Contains(joined, "start unit.service") || strings.Contains(joined, "mask") {
|
||||||
|
t.Errorf("the host started or masked a unit on its way out: %s", joined)
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, "s"); o.Action != c.action {
|
||||||
|
t.Errorf("outcome %+v, want %s", o, c.action)
|
||||||
|
}
|
||||||
|
if _, still := after.Find("s"); still {
|
||||||
|
t.Error("the host still believes it owns the undeclared service")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAServiceRecordsTheStateItWasFoundInOnceAndCarriesIt(t *testing.T) {
|
||||||
|
// Read the first time the host applies the unit — before it starts or enables anything —
|
||||||
|
// and never again: by the next apply, the unit's state is the mesh's doing.
|
||||||
|
active := "inactive"
|
||||||
|
enabled := "disabled"
|
||||||
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||||
commands = append(commands, strings.Join(args, " "))
|
switch args[0] {
|
||||||
if args[0] == "show" {
|
case "show":
|
||||||
return "LoadState=loaded\nActiveState=active\n", nil
|
return "LoadState=loaded\nActiveState=" + active + "\n", nil
|
||||||
|
case "is-enabled":
|
||||||
|
return enabled, nil
|
||||||
|
case "start":
|
||||||
|
active = "active"
|
||||||
|
case "enable":
|
||||||
|
enabled = "enabled"
|
||||||
}
|
}
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
state := store.State{Resources: []store.Applied{
|
|
||||||
{ID: "s", Type: "service", Target: "gone.service"},
|
|
||||||
}}
|
|
||||||
d := parse(t, `{"declaration":1,"resources":[
|
d := parse(t, `{"declaration":1,"resources":[
|
||||||
{"id":"other","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
|
{"id":"s","type":"service","unit":"filter.service","state":"running","boot":"enabled"}
|
||||||
]}`)
|
]}`)
|
||||||
|
_, first, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
||||||
if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil, nil); err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
joined := strings.Join(commands, "; ")
|
rec, _ := first.Find("s")
|
||||||
if !strings.Contains(joined, "stop gone.service") {
|
if rec.Found == nil || rec.Found.State != "stopped" || rec.Found.Boot != "disabled" {
|
||||||
t.Errorf("the dropped service was not stopped: %s", joined)
|
t.Fatalf("first apply recorded %+v, want stopped and disabled", rec.Found)
|
||||||
}
|
}
|
||||||
if strings.Contains(joined, "disable") || strings.Contains(joined, "mask") {
|
_, second, err := Apply(context.Background(), archHost(t), d, first, store.OriginCarried, run, nil, nil)
|
||||||
t.Errorf("the host did more than stop a unit it does not own: %s", joined)
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec, _ = second.Find("s")
|
||||||
|
if rec.Found == nil || rec.Found.State != "stopped" {
|
||||||
|
t.Errorf("a later apply replaced what was found with what the mesh made: %+v", rec.Found)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A record from before the host kept what it found is not given one later.
|
||||||
|
old := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "filter.service"}}}
|
||||||
|
_, third, err := Apply(context.Background(), archHost(t), d, old, store.OriginCarried, run, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rec, _ = third.Find("s"); rec.Found != nil {
|
||||||
|
t.Errorf("a record that predates the field was given one after the mesh had acted: %+v", rec.Found)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -635,7 +713,7 @@ func TestAContainerThatExitsImmediatelyFailsTheApply(t *testing.T) {
|
|||||||
switch {
|
switch {
|
||||||
case args[0] == "info":
|
case args[0] == "info":
|
||||||
return "27.0\n", nil
|
return "27.0\n", nil
|
||||||
case args[0] == "inspect":
|
case args[0] == "container":
|
||||||
return "false\t" + "", nil // exists, not running
|
return "false\t" + "", nil // exists, not running
|
||||||
case args[0] == "run":
|
case args[0] == "run":
|
||||||
return "deadbeef\n", nil
|
return "deadbeef\n", nil
|
||||||
@@ -673,7 +751,7 @@ func TestAContainerWhoseDeclarationChangedIsReplaced(t *testing.T) {
|
|||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "info":
|
case "info":
|
||||||
return "27.0\n", nil
|
return "27.0\n", nil
|
||||||
case "inspect":
|
case "container":
|
||||||
if created {
|
if created {
|
||||||
return "true\t" + want, nil
|
return "true\t" + want, nil
|
||||||
}
|
}
|
||||||
@@ -711,7 +789,7 @@ func TestAContainerThatMatchesIsLeftAlone(t *testing.T) {
|
|||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "info":
|
case "info":
|
||||||
return "27.0\n", nil
|
return "27.0\n", nil
|
||||||
case "inspect":
|
case "container":
|
||||||
return "true\t" + spec, nil
|
return "true\t" + spec, nil
|
||||||
}
|
}
|
||||||
touched = true
|
touched = true
|
||||||
@@ -756,7 +834,7 @@ func TestAContainerIsRecreatedWhenARestartOnResourceChanged(t *testing.T) {
|
|||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "info":
|
case "info":
|
||||||
return "27.0\n", nil
|
return "27.0\n", nil
|
||||||
case "inspect":
|
case "container":
|
||||||
// Already there and running, created against the file as it was. After the host
|
// Already there and running, created against the file as it was. After the host
|
||||||
// recreates it, the runtime holds the one it just made — as a real one would.
|
// recreates it, the runtime holds the one it just made — as a real one would.
|
||||||
if created {
|
if created {
|
||||||
@@ -1000,7 +1078,7 @@ func TestAContainerUsesTheRuntimeTheMachineHas(t *testing.T) {
|
|||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "info":
|
case "info":
|
||||||
return "6.1.0\n", nil
|
return "6.1.0\n", nil
|
||||||
case "inspect":
|
case "container":
|
||||||
return "false\t\n", errors.New("no such container")
|
return "false\t\n", errors.New("no such container")
|
||||||
case "run":
|
case "run":
|
||||||
return "deadbeef\n", nil
|
return "deadbeef\n", nil
|
||||||
@@ -1326,7 +1404,7 @@ func TestAContainerIsGivenTheMeshsNames(t *testing.T) {
|
|||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "info":
|
case "info":
|
||||||
return "29.0.0\n", nil
|
return "29.0.0\n", nil
|
||||||
case "inspect":
|
case "container":
|
||||||
return "false\t\n", errors.New("no such container")
|
return "false\t\n", errors.New("no such container")
|
||||||
case "run":
|
case "run":
|
||||||
ran = args
|
ran = args
|
||||||
@@ -1361,7 +1439,7 @@ func TestAContainerGivenNoNamesIsRunAsBefore(t *testing.T) {
|
|||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "info":
|
case "info":
|
||||||
return "29.0.0\n", nil
|
return "29.0.0\n", nil
|
||||||
case "inspect":
|
case "container":
|
||||||
return "false\t\n", errors.New("no such container")
|
return "false\t\n", errors.New("no such container")
|
||||||
case "run":
|
case "run":
|
||||||
ran = args
|
ran = args
|
||||||
@@ -1546,7 +1624,7 @@ func TestAContainerStaleFromAnEarlierApplyIsReplaced(t *testing.T) {
|
|||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "info":
|
case "info":
|
||||||
return "27.0\n", nil
|
return "27.0\n", nil
|
||||||
case "inspect":
|
case "container":
|
||||||
if created {
|
if created {
|
||||||
return "true\t" + fresh, nil
|
return "true\t" + fresh, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,433 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A file written into a marked block, never over (novox/hq issue 128, ADR 0102).
|
||||||
|
//
|
||||||
|
// **The file is the machine's; the mesh owns lines in it.** The machine's hosts file is the case
|
||||||
|
// that needed it. The mesh wrote it whole — its own header, localhost, the machine's name and every
|
||||||
|
// name in the mesh — and on a workstation that file is shared: the distribution's lines, a local
|
||||||
|
// development tool's own marked blocks rewritten whenever its projects change, the operator's
|
||||||
|
// hand-added names. Written whole, all of those went at the next change to the mesh's names, with
|
||||||
|
// no failure anywhere: the tool believed it had written its block, and the mesh believed it owned
|
||||||
|
// the file. It is ADR 0102's failure exactly, in a file ADR 0102's JSON verb cannot speak.
|
||||||
|
//
|
||||||
|
// So the host finds the lines between `# BEGIN mesh <id>` and `# END mesh <id>`, rewrites those and
|
||||||
|
// nothing else, and records what they held before. Every line outside the markers is kept byte for
|
||||||
|
// byte — including another tool's `# BEGIN …` blocks, which are that tool's. Undeclared, the region
|
||||||
|
// is given back what it held, or taken out with its markers when it held nothing, and a file the
|
||||||
|
// mesh created goes only if nothing but whitespace is left.
|
||||||
|
|
||||||
|
// applyBlock writes a file's declared lines into its region of the file already at its path.
|
||||||
|
//
|
||||||
|
// **A link stays a link.** Where the path is a symbolic link — a hosts file some distributions keep
|
||||||
|
// elsewhere and link into /etc — the file read, written and renamed over is the one it points to,
|
||||||
|
// so the link and whatever manages it are left as they were. A file written whole, or into JSON,
|
||||||
|
// still replaces a link with a file; that is unchanged here.
|
||||||
|
func applyBlock(r *declaration.File, previous store.Applied) (Outcome, error) {
|
||||||
|
out := begin(r)
|
||||||
|
opening, closing := declaration.BlockMarkers(r.ID)
|
||||||
|
want := blockBody(r.Content)
|
||||||
|
|
||||||
|
real, err := realPath(r.Path)
|
||||||
|
if err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(real)
|
||||||
|
existed := err == nil
|
||||||
|
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
// What the file is, taken once with what it holds: its mode and owner are the machine's and
|
||||||
|
// go back onto what is written. A file read and then not there to stat is a failure, never a
|
||||||
|
// file with no owner.
|
||||||
|
var info os.FileInfo
|
||||||
|
if existed {
|
||||||
|
if info, err = os.Stat(real); err != nil {
|
||||||
|
return out, fmt.Errorf("read %s and cannot see it: %w", r.Path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
existing := string(raw)
|
||||||
|
|
||||||
|
rec := store.Into{Format: declaration.IntoBlock}
|
||||||
|
var note string
|
||||||
|
rebuilt := false
|
||||||
|
|
||||||
|
// **A file the mesh once wrote whole** (novox/hq issue 128). The resource keeps its id when its
|
||||||
|
// module moves from writing the file whole to writing into it, and the file on the machine is
|
||||||
|
// then the mesh's own old write — its header, its loopback lines, its names. Adding the region
|
||||||
|
// after that would leave the old names above the new ones, and a resolver takes the first
|
||||||
|
// line that answers: the region would be shadowed by what it replaced. So the file is rebuilt:
|
||||||
|
// the original the mesh kept before its first write, with the region in it; or, where the mesh
|
||||||
|
// made the file itself, the loopback lines every machine needs, kept as the machine's, with the
|
||||||
|
// region beside them. Changed since the mesh wrote it, the file is somebody's again and is
|
||||||
|
// written into as it stands, and the outcome says so.
|
||||||
|
if existed && previous.Into == nil && previous.Wrote != "" {
|
||||||
|
if digestOf(existing) == previous.Wrote {
|
||||||
|
if previous.Kept != "" {
|
||||||
|
original, err := os.ReadFile(previous.Kept)
|
||||||
|
if err != nil {
|
||||||
|
return out, fmt.Errorf("%s was written whole by the mesh over an original kept at %s, "+
|
||||||
|
"which cannot be read to give it back: %w; it was left as it is", r.Path, previous.Kept, err)
|
||||||
|
}
|
||||||
|
existing = string(original)
|
||||||
|
note = "the mesh's old whole file replaced by the original kept at " + previous.Kept + ", with the region in it"
|
||||||
|
} else {
|
||||||
|
existing = loopbackOf(existing)
|
||||||
|
rec.Created = true
|
||||||
|
note = "the mesh's old whole file replaced by its loopback lines and the region"
|
||||||
|
}
|
||||||
|
// Not what was read: the whole of it was the mesh's, and the file is written afresh.
|
||||||
|
rebuilt = true
|
||||||
|
} else {
|
||||||
|
note = "a file the mesh once wrote whole, changed since; its old lines were kept"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := linesOf(existing)
|
||||||
|
at, found, err := regionIn(lines, opening, closing)
|
||||||
|
if err != nil {
|
||||||
|
// Refused, never guessed at: markers the host cannot pair are markers it cannot write
|
||||||
|
// between without risking lines that are not the mesh's.
|
||||||
|
return out, fmt.Errorf("%s: %w; it was left as it is", r.Path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A record of a block is carried; anything else — no record, a file once written whole, one
|
||||||
|
// once written into as JSON — is a file the host is seeing for the first time as a block.
|
||||||
|
recorded := previous.Into != nil && previous.Into.Format == declaration.IntoBlock
|
||||||
|
if recorded && existed {
|
||||||
|
rec.Created = previous.Into.Created
|
||||||
|
rec.Region = previous.Into.Region
|
||||||
|
rec.Separated = previous.Into.Separated
|
||||||
|
rec.At = previous.Into.At
|
||||||
|
rec.Ended = previous.Into.Ended
|
||||||
|
} else if !rebuilt {
|
||||||
|
// A file gone since the last apply is made again, and made by the mesh: what it held
|
||||||
|
// before went with it, so there is nothing to give back but the file's absence.
|
||||||
|
rec.Created = !existed
|
||||||
|
if found {
|
||||||
|
// **What the host may have written itself is not the machine's** — the same reasoning
|
||||||
|
// as a key in a JSON file (novox/hq ADR 0102). With no record, a region already holding
|
||||||
|
// exactly the declared lines cannot be told from one this host wrote a moment ago and
|
||||||
|
// died before saving; remembered as the machine's, it would be put back on undeclare
|
||||||
|
// for ever. So it is the mesh's, and undeclaring takes it out.
|
||||||
|
if held := at.body(lines); held != want {
|
||||||
|
rec.Region = &held
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drift: the machine no longer holds, between the mesh's markers, what this host last put
|
||||||
|
// there. Judged only against a record of a block: a digest of a whole file says nothing about
|
||||||
|
// a region of it.
|
||||||
|
drifted := recorded && previous.Wrote != "" && existed &&
|
||||||
|
(!found || digestOf(at.body(lines)) != previous.Wrote)
|
||||||
|
|
||||||
|
var next string
|
||||||
|
switch {
|
||||||
|
case !existed:
|
||||||
|
next = regionOf(opening, closing, want)
|
||||||
|
case found:
|
||||||
|
// Where it is, whatever At says: the region is never moved, because moving it moves the
|
||||||
|
// machine's lines around it.
|
||||||
|
next = strings.Join(lines[:at.begin+1], "") + want + strings.Join(lines[at.end:], "")
|
||||||
|
case r.At == declaration.AtStart:
|
||||||
|
// Above everything, and one blank line between the region and the machine's first line
|
||||||
|
// unless there is one already — a line in some files means what the lines above it say.
|
||||||
|
rec.At, rec.Separated, rec.Ended = declaration.AtStart, false, false
|
||||||
|
next = regionOf(opening, closing, want)
|
||||||
|
if existing != "" && !strings.HasPrefix(existing, "\n") {
|
||||||
|
next += "\n"
|
||||||
|
rec.Separated = true
|
||||||
|
}
|
||||||
|
next += existing
|
||||||
|
default:
|
||||||
|
// At the end, apart from whatever is there: the file's last line is ended if it was not,
|
||||||
|
// and one blank line separates the region from the machine's lines unless there is one.
|
||||||
|
rec.At, rec.Separated, rec.Ended = "", false, false
|
||||||
|
next = existing
|
||||||
|
if next != "" && !strings.HasSuffix(next, "\n") {
|
||||||
|
next += "\n"
|
||||||
|
rec.Ended = true
|
||||||
|
}
|
||||||
|
if next != "" && next != "\n" && !strings.HasSuffix(next, "\n\n") {
|
||||||
|
next += "\n"
|
||||||
|
rec.Separated = true
|
||||||
|
}
|
||||||
|
next += regionOf(opening, closing, want)
|
||||||
|
}
|
||||||
|
// What was not the mesh's is what it was. By construction — and checked, because a slip in
|
||||||
|
// splicing lines is exactly the fault this mode exists to prevent, and it must never be written.
|
||||||
|
if found {
|
||||||
|
after := linesOf(next)
|
||||||
|
if where, ok, err := regionIn(after, opening, closing); err != nil || !ok || outside(after, where) != outside(lines, at) {
|
||||||
|
return out, fmt.Errorf("%s: writing the region would change lines outside it; it was left as it is", r.Path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
same := existed && next == string(raw)
|
||||||
|
if !same {
|
||||||
|
mode := os.FileMode(0o644)
|
||||||
|
if info != nil {
|
||||||
|
mode = info.Mode().Perm() // the machine's file keeps the machine's mode
|
||||||
|
} else if mode, err = modeOf(r.Mode, mode); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(real), 0o755); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if err := writeAtomically(real, []byte(next), mode); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
if info != nil {
|
||||||
|
// The write is a new file renamed over the old, so it belongs to whoever wrote it. The
|
||||||
|
// machine's file keeps the machine's owner, as it keeps its mode.
|
||||||
|
if err := keepOwner(real, info); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
} else if err := own(real, r.Owner); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read back: the region holds what was declared. Only the region — another tool writing its
|
||||||
|
// own lines in the moment after the rename is not a failed write. What remains is the moment
|
||||||
|
// between reading the file and renaming over it: a line another tool writes there is lost, and
|
||||||
|
// found again at its next write. Nothing short of a lock every writer honours closes that, and
|
||||||
|
// the other writers of a hosts file honour none.
|
||||||
|
written, err := os.ReadFile(real)
|
||||||
|
if err != nil {
|
||||||
|
return out, fmt.Errorf("wrote into %s and cannot read it back: %w", r.Path, err)
|
||||||
|
}
|
||||||
|
back := linesOf(string(written))
|
||||||
|
if where, ok, err := regionIn(back, opening, closing); err != nil || !ok || where.body(back) != want {
|
||||||
|
return out, fmt.Errorf("%s does not hold the mesh's region after writing into it", r.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
out.into = &rec
|
||||||
|
out.wrote = digestOf(want)
|
||||||
|
switch {
|
||||||
|
case note != "" && !same:
|
||||||
|
out.Action = "updated"
|
||||||
|
out.Detail = note
|
||||||
|
case !existed:
|
||||||
|
out.Action = "created"
|
||||||
|
out.Detail = "written into; the file was not there"
|
||||||
|
case same:
|
||||||
|
out.Action = "unchanged"
|
||||||
|
case drifted:
|
||||||
|
out.Action = "corrected"
|
||||||
|
out.Detail = "the mesh's region had been changed on the machine; every line outside it was kept"
|
||||||
|
case !found:
|
||||||
|
out.Action = "updated"
|
||||||
|
where := "end"
|
||||||
|
if rec.At == declaration.AtStart {
|
||||||
|
where = "start"
|
||||||
|
}
|
||||||
|
out.Detail = "the mesh's region added at the " + where + "; every other line kept as it was"
|
||||||
|
default:
|
||||||
|
out.Action = "updated"
|
||||||
|
out.Detail = "the mesh's region rewritten; every line outside it kept as it was"
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeBlock gives back what a file written into a block held before the mesh's region.
|
||||||
|
func removeBlock(a store.Applied) (string, string, error) {
|
||||||
|
real, err := realPath(a.Target)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(real)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return "forgotten", "no longer there", nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
info, err := os.Stat(real)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", fmt.Errorf("read %s and cannot see it: %w", a.Target, err)
|
||||||
|
}
|
||||||
|
opening, closing := declaration.BlockMarkers(a.ID)
|
||||||
|
lines := linesOf(string(raw))
|
||||||
|
at, found, err := regionIn(lines, opening, closing)
|
||||||
|
if err != nil {
|
||||||
|
return "kept", err.Error() + ", so nothing was taken out of it; remove the mesh's region by hand", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
next, action, detail := string(raw), "forgotten", "the mesh's region was no longer in it"
|
||||||
|
switch {
|
||||||
|
case found && a.Into.Region != nil:
|
||||||
|
next = strings.Join(lines[:at.begin+1], "") + *a.Into.Region + strings.Join(lines[at.end:], "")
|
||||||
|
action, detail = "restored", "no longer declared; the region was given back what it held"
|
||||||
|
case found:
|
||||||
|
from, to := at.begin, at.end+1
|
||||||
|
// The blank line the host added beside the region, when a blank line still stands there.
|
||||||
|
// Whether it is the same one the host added cannot be known from the file; a blank line
|
||||||
|
// is the one line whose going changes nothing any program reads, so it is taken. A line
|
||||||
|
// that is not blank is never taken, whoever put it there.
|
||||||
|
if a.Into.Separated {
|
||||||
|
if a.Into.At == declaration.AtStart {
|
||||||
|
if to < len(lines) && lines[to] == "\n" {
|
||||||
|
to++
|
||||||
|
}
|
||||||
|
} else if from > 0 && lines[from-1] == "\n" {
|
||||||
|
from--
|
||||||
|
}
|
||||||
|
}
|
||||||
|
next = strings.Join(lines[:from], "") + strings.Join(lines[to:], "")
|
||||||
|
// And the line end the host gave the machine's last line, if that line is still last.
|
||||||
|
if a.Into.Ended && strings.Join(lines[to:], "") == "" {
|
||||||
|
next = strings.TrimSuffix(next, "\n")
|
||||||
|
}
|
||||||
|
action, detail = "restored", "no longer declared; the mesh's region was taken out and every other line kept"
|
||||||
|
}
|
||||||
|
|
||||||
|
if a.Into.Created && strings.TrimSpace(next) == "" && real == a.Target {
|
||||||
|
if err := os.Remove(real); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
return "removed", "no longer declared; the mesh had created it and nothing else was in it", nil
|
||||||
|
}
|
||||||
|
if next == string(raw) {
|
||||||
|
return action, detail, nil
|
||||||
|
}
|
||||||
|
if err := writeAtomically(real, []byte(next), info.Mode().Perm()); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
if err := keepOwner(real, info); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
return action, detail, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// realPath is the file a path names, through any links; a path that is not there yet is itself.
|
||||||
|
// A link to nothing is refused: writing through it would replace the link with a file.
|
||||||
|
func realPath(path string) (string, error) {
|
||||||
|
real, err := filepath.EvalSymlinks(path)
|
||||||
|
if err == nil {
|
||||||
|
return real, nil
|
||||||
|
}
|
||||||
|
if _, lerr := os.Lstat(path); errors.Is(lerr, os.ErrNotExist) {
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("%s is a link the host cannot follow to a file: %w; it was left as it is", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// loopbackOf is the lines of a file that answer for the machine itself — localhost, its own name on
|
||||||
|
// 127.0.1.1, ::1 — and nothing else: what the mesh's old whole hosts file carried that the machine
|
||||||
|
// needs, without the mesh's header or its names.
|
||||||
|
func loopbackOf(text string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, line := range linesOf(text) {
|
||||||
|
fields := strings.Fields(line)
|
||||||
|
if len(fields) < 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ip := net.ParseIP(fields[0]); ip != nil && ip.IsLoopback() {
|
||||||
|
b.WriteString(strings.TrimSuffix(line, "\n") + "\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// outside is every line of a file but the mesh's region, markers included, as one string.
|
||||||
|
func outside(lines []string, at region) string {
|
||||||
|
end := at.end + 1
|
||||||
|
if end > len(lines) {
|
||||||
|
end = len(lines)
|
||||||
|
}
|
||||||
|
return strings.Join(lines[:at.begin], "") + "\x00" + strings.Join(lines[end:], "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// blockBody is the declared lines as they stand in the region: ending in exactly one line end, or
|
||||||
|
// nothing at all when there are no lines.
|
||||||
|
func blockBody(content string) string {
|
||||||
|
trimmed := strings.TrimRight(content, "\n")
|
||||||
|
if trimmed == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return trimmed + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
func regionOf(begin, end, body string) string {
|
||||||
|
return begin + "\n" + body + end + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// linesOf splits text into lines that keep their line ends, so joining them again gives back
|
||||||
|
// exactly the bytes that were read — a last line without one included.
|
||||||
|
func linesOf(text string) []string {
|
||||||
|
return strings.SplitAfter(text, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// region is where the mesh's markers stand, as indices into the lines of a file.
|
||||||
|
type region struct{ begin, end int }
|
||||||
|
|
||||||
|
// body is what stands between the markers.
|
||||||
|
func (r region) body(lines []string) string {
|
||||||
|
return strings.Join(lines[r.begin+1:r.end], "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// regionIn finds the mesh's markers for one resource. A line is a marker only if it is exactly the
|
||||||
|
// marker, so another tool's block and another resource's region are never it. Markers that do not
|
||||||
|
// form one pair — a begin with no end, an end before its begin, either twice — are an error rather
|
||||||
|
// than a best guess, because a guess is how the host would rewrite lines that are not its own.
|
||||||
|
func regionIn(lines []string, begin, end string) (region, bool, error) {
|
||||||
|
at := region{begin: -1, end: -1}
|
||||||
|
for i, line := range lines {
|
||||||
|
switch strings.TrimSuffix(line, "\n") {
|
||||||
|
case begin:
|
||||||
|
if at.begin >= 0 {
|
||||||
|
return at, false, fmt.Errorf("%q is in it more than once", begin)
|
||||||
|
}
|
||||||
|
at.begin = i
|
||||||
|
case end:
|
||||||
|
if at.end >= 0 {
|
||||||
|
return at, false, fmt.Errorf("%q is in it more than once", end)
|
||||||
|
}
|
||||||
|
at.end = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case at.begin < 0 && at.end < 0:
|
||||||
|
return at, false, nil
|
||||||
|
case at.begin < 0:
|
||||||
|
return at, false, fmt.Errorf("%q is in it with no %q before it", end, begin)
|
||||||
|
case at.end < 0:
|
||||||
|
return at, false, fmt.Errorf("%q is in it with no %q after it", begin, end)
|
||||||
|
case at.end < at.begin:
|
||||||
|
return at, false, fmt.Errorf("%q stands before %q", end, begin)
|
||||||
|
}
|
||||||
|
return at, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepOwner gives a file rewritten through a new one back to whoever owned what it replaced.
|
||||||
|
// Changed only where it differs, so a host that is not root can still write a file it owns.
|
||||||
|
func keepOwner(path string, was os.FileInfo) error {
|
||||||
|
uid, gid, ok := ownerOf(was)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
now, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if u, g, ok := ownerOf(now); ok && u == uid && g == gid {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.Chown(path, uid, gid); err != nil {
|
||||||
|
return fmt.Errorf("cannot give %s back to its owner %d:%d: %w", path, uid, gid, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,618 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq issue 128 and ADR 0102: a text file the mesh shares with software it did not
|
||||||
|
// install is written into a marked block, never over — every line outside the mesh's markers is
|
||||||
|
// the machine's and is kept byte for byte, and undeclaring gives the file back.
|
||||||
|
|
||||||
|
const namesID = "mesh-wireguard.fact-node-names"
|
||||||
|
|
||||||
|
func blockDecl(t *testing.T, path, content string, extra ...string) string {
|
||||||
|
t.Helper()
|
||||||
|
more := ""
|
||||||
|
for _, e := range extra {
|
||||||
|
more += "," + e
|
||||||
|
}
|
||||||
|
return fmt.Sprintf(`{"declaration":1,"resources":[
|
||||||
|
{"id":%q,"type":"file","path":%q,"into":"block","content":%q%s}
|
||||||
|
]}`, namesID, path, content, more)
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyBlockDecl(t *testing.T, raw string, known store.State) (Report, store.State) {
|
||||||
|
t.Helper()
|
||||||
|
report, state, err := Apply(context.Background(), archHost(t), parse(t, raw), known, store.OriginDeclared, nil, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return report, state
|
||||||
|
}
|
||||||
|
|
||||||
|
func undeclare(t *testing.T, known store.State) (Report, store.State) {
|
||||||
|
t.Helper()
|
||||||
|
report, state, err := Apply(context.Background(), archHost(t), somethingElse(t), known, store.OriginDeclared, nil, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return report, state
|
||||||
|
}
|
||||||
|
|
||||||
|
func readText(t *testing.T, path string) string {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
func marked(id, body string) string {
|
||||||
|
return "# BEGIN mesh " + id + "\n" + body + "# END mesh " + id + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// A workstation's hosts file, the way issue 128 found it: the distribution's lines, a development
|
||||||
|
// tool's own marked blocks, and the operator's hand-added names.
|
||||||
|
const workstationHosts = "127.0.0.1\tlocalhost\n" +
|
||||||
|
"127.0.1.1\tg14.localdomain g14\n" +
|
||||||
|
"\n" +
|
||||||
|
"# BEGIN devtool project-a\n" +
|
||||||
|
"127.0.0.1 a.test api.a.test\n" +
|
||||||
|
"# END devtool project-a\n" +
|
||||||
|
"# BEGIN devtool project-b\n" +
|
||||||
|
"127.0.0.1 b.test\n" +
|
||||||
|
"# END devtool project-b\n" +
|
||||||
|
"192.168.1.20 printer # the operator's\n"
|
||||||
|
|
||||||
|
const meshNames = "10.42.0.1 ace\n10.42.0.2 novox\n"
|
||||||
|
|
||||||
|
func TestABlockKeepsEveryLineOutsideItsMarkers(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
if err := os.WriteFile(path, []byte(workstationHosts), 0o640); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
|
||||||
|
want := workstationHosts + "\n" + marked(namesID, meshNames)
|
||||||
|
if got := readText(t, path); got != want {
|
||||||
|
t.Fatalf("the file after writing into it:\n%q\nwant\n%q", got, want)
|
||||||
|
}
|
||||||
|
if got := report.Outcomes[0].Action; got != "updated" {
|
||||||
|
t.Errorf("adding the region was %q", got)
|
||||||
|
}
|
||||||
|
if info, _ := os.Stat(path); info.Mode().Perm() != 0o640 {
|
||||||
|
t.Errorf("the machine's file mode was changed to %o", info.Mode().Perm())
|
||||||
|
}
|
||||||
|
rec, _ := state.Find(namesID)
|
||||||
|
if rec.Into == nil || rec.Into.Format != "block" || rec.Into.Region != nil || rec.Into.Created {
|
||||||
|
t.Fatalf("recorded as %+v", rec.Into)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Again, with nothing changed: nothing written.
|
||||||
|
report, state = applyBlockDecl(t, blockDecl(t, path, meshNames), state)
|
||||||
|
if got := report.Outcomes[0].Action; got != "unchanged" {
|
||||||
|
t.Errorf("a second apply was %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The development tool rewrites its block, and the operator adds a line after the mesh's
|
||||||
|
// region; the mesh's names change. Only the region moves.
|
||||||
|
edited := strings.Replace(readText(t, path), "127.0.0.1 b.test\n", "127.0.0.1 b.test c.test\n", 1) +
|
||||||
|
"10.0.0.5 nas # added after\n"
|
||||||
|
_ = os.WriteFile(path, []byte(edited), 0o640)
|
||||||
|
changed := meshNames + "10.42.0.3 shanks\n"
|
||||||
|
report, state = applyBlockDecl(t, blockDecl(t, path, changed), state)
|
||||||
|
want = strings.Replace(edited, marked(namesID, meshNames), marked(namesID, changed), 1)
|
||||||
|
if got := readText(t, path); got != want {
|
||||||
|
t.Fatalf("rewriting the region moved something else:\n%q\nwant\n%q", got, want)
|
||||||
|
}
|
||||||
|
if got := report.Outcomes[0].Action; got != "updated" {
|
||||||
|
t.Errorf("rewriting the region was %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Undeclared: the region, its markers and the blank line the host put before it go; every
|
||||||
|
// other line is where it was.
|
||||||
|
report, _ = undeclare(t, state)
|
||||||
|
want = strings.Replace(edited, "\n"+marked(namesID, meshNames), "", 1)
|
||||||
|
if got := readText(t, path); got != want {
|
||||||
|
t.Fatalf("undeclaring left:\n%q\nwant\n%q", got, want)
|
||||||
|
}
|
||||||
|
if got := report.Outcomes[0].Action; got != "restored" {
|
||||||
|
t.Errorf("undeclaring was %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUndeclaringABlockAddedAtTheEndGivesTheFileBackExactly(t *testing.T) {
|
||||||
|
for name, original := range map[string]string{
|
||||||
|
"ending in a line": "127.0.0.1 localhost\n",
|
||||||
|
"ending in a blank line": "127.0.0.1 localhost\n\n",
|
||||||
|
"empty": "",
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
_ = os.WriteFile(path, []byte(original), 0o644)
|
||||||
|
_, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
|
||||||
|
undeclare(t, state)
|
||||||
|
if got := readText(t, path); got != original {
|
||||||
|
t.Errorf("undeclaring left %q, the machine had %q", got, original)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABlockAddedAtTheEndIsSetApartFromTheMachinesLines(t *testing.T) {
|
||||||
|
for name, c := range map[string]struct{ before, after string }{
|
||||||
|
"no line end": {"127.0.0.1 localhost", "127.0.0.1 localhost\n\n" + marked(namesID, meshNames)},
|
||||||
|
"a line end": {"127.0.0.1 localhost\n", "127.0.0.1 localhost\n\n" + marked(namesID, meshNames)},
|
||||||
|
"a blank line already": {"127.0.0.1 localhost\n\n", "127.0.0.1 localhost\n\n" + marked(namesID, meshNames)},
|
||||||
|
"empty": {"", marked(namesID, meshNames)},
|
||||||
|
"only a blank line": {"\n", "\n" + marked(namesID, meshNames)},
|
||||||
|
"content without an end": {"x\n\n", "x\n\n" + marked(namesID, meshNames)},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
_ = os.WriteFile(path, []byte(c.before), 0o644)
|
||||||
|
applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
|
||||||
|
if got := readText(t, path); got != c.after {
|
||||||
|
t.Errorf("got %q, want %q", got, c.after)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheRegionEndsInExactlyOneLineEnd(t *testing.T) {
|
||||||
|
for content, body := range map[string]string{
|
||||||
|
"10.42.0.1 ace": "10.42.0.1 ace\n",
|
||||||
|
"10.42.0.1 ace\n": "10.42.0.1 ace\n",
|
||||||
|
"10.42.0.1 ace\n\n\n": "10.42.0.1 ace\n",
|
||||||
|
"": "",
|
||||||
|
"\n\n": "",
|
||||||
|
} {
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
_, state := applyBlockDecl(t, blockDecl(t, path, content), store.State{})
|
||||||
|
if got := readText(t, path); got != marked(namesID, body) {
|
||||||
|
t.Errorf("content %q was written as %q", content, got)
|
||||||
|
}
|
||||||
|
// And the same content again is not a change.
|
||||||
|
report, _ := applyBlockDecl(t, blockDecl(t, path, content), state)
|
||||||
|
if got := report.Outcomes[0].Action; got != "unchanged" {
|
||||||
|
t.Errorf("content %q applied twice was %q", content, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARegionAlreadyThereIsRewrittenInPlaceAndGivenBack(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
before := "127.0.0.1 localhost\n"
|
||||||
|
after := "# BEGIN devtool x\n127.0.0.1 x.test\n# END devtool x\n192.168.1.20 printer\n"
|
||||||
|
found := "10.42.0.9 old-name\n"
|
||||||
|
original := before + marked(namesID, found) + after
|
||||||
|
_ = os.WriteFile(path, []byte(original), 0o644)
|
||||||
|
|
||||||
|
report, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
|
||||||
|
if got := readText(t, path); got != before+marked(namesID, meshNames)+after {
|
||||||
|
t.Fatalf("the region was not rewritten in place: %q", got)
|
||||||
|
}
|
||||||
|
if got := report.Outcomes[0].Action; got != "updated" {
|
||||||
|
t.Errorf("rewriting a found region was %q", got)
|
||||||
|
}
|
||||||
|
rec, _ := state.Find(namesID)
|
||||||
|
if rec.Into.Region == nil || *rec.Into.Region != found {
|
||||||
|
t.Fatalf("what the region held before was recorded as %v", rec.Into.Region)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Undeclared: what the region held goes back, where it was.
|
||||||
|
report, _ = undeclare(t, state)
|
||||||
|
if got := readText(t, path); got != original {
|
||||||
|
t.Errorf("undeclaring left %q, the machine had %q", got, original)
|
||||||
|
}
|
||||||
|
if got := report.Outcomes[0].Action; got != "restored" {
|
||||||
|
t.Errorf("undeclaring was %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARegionWithNoRecordHoldingExactlyTheDeclaredLinesIsTheMeshs(t *testing.T) {
|
||||||
|
// A host that wrote the region and died before saving its state: what is between the markers
|
||||||
|
// is exactly what the mesh declares, and remembered as the machine's it would never go.
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
original := "127.0.0.1 localhost\n"
|
||||||
|
_ = os.WriteFile(path, []byte(original+"\n"+marked(namesID, meshNames)), 0o644)
|
||||||
|
_, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
|
||||||
|
if rec, _ := state.Find(namesID); rec.Into.Region != nil {
|
||||||
|
t.Fatalf("the mesh's own lines were recorded as the machine's: %q", *rec.Into.Region)
|
||||||
|
}
|
||||||
|
undeclare(t, state)
|
||||||
|
if got := readText(t, path); !strings.HasPrefix(got, original) || strings.Contains(got, "BEGIN mesh") {
|
||||||
|
t.Errorf("undeclaring left the mesh's region behind: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADriftedRegionIsCorrected(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
_ = os.WriteFile(path, []byte(workstationHosts), 0o644)
|
||||||
|
_, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
|
||||||
|
written := readText(t, path)
|
||||||
|
|
||||||
|
_ = os.WriteFile(path, []byte(strings.Replace(written, "10.42.0.2 novox\n", "10.42.0.2 novox\n6.6.6.6 evil\n", 1)), 0o644)
|
||||||
|
report, _ := applyBlockDecl(t, blockDecl(t, path, meshNames), state)
|
||||||
|
if got := report.Outcomes[0].Action; got != "corrected" {
|
||||||
|
t.Errorf("a region edited on the machine was %q", got)
|
||||||
|
}
|
||||||
|
if got := readText(t, path); got != written {
|
||||||
|
t.Errorf("the region was not put back: %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The region taken out by hand is drift too, and it is put back.
|
||||||
|
_ = os.WriteFile(path, []byte(workstationHosts), 0o644)
|
||||||
|
report, _ = applyBlockDecl(t, blockDecl(t, path, meshNames), state)
|
||||||
|
if got := report.Outcomes[0].Action; got != "corrected" {
|
||||||
|
t.Errorf("a region removed on the machine was %q", got)
|
||||||
|
}
|
||||||
|
if got := readText(t, path); got != written {
|
||||||
|
t.Errorf("the region was not put back: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTwoRegionsInOneFileAreEachTheirOwn(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
_ = os.WriteFile(path, []byte(workstationHosts), 0o644)
|
||||||
|
raw := fmt.Sprintf(`{"declaration":1,"resources":[
|
||||||
|
{"id":"a.names","type":"file","path":%q,"into":"block","content":"10.42.0.1 ace\n"},
|
||||||
|
{"id":"b.names","type":"file","path":%q,"into":"block","content":"10.43.0.1 lab\n"}
|
||||||
|
]}`, path, path)
|
||||||
|
_, state := applyBlockDecl(t, raw, store.State{})
|
||||||
|
want := workstationHosts + "\n" + marked("a.names", "10.42.0.1 ace\n") + "\n" + marked("b.names", "10.43.0.1 lab\n")
|
||||||
|
if got := readText(t, path); got != want {
|
||||||
|
t.Fatalf("two regions:\n%q\nwant\n%q", got, want)
|
||||||
|
}
|
||||||
|
report, state := applyBlockDecl(t, raw, state)
|
||||||
|
for _, o := range report.Outcomes {
|
||||||
|
if o.Action != "unchanged" {
|
||||||
|
t.Errorf("%s applied twice was %q", o.ID, o.Action)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One undeclared: only its region goes.
|
||||||
|
only := fmt.Sprintf(`{"declaration":1,"resources":[
|
||||||
|
{"id":"b.names","type":"file","path":%q,"into":"block","content":"10.43.0.1 lab\n"}
|
||||||
|
]}`, path)
|
||||||
|
applyBlockDecl(t, only, state)
|
||||||
|
want = workstationHosts + "\n" + marked("b.names", "10.43.0.1 lab\n")
|
||||||
|
if got := readText(t, path); got != want {
|
||||||
|
t.Errorf("undeclaring one region:\n%q\nwant\n%q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABlockInAFileThatWasNotThereIsCreatedAndRemovedWithIt(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "conf.d", "mesh.conf")
|
||||||
|
report, state := applyBlockDecl(t, blockDecl(t, path, meshNames, `"mode":"0600"`), store.State{})
|
||||||
|
if got := readText(t, path); got != marked(namesID, meshNames) {
|
||||||
|
t.Fatalf("a created file holds %q", got)
|
||||||
|
}
|
||||||
|
if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 {
|
||||||
|
t.Errorf("a created file is mode %o, declared 0600", info.Mode().Perm())
|
||||||
|
}
|
||||||
|
if got := report.Outcomes[0].Action; got != "created" {
|
||||||
|
t.Errorf("creating was %q", got)
|
||||||
|
}
|
||||||
|
if rec, _ := state.Find(namesID); !rec.Into.Created {
|
||||||
|
t.Error("the mesh creating the file was not recorded")
|
||||||
|
}
|
||||||
|
report, _ = undeclare(t, state)
|
||||||
|
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||||
|
t.Errorf("a file the mesh created, holding only its region, was left behind")
|
||||||
|
}
|
||||||
|
if got := report.Outcomes[0].Action; got != "removed" {
|
||||||
|
t.Errorf("undeclaring was %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Somebody else wrote into it meanwhile: it is no longer only the mesh's, and it stays.
|
||||||
|
_, state = applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
|
||||||
|
_ = os.WriteFile(path, []byte(readText(t, path)+"their = line\n"), 0o600)
|
||||||
|
undeclare(t, state)
|
||||||
|
if got := readText(t, path); got != "their = line\n" {
|
||||||
|
t.Errorf("undeclaring a created file somebody wrote into left %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarkersThatDoNotPairAreRefusedAndLeftAlone(t *testing.T) {
|
||||||
|
for name, text := range map[string]string{
|
||||||
|
"a begin with no end": "a\n# BEGIN mesh " + namesID + "\nb\n",
|
||||||
|
"an end with no begin": "a\n# END mesh " + namesID + "\n",
|
||||||
|
"an end before a begin": "# END mesh " + namesID + "\n# BEGIN mesh " + namesID + "\n",
|
||||||
|
"a begin twice": marked(namesID, "x\n") + "# BEGIN mesh " + namesID + "\n",
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
_ = os.WriteFile(path, []byte(text), 0o644)
|
||||||
|
if _, _, err := Apply(context.Background(), archHost(t), parse(t, blockDecl(t, path, meshNames)),
|
||||||
|
store.State{}, store.OriginDeclared, nil, nil, nil); err == nil {
|
||||||
|
t.Fatal("markers that do not pair were written between")
|
||||||
|
}
|
||||||
|
if got := readText(t, path); got != text {
|
||||||
|
t.Errorf("the file was changed: %q", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMarkerOfAnotherIDIsNotThisRegion(t *testing.T) {
|
||||||
|
// The id is part of the marker: a region whose id merely starts with this one is not it.
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
other := marked(namesID+"-extra", "10.9.9.9 other\n")
|
||||||
|
_ = os.WriteFile(path, []byte(other), 0o644)
|
||||||
|
_, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
|
||||||
|
if got := readText(t, path); got != other+"\n"+marked(namesID, meshNames) {
|
||||||
|
t.Fatalf("got %q", got)
|
||||||
|
}
|
||||||
|
undeclare(t, state)
|
||||||
|
if got := readText(t, path); got != other {
|
||||||
|
t.Errorf("undeclaring touched the other region: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABlockAtTheStartStandsAboveEverything(t *testing.T) {
|
||||||
|
// dhcpcd scopes every line after `interface X` to that interface, so the mesh's global options
|
||||||
|
// go above all of it.
|
||||||
|
dhcpcd := "hostname\nduid\n\ninterface enp6s0\nstatic ip_address=192.168.1.5/24\n"
|
||||||
|
opts := "nohook resolv.conf\ndenyinterfaces mesh0\n"
|
||||||
|
for name, c := range map[string]struct{ before, after string }{
|
||||||
|
"a file with content": {dhcpcd, marked(namesID, opts) + "\n" + dhcpcd},
|
||||||
|
"an empty file": {"", marked(namesID, opts)},
|
||||||
|
"a file opening blank": {"\n" + dhcpcd, marked(namesID, opts) + "\n" + dhcpcd},
|
||||||
|
"a last line with no end": {"interface enp6s0", marked(namesID, opts) + "\ninterface enp6s0"},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "dhcpcd.conf")
|
||||||
|
_ = os.WriteFile(path, []byte(c.before), 0o644)
|
||||||
|
report, state := applyBlockDecl(t, blockDecl(t, path, opts, `"at":"start"`), store.State{})
|
||||||
|
if got := readText(t, path); got != c.after {
|
||||||
|
t.Fatalf("got %q, want %q", got, c.after)
|
||||||
|
}
|
||||||
|
if got := report.Outcomes[0].Action; got != "updated" {
|
||||||
|
t.Errorf("adding the region was %q", got)
|
||||||
|
}
|
||||||
|
report, state = applyBlockDecl(t, blockDecl(t, path, opts, `"at":"start"`), state)
|
||||||
|
if got := report.Outcomes[0].Action; got != "unchanged" {
|
||||||
|
t.Errorf("a second apply was %q", got)
|
||||||
|
}
|
||||||
|
undeclare(t, state)
|
||||||
|
if got := readText(t, path); got != c.before {
|
||||||
|
t.Errorf("undeclaring left %q, the machine had %q", got, c.before)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("a file that was not there", func(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "dhcpcd.conf")
|
||||||
|
applyBlockDecl(t, blockDecl(t, path, opts, `"at":"start"`), store.State{})
|
||||||
|
if got := readText(t, path); got != marked(namesID, opts) {
|
||||||
|
t.Errorf("got %q", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARegionAlreadyThereIsNotMovedWhereverAtSaysItGoes(t *testing.T) {
|
||||||
|
for _, at := range []string{`"at":"start"`, `"at":"end"`} {
|
||||||
|
path := filepath.Join(t.TempDir(), "dhcpcd.conf")
|
||||||
|
original := "hostname\n" + marked(namesID, "old\n") + "interface enp6s0\n"
|
||||||
|
_ = os.WriteFile(path, []byte(original), 0o644)
|
||||||
|
applyBlockDecl(t, blockDecl(t, path, "nohook resolv.conf\n", at), store.State{})
|
||||||
|
want := "hostname\n" + marked(namesID, "nohook resolv.conf\n") + "interface enp6s0\n"
|
||||||
|
if got := readText(t, path); got != want {
|
||||||
|
t.Errorf("%s: a found region was moved: %q", at, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFileWrittenIntoABlockIsNeverHeldOnAnAdoptedNode(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
_ = os.WriteFile(path, []byte(workstationHosts), 0o644)
|
||||||
|
resource := fmt.Sprintf(`{"id":%q,"type":"file","path":%q,"into":"block","content":%q}`, namesID, path, meshNames)
|
||||||
|
d := adopted(t, `{"taken":[],"untaken":{"mesh-wireguard":["`+namesID+`"]}}`, resource)
|
||||||
|
report, state := applyAdopted(t, d, store.State{}, &machine{}, t.TempDir())
|
||||||
|
if got := outcomeOf(report, namesID).Action; got == "held" {
|
||||||
|
t.Fatal("a file written into a block was held, though it replaces nothing that was found")
|
||||||
|
}
|
||||||
|
if len(state.Held) != 0 {
|
||||||
|
t.Errorf("something was held: %+v", state.Held)
|
||||||
|
}
|
||||||
|
if got := readText(t, path); got != workstationHosts+"\n"+marked(namesID, meshNames) {
|
||||||
|
t.Errorf("the adopted node's file was not written into: %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Held from when it was declared whole, the hold does not keep the region out.
|
||||||
|
path2 := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
_ = os.WriteFile(path2, []byte(workstationHosts), 0o644)
|
||||||
|
known := store.State{Held: []store.Held{{ID: namesID, Module: "mesh-wireguard", Kind: "file", Target: path2}}}
|
||||||
|
resource2 := fmt.Sprintf(`{"id":%q,"type":"file","path":%q,"into":"block","content":%q}`, namesID, path2, meshNames)
|
||||||
|
d2 := adopted(t, `{"taken":[],"untaken":{"mesh-wireguard":["`+namesID+`"]}}`, resource2)
|
||||||
|
report, state = applyAdopted(t, d2, known, &machine{}, t.TempDir())
|
||||||
|
if got := outcomeOf(report, namesID).Action; got != "updated" {
|
||||||
|
t.Errorf("the file was %q, not written into", got)
|
||||||
|
}
|
||||||
|
if len(state.Held) != 0 {
|
||||||
|
t.Errorf("the old hold outlived the block declaration: %+v", state.Held)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the preview says the same: written into, not held.
|
||||||
|
for _, s := range Plan(d2, known, store.OriginDeclared) {
|
||||||
|
if s.ID == namesID && s.Verb == "hold" {
|
||||||
|
t.Errorf("the preview holds a file written into a block: %+v", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheRecordOfABlockSurvivesTheStateFile(t *testing.T) {
|
||||||
|
// What undeclaring needs is in the state a host saves, not only in memory.
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
original := "a\n" + marked(namesID, "old\n")
|
||||||
|
_ = os.WriteFile(path, []byte(original), 0o644)
|
||||||
|
_, state := applyBlockDecl(t, blockDecl(t, path, meshNames, `"at":"start"`), store.State{})
|
||||||
|
raw, err := json.Marshal(state)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var back store.State
|
||||||
|
if err := json.Unmarshal(raw, &back); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
undeclare(t, back)
|
||||||
|
if got := readText(t, path); got != original {
|
||||||
|
t.Errorf("undeclaring from a saved state left %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The mesh's old whole hosts file, as the controller composed it before issue 128.
|
||||||
|
const oldWholeHosts = "# Generated by the mesh. Do not edit — this file is replaced whenever a machine\n" +
|
||||||
|
"# joins or leaves, and an edit would survive until then and vanish.\n\n" +
|
||||||
|
"127.0.0.1\tlocalhost\n" +
|
||||||
|
"::1\t\tlocalhost ip6-localhost ip6-loopback\n" +
|
||||||
|
"127.0.1.1\tg14\n" +
|
||||||
|
"\n" +
|
||||||
|
"10.42.0.1\tace.internal\tace\n" +
|
||||||
|
"10.42.0.9\tg14.internal\tg14\t# this machine\n"
|
||||||
|
|
||||||
|
func wholeDecl(path, content string) string {
|
||||||
|
return fmt.Sprintf(`{"declaration":1,"resources":[
|
||||||
|
{"id":%q,"type":"file","path":%q,"content":%q}
|
||||||
|
]}`, namesID, path, content)
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyKeepingIn(t *testing.T, raw string, known store.State, keepDir string) (Report, store.State) {
|
||||||
|
t.Helper()
|
||||||
|
report, state, err := ApplyKeeping(context.Background(), archHost(t), parse(t, raw), known,
|
||||||
|
store.OriginDeclared, (&machine{}).run, nil, nil, KeepIn(keepDir))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("apply failed: %v", err)
|
||||||
|
}
|
||||||
|
return report, state
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFileTheMeshWroteWholeAndMadeItselfKeepsOnlyItsLoopbackLines(t *testing.T) {
|
||||||
|
// Written whole into a file that was not there, then declared as a block under the same id:
|
||||||
|
// the old names must not stay above the region, where a resolver would answer from them first.
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
_, state := applyKeepingIn(t, wholeDecl(path, oldWholeHosts), store.State{}, t.TempDir())
|
||||||
|
report, state := applyKeepingIn(t, blockDecl(t, path, meshNames), state, t.TempDir())
|
||||||
|
floor := "127.0.0.1\tlocalhost\n::1\t\tlocalhost ip6-localhost ip6-loopback\n127.0.1.1\tg14\n"
|
||||||
|
if got := readText(t, path); got != floor+"\n"+marked(namesID, meshNames) {
|
||||||
|
t.Fatalf("the old whole file became:\n%q", got)
|
||||||
|
}
|
||||||
|
o := outcomeOf(report, namesID)
|
||||||
|
if o.Action != "updated" || !strings.Contains(o.Detail, "loopback lines") {
|
||||||
|
t.Errorf("the rebuild was reported as %q: %s", o.Action, o.Detail)
|
||||||
|
}
|
||||||
|
if rec, _ := state.Find(namesID); !rec.Into.Created {
|
||||||
|
t.Error("a file the mesh made itself was not recorded as the mesh's")
|
||||||
|
}
|
||||||
|
report, _ = applyKeepingIn(t, blockDecl(t, path, meshNames), state, t.TempDir())
|
||||||
|
if got := outcomeOf(report, namesID).Action; got != "unchanged" {
|
||||||
|
t.Errorf("applied again, the rebuilt file was %q", got)
|
||||||
|
}
|
||||||
|
undeclare(t, state)
|
||||||
|
if got := readText(t, path); got != floor {
|
||||||
|
t.Errorf("undeclared, the file holds %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFileTheMeshWroteWholeOverAnOriginalGetsTheOriginalBack(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
_ = os.WriteFile(path, []byte(workstationHosts), 0o644)
|
||||||
|
keep := t.TempDir()
|
||||||
|
_, state := applyKeepingIn(t, wholeDecl(path, oldWholeHosts), store.State{}, keep)
|
||||||
|
if rec, _ := state.Find(namesID); rec.Kept == "" {
|
||||||
|
t.Fatal("where the original was kept was not recorded")
|
||||||
|
}
|
||||||
|
report, state := applyKeepingIn(t, blockDecl(t, path, meshNames), state, keep)
|
||||||
|
if got := readText(t, path); got != workstationHosts+"\n"+marked(namesID, meshNames) {
|
||||||
|
t.Fatalf("the old whole file became:\n%q", got)
|
||||||
|
}
|
||||||
|
if d := outcomeOf(report, namesID).Detail; !strings.Contains(d, "original kept at") {
|
||||||
|
t.Errorf("the rebuild was reported as: %s", d)
|
||||||
|
}
|
||||||
|
undeclare(t, state)
|
||||||
|
if got := readText(t, path); got != workstationHosts {
|
||||||
|
t.Errorf("undeclared, the machine did not get its original back: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFileTheMeshWroteWholeAndSomebodyChangedIsWrittenIntoAsItStands(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
_, state := applyKeepingIn(t, wholeDecl(path, oldWholeHosts), store.State{}, t.TempDir())
|
||||||
|
edited := oldWholeHosts + "192.168.1.20 printer\n"
|
||||||
|
_ = os.WriteFile(path, []byte(edited), 0o644)
|
||||||
|
report, _ := applyKeepingIn(t, blockDecl(t, path, meshNames), state, t.TempDir())
|
||||||
|
if got := readText(t, path); got != edited+"\n"+marked(namesID, meshNames) {
|
||||||
|
t.Fatalf("an edited whole file became:\n%q", got)
|
||||||
|
}
|
||||||
|
if d := outcomeOf(report, namesID).Detail; !strings.Contains(d, "changed since; its old lines were kept") {
|
||||||
|
t.Errorf("the outcome does not say so: %s", d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestALinkedFileStaysALink(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
real := filepath.Join(dir, "static", "hosts")
|
||||||
|
_ = os.MkdirAll(filepath.Dir(real), 0o755)
|
||||||
|
_ = os.WriteFile(real, []byte(workstationHosts), 0o644)
|
||||||
|
link := filepath.Join(dir, "hosts")
|
||||||
|
if err := os.Symlink(real, link); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, state := applyBlockDecl(t, blockDecl(t, link, meshNames), store.State{})
|
||||||
|
if info, err := os.Lstat(link); err != nil || info.Mode()&os.ModeSymlink == 0 {
|
||||||
|
t.Fatalf("the link was replaced by a file")
|
||||||
|
}
|
||||||
|
if got := readText(t, real); got != workstationHosts+"\n"+marked(namesID, meshNames) {
|
||||||
|
t.Errorf("the file the link names holds %q", got)
|
||||||
|
}
|
||||||
|
undeclare(t, state)
|
||||||
|
if info, err := os.Lstat(link); err != nil || info.Mode()&os.ModeSymlink == 0 {
|
||||||
|
t.Fatalf("undeclaring replaced the link with a file")
|
||||||
|
}
|
||||||
|
if got := readText(t, real); got != workstationHosts {
|
||||||
|
t.Errorf("undeclared, the file the link names holds %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestALastLineWithNoEndIsGivenBackWithNone(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
_ = os.WriteFile(path, []byte("x"), 0o644)
|
||||||
|
_, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{})
|
||||||
|
if got := readText(t, path); got != "x\n\n"+marked(namesID, meshNames) {
|
||||||
|
t.Fatalf("got %q", got)
|
||||||
|
}
|
||||||
|
undeclare(t, state)
|
||||||
|
if got := readText(t, path); got != "x" {
|
||||||
|
t.Errorf("undeclaring left %q, the machine had %q", got, "x")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFailedBlockWriteKeepsItsHold(t *testing.T) {
|
||||||
|
// Held from when it was declared whole, then declared as a block into a file whose markers do
|
||||||
|
// not pair: the write is refused, and the hold — with where its original is — stays.
|
||||||
|
path := filepath.Join(t.TempDir(), "hosts")
|
||||||
|
broken := "a\n# BEGIN mesh " + namesID + "\n"
|
||||||
|
_ = os.WriteFile(path, []byte(broken), 0o644)
|
||||||
|
known := store.State{Held: []store.Held{{ID: namesID, Module: "mesh-wireguard", Kind: "file",
|
||||||
|
Target: path, Kept: "/var/lib/mesh/kept/hosts"}}}
|
||||||
|
resource := fmt.Sprintf(`{"id":%q,"type":"file","path":%q,"into":"block","content":%q}`, namesID, path, meshNames)
|
||||||
|
d := adopted(t, `{"taken":[],"untaken":{"mesh-wireguard":["`+namesID+`"]}}`, resource)
|
||||||
|
_, state, err := ApplyKeeping(context.Background(), archHost(t), d, known,
|
||||||
|
store.OriginDeclared, (&machine{}).run, nil, nil, KeepIn(t.TempDir()))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a write into unpaired markers was not refused")
|
||||||
|
}
|
||||||
|
h, held := state.HeldAt(namesID)
|
||||||
|
if !held || h.Kept != "/var/lib/mesh/kept/hosts" {
|
||||||
|
t.Errorf("a failed write released the hold: %+v", state.Held)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,387 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq ADR 0118: undeclaring removes what the mesh made, gives back what it changed,
|
||||||
|
// and leaves what was the machine's — which needs what was found kept exactly, and a unit the mesh
|
||||||
|
// made known for the mesh's whatever its record says.
|
||||||
|
|
||||||
|
func unitsMachine(units map[string]*fakeUnit) *machine {
|
||||||
|
return &machine{containers: map[string]*fakeContainer{}, units: units}
|
||||||
|
}
|
||||||
|
|
||||||
|
// nothingButA is a declaration of one unrelated file, so everything recorded is undeclared.
|
||||||
|
func nothingButA(t *testing.T) string {
|
||||||
|
return `{"declaration":1,"resources":[
|
||||||
|
{"id":"other","type":"file","path":"` + filepath.Join(t.TempDir(), "a") + `","content":"a\n"}]}`
|
||||||
|
}
|
||||||
|
|
||||||
|
// copyOf is a state as a later load of it would be: sharing nothing with the one it came from.
|
||||||
|
func copyOf(t *testing.T, s store.State) store.State {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(s)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out store.State
|
||||||
|
if err := json.Unmarshal(raw, &out); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyOn(t *testing.T, raw string, known store.State, m *machine) (Report, store.State, error) {
|
||||||
|
t.Helper()
|
||||||
|
return Apply(context.Background(), archHost(t), parse(t, raw), known, store.OriginDeclared, m.run, nil, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAUnitWhoseFileTheMeshWroteIsStoppedWhateverItsRecordSays(t *testing.T) {
|
||||||
|
// The adoption guard's unit file is the mesh's own file resource, written where there was none.
|
||||||
|
// Its service recorded before the host kept what it found has no Found, and forgetting it left
|
||||||
|
// the guard's table loaded on a converged node and its unit file deleted from under it.
|
||||||
|
units := t.TempDir()
|
||||||
|
was := unitDir
|
||||||
|
unitDir = units
|
||||||
|
t.Cleanup(func() { unitDir = was })
|
||||||
|
unitFile := filepath.Join(units, "mesh-guard.service")
|
||||||
|
if err := os.WriteFile(unitFile, []byte("[Unit]\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := unitsMachine(map[string]*fakeUnit{"mesh-guard.service": {active: "active", enabled: "enabled"}})
|
||||||
|
fileThereAtStop := false
|
||||||
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||||
|
if name == "systemctl" && args[0] == "stop" {
|
||||||
|
_, err := os.Stat(unitFile)
|
||||||
|
fileThereAtStop = err == nil
|
||||||
|
}
|
||||||
|
return m.run(ctx, name, args...)
|
||||||
|
}
|
||||||
|
// As a host before this change recorded them: the unit file first, then the service it
|
||||||
|
// starts, and no Found on the service.
|
||||||
|
known := store.State{Resources: []store.Applied{
|
||||||
|
{ID: "adoption.guard-unit", Type: "file", Target: unitFile, Origin: store.OriginDeclared},
|
||||||
|
{ID: "adoption.guard-running", Type: "service", Target: "mesh-guard.service", Origin: store.OriginDeclared},
|
||||||
|
}}
|
||||||
|
report, after, err := applyWith(t, parse(t, nothingButA(t)), known, run)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if u := m.units["mesh-guard.service"]; u.active != "inactive" || u.enabled != "disabled" {
|
||||||
|
t.Errorf("the mesh's own unit was left %s and %s: %v", u.active, u.enabled, m.asked)
|
||||||
|
}
|
||||||
|
if !fileThereAtStop {
|
||||||
|
t.Error("the unit was stopped after its file was deleted, or not at all")
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, "adoption.guard-running"); o.Action != "removed" || !strings.Contains(o.Detail, "unit file") {
|
||||||
|
t.Errorf("outcome %+v", o)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(unitFile); !os.IsNotExist(err) {
|
||||||
|
t.Error("the unit file outlived its record")
|
||||||
|
}
|
||||||
|
if len(after.Resources) != 1 {
|
||||||
|
t.Errorf("still recorded: %v", after.IDs())
|
||||||
|
}
|
||||||
|
|
||||||
|
// A unit file the host wrote OVER — its original kept — is the machine's unit, and a record
|
||||||
|
// with no Found leaves it as it is.
|
||||||
|
if err := os.WriteFile(unitFile, []byte("[Unit]\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m = unitsMachine(map[string]*fakeUnit{"mesh-guard.service": {active: "active", enabled: "enabled"}})
|
||||||
|
known.Resources[0].Kept = filepath.Join(t.TempDir(), "original")
|
||||||
|
if err := os.WriteFile(known.Resources[0].Kept, []byte("[Unit]\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, _, err := applyWith(t, parse(t, nothingButA(t)), known, m.run); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.did("systemctl stop") || m.did("systemctl disable") {
|
||||||
|
t.Errorf("a unit whose file the mesh only wrote over was stopped: %v", m.asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOnlyAUnitsOwnFileTheMeshCreatedMakesItTheMeshs(t *testing.T) {
|
||||||
|
known := store.State{Resources: []store.Applied{
|
||||||
|
{ID: "a", Type: "file", Target: "/etc/systemd/system/made.service"},
|
||||||
|
{ID: "b", Type: "file", Target: "/run/systemd/system/runtime.service"},
|
||||||
|
{ID: "c", Type: "file", Target: "/etc/systemd/system/kept.service", Kept: "/var/lib/mesh-host/kept/x"},
|
||||||
|
{ID: "d", Type: "file", Target: "/etc/systemd/system/into.service", Into: &store.Into{Format: "block"}},
|
||||||
|
{ID: "e", Type: "file", Target: "/etc/systemd/system/docker.service.d/mesh.conf"},
|
||||||
|
{ID: "f", Type: "file", Target: "/etc/mesh/elsewhere.service"},
|
||||||
|
{ID: "g", Type: "directory", Target: "/etc/systemd/system/dir.service"},
|
||||||
|
}}
|
||||||
|
made := meshMadeUnits(known)
|
||||||
|
for unit, want := range map[string]bool{"made.service": true, "runtime.service": true, "kept.service": false,
|
||||||
|
"into.service": false, "mesh.conf": false, "docker.service.d": false, "elsewhere.service": false, "dir.service": false} {
|
||||||
|
if made[unit] != want {
|
||||||
|
t.Errorf("%s: made %v, want %v", unit, made[unit], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWhatWasFoundOutlivesAFirstApplyThatFailed(t *testing.T) {
|
||||||
|
// Enabled, then it would not start: no record. The next apply must not read the enable as
|
||||||
|
// the machine's — or undeclaring leaves enabled a unit the mesh enabled.
|
||||||
|
m := unitsMachine(map[string]*fakeUnit{"filter.service": {active: "inactive", enabled: "disabled", wontStart: true}})
|
||||||
|
declared := `{"declaration":1,"resources":[
|
||||||
|
{"id":"s","type":"service","unit":"filter.service","state":"running","boot":"enabled"}]}`
|
||||||
|
_, first, err := applyOn(t, declared, store.State{}, m)
|
||||||
|
if err == nil || !m.did("systemctl enable filter.service") {
|
||||||
|
t.Fatalf("the fixture did not enable and then fail: %v, %v", err, m.asked)
|
||||||
|
}
|
||||||
|
if _, ok := first.Find("s"); ok {
|
||||||
|
t.Fatal("a failed apply was recorded")
|
||||||
|
}
|
||||||
|
if p := first.FoundFirst["s"]; p.State != "stopped" || p.Boot != "disabled" || p.Unit != "filter.service" {
|
||||||
|
t.Fatalf("what was found was not kept through the failure: %+v", first.FoundFirst)
|
||||||
|
}
|
||||||
|
|
||||||
|
failed := copyOf(t, first)
|
||||||
|
|
||||||
|
m.units["filter.service"].wontStart = false
|
||||||
|
_, second, err := applyOn(t, declared, first, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rec, _ := second.Find("s"); rec.Found == nil || rec.Found.State != "stopped" || rec.Found.Boot != "disabled" {
|
||||||
|
t.Errorf("the record carries the mesh's own effect as found: %+v", rec.Found)
|
||||||
|
}
|
||||||
|
if second.FoundFirst != nil {
|
||||||
|
t.Errorf("kept apart after the record carried it: %+v", second.FoundFirst)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, _, err := applyOn(t, nothingButA(t), second, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if u := m.units["filter.service"]; u.active != "inactive" || u.enabled != "disabled" {
|
||||||
|
t.Errorf("undeclared, the unit was left %s and %s", u.active, u.enabled)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Undeclared before it was ever recorded, what was found goes with it — only for its origin.
|
||||||
|
if _, kept, _ := Apply(context.Background(), archHost(t), parse(t, nothingButA(t)), copyOf(t, failed),
|
||||||
|
store.OriginCarried, m.run, nil, nil); kept.FoundFirst["s"].State == "" {
|
||||||
|
t.Error("a carried apply dropped what the mesh's declaration found")
|
||||||
|
}
|
||||||
|
if _, dropped, err := applyOn(t, nothingButA(t), copyOf(t, failed), m); err != nil || dropped.FoundFirst != nil {
|
||||||
|
t.Errorf("kept for a service no longer declared: %+v, %v", dropped.FoundFirst, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBootIsFoundTheFirstTimeTheMeshSetsIt(t *testing.T) {
|
||||||
|
// The declaration said nothing about boot at first, so the mesh never touched it: what is
|
||||||
|
// there when a declaration first does is still the machine's.
|
||||||
|
m := unitsMachine(map[string]*fakeUnit{"web.service": {active: "active", enabled: "disabled"}})
|
||||||
|
known := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "web.service",
|
||||||
|
Origin: store.OriginDeclared, Found: &store.FoundUnit{Unit: "web.service", State: "running"}}}}
|
||||||
|
_, after, err := applyOn(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"s","type":"service","unit":"web.service","state":"running","boot":"enabled"}]}`, known, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec, _ := after.Find("s")
|
||||||
|
if rec.Found == nil || rec.Found.State != "running" || rec.Found.Boot != "disabled" {
|
||||||
|
t.Fatalf("found %+v, want running and disabled", rec.Found)
|
||||||
|
}
|
||||||
|
report, _, err := applyOn(t, nothingButA(t), after, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if u := m.units["web.service"]; u.active != "active" || u.enabled != "disabled" {
|
||||||
|
t.Errorf("undeclared, the unit is %s and %s; want running, and disabled again", u.active, u.enabled)
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, "s"); o.Action != "restored" || o.Detail != "disabled at boot, as the host found it" {
|
||||||
|
t.Errorf("outcome %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAServiceOnceDeclaredWithNoStateIsFoundWhenFirstGivenOne(t *testing.T) {
|
||||||
|
// Declared with no state (novox/hq ADR 0117), the mesh never started or stopped it — so what
|
||||||
|
// is there when a declaration first gives it one is what the machine had.
|
||||||
|
m := unitsMachine(map[string]*fakeUnit{"net.service": {active: "inactive", enabled: "disabled"}})
|
||||||
|
known := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "net.service",
|
||||||
|
Origin: store.OriginDeclared, Stateless: true}}}
|
||||||
|
_, after, err := applyOn(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"s","type":"service","unit":"net.service","state":"running"}]}`, known, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rec, _ := after.Find("s"); rec.Found == nil || rec.Found.State != "stopped" {
|
||||||
|
t.Fatalf("found %+v, want stopped", rec.Found)
|
||||||
|
}
|
||||||
|
if _, _, err := applyOn(t, nothingButA(t), after, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.units["net.service"].active != "inactive" {
|
||||||
|
t.Error("the unit the mesh started outlived its declaration")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAUnitWrittenInTheSameApplyIsLoadedBeforeItIsRead(t *testing.T) {
|
||||||
|
// Read before the service manager is told about its new file, the unit is not there to find.
|
||||||
|
dir := t.TempDir()
|
||||||
|
m := unitsMachine(map[string]*fakeUnit{"fresh.service": {active: "inactive", enabled: "disabled"}})
|
||||||
|
_, _, err := applyOn(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"unit","type":"file","path":"`+filepath.Join(dir, "fresh.service")+`","content":"[Unit]\n"},
|
||||||
|
{"id":"s","type":"service","unit":"fresh.service","state":"running","restart-on":["unit"]}]}`,
|
||||||
|
store.State{}, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reload, show := -1, -1
|
||||||
|
for i, a := range m.asked {
|
||||||
|
if a == "systemctl daemon-reload" && reload < 0 {
|
||||||
|
reload = i
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(a, "systemctl show fresh.service") && show < 0 {
|
||||||
|
show = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if reload < 0 || show < 0 || reload > show {
|
||||||
|
t.Errorf("the unit was read before its file was loaded: %v", m.asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAServiceMovedToAnotherUnitGivesTheOldOneBackAndFindsTheNewOne(t *testing.T) {
|
||||||
|
m := unitsMachine(map[string]*fakeUnit{
|
||||||
|
"old.service": {active: "active", enabled: "enabled"},
|
||||||
|
"new.service": {active: "inactive", enabled: "disabled"},
|
||||||
|
})
|
||||||
|
known := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "old.service",
|
||||||
|
Origin: store.OriginDeclared, Found: &store.FoundUnit{Unit: "old.service", State: "stopped"}}}}
|
||||||
|
report, after, err := applyOn(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"s","type":"service","unit":"new.service","state":"running"}]}`, known, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.units["old.service"].active != "inactive" {
|
||||||
|
t.Error("the unit the mesh started is still running though nothing declares it")
|
||||||
|
}
|
||||||
|
if m.units["new.service"].active != "active" {
|
||||||
|
t.Error("the unit now declared was not started")
|
||||||
|
}
|
||||||
|
rec, _ := after.Find("s")
|
||||||
|
if rec.Found == nil || rec.Found.Unit != "new.service" || rec.Found.State != "stopped" {
|
||||||
|
t.Errorf("what was found about the old unit was carried to the new one: %+v", rec.Found)
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, "s"); !strings.Contains(o.Detail, "old.service stopped, as the host found it") {
|
||||||
|
t.Errorf("giving the old unit back went unsaid: %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARemovalSaysWhatItDid(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
found *store.FoundUnit
|
||||||
|
unit *fakeUnit
|
||||||
|
action, detail string
|
||||||
|
}{
|
||||||
|
{"found stopped and still stopped", &store.FoundUnit{State: "stopped"},
|
||||||
|
&fakeUnit{active: "inactive", enabled: "disabled"}, "forgotten", "already as the host found it (stopped)"},
|
||||||
|
{"started by the mesh", &store.FoundUnit{State: "stopped"},
|
||||||
|
&fakeUnit{active: "active", enabled: "disabled"}, "restored", "stopped, as the host found it"},
|
||||||
|
{"started and enabled by the mesh", &store.FoundUnit{State: "stopped", Boot: "disabled"},
|
||||||
|
&fakeUnit{active: "active", enabled: "enabled"}, "restored", "stopped, disabled at boot, as the host found it"},
|
||||||
|
{"found running, stopped by the mesh", &store.FoundUnit{State: "running"},
|
||||||
|
&fakeUnit{active: "inactive", enabled: "disabled"}, "forgotten",
|
||||||
|
"left as it is; the mesh stopped it and does not start anything on the way out"},
|
||||||
|
{"found running and enabled, disabled by the mesh", &store.FoundUnit{State: "running", Boot: "enabled"},
|
||||||
|
&fakeUnit{active: "active", enabled: "disabled"}, "forgotten",
|
||||||
|
"left as it is; the mesh disabled it at boot and does not start anything on the way out"},
|
||||||
|
{"found running, still running", &store.FoundUnit{State: "running"},
|
||||||
|
&fakeUnit{active: "active", enabled: "enabled"}, "forgotten", "it was running before the mesh; left as it is"},
|
||||||
|
// Found says to stop it, so the machine is asked about it — and it is gone.
|
||||||
|
{"started by the mesh, since uninstalled", &store.FoundUnit{State: "stopped"},
|
||||||
|
nil, "forgotten", "the unit no longer exists"},
|
||||||
|
{"recorded before the host kept what it found", nil,
|
||||||
|
&fakeUnit{active: "active", enabled: "enabled"}, "forgotten", "recorded before the host kept what it found; left as it is"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
units := map[string]*fakeUnit{}
|
||||||
|
if c.unit != nil {
|
||||||
|
units["unit.service"] = c.unit
|
||||||
|
}
|
||||||
|
m := unitsMachine(units)
|
||||||
|
known := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "unit.service",
|
||||||
|
Origin: store.OriginDeclared, Found: c.found}}}
|
||||||
|
report, after, err := applyOn(t, nothingButA(t), known, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, "s"); o.Action != c.action || o.Detail != c.detail {
|
||||||
|
t.Errorf("said %s: %s; want %s: %s", o.Action, o.Detail, c.action, c.detail)
|
||||||
|
}
|
||||||
|
if c.unit == nil && !m.did("systemctl show unit.service") {
|
||||||
|
t.Errorf("the machine was never asked whether the unit is there: %v", m.asked)
|
||||||
|
}
|
||||||
|
if m.did("systemctl start") || m.did("systemctl enable") {
|
||||||
|
t.Errorf("something was started on the way out: %v", m.asked)
|
||||||
|
}
|
||||||
|
if _, still := after.Find("s"); still {
|
||||||
|
t.Error("still recorded")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAUnitTheMeshStartedIsStoppedWhenUndeclaredAndOneFoundRunningIsNot(t *testing.T) {
|
||||||
|
// End to end: found by the first apply, carried, given back.
|
||||||
|
m := unitsMachine(map[string]*fakeUnit{
|
||||||
|
"filter.service": {active: "inactive", enabled: "disabled"},
|
||||||
|
"runtime.service": {active: "active", enabled: "enabled"},
|
||||||
|
})
|
||||||
|
_, state, err := applyOn(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"filter","type":"service","unit":"filter.service","state":"running","boot":"enabled"},
|
||||||
|
{"id":"runtime","type":"service","unit":"runtime.service","state":"running","boot":"enabled"}]}`,
|
||||||
|
store.State{}, m)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.units["filter.service"].active != "active" {
|
||||||
|
t.Fatal("the fixture did not start the filter")
|
||||||
|
}
|
||||||
|
if _, _, err := applyOn(t, nothingButA(t), state, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if u := m.units["filter.service"]; u.active != "inactive" || u.enabled != "disabled" {
|
||||||
|
t.Errorf("the filter the mesh started and enabled is %s and %s", u.active, u.enabled)
|
||||||
|
}
|
||||||
|
if u := m.units["runtime.service"]; u.active != "active" || u.enabled != "enabled" {
|
||||||
|
t.Errorf("the runtime that was running before the mesh is %s and %s", u.active, u.enabled)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAUnitHeldAndThenTakenIsFoundAsThePredecessorLeftIt(t *testing.T) {
|
||||||
|
// Held while its module was untaken, nothing was applied and nothing found; taken, the first
|
||||||
|
// apply finds the predecessor's unit — stopped, but started at boot — before starting it.
|
||||||
|
dir := t.TempDir()
|
||||||
|
m := unitsMachine(map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "enabled"}})
|
||||||
|
service := `{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled"}`
|
||||||
|
_, held := applyAdopted(t, adopted(t, untaken("hello-web.unit"), service), store.State{}, m, dir)
|
||||||
|
if _, ok := held.HeldAt("hello-web.unit"); !ok {
|
||||||
|
t.Fatal("the fixture's unit was not held")
|
||||||
|
}
|
||||||
|
_, taken := applyAdopted(t, adopted(t, `{"taken":["hello-web"]}`, service), held, m, dir)
|
||||||
|
rec, _ := taken.Find("hello-web.unit")
|
||||||
|
if rec.Found == nil || rec.Found.State != "stopped" || rec.Found.Boot != "enabled" {
|
||||||
|
t.Fatalf("found %+v, want the predecessor's stopped and enabled", rec.Found)
|
||||||
|
}
|
||||||
|
if m.units["hello.service"].active != "active" {
|
||||||
|
t.Fatal("the taken unit was not started")
|
||||||
|
}
|
||||||
|
if _, _, err := applyOn(t, nothingButA(t), taken, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if u := m.units["hello.service"]; u.active != "inactive" || u.enabled != "enabled" {
|
||||||
|
t.Errorf("given back as %s and %s; the predecessor left it stopped and enabled", u.active, u.enabled)
|
||||||
|
}
|
||||||
|
}
|
||||||
+29
-7
@@ -108,7 +108,7 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
case *declaration.Service:
|
case *declaration.Service:
|
||||||
if known.Recorded(string(declaration.TypeService), res.Unit) {
|
if res.Stateless() || known.Recorded(string(declaration.TypeService), res.Unit) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// **Found is a unit somebody put on this machine, or one the machine uses.**
|
// **Found is a unit somebody put on this machine, or one the machine uses.**
|
||||||
@@ -261,6 +261,19 @@ func heldContainer(known store.State, name string) (store.Held, bool) {
|
|||||||
return store.Held{}, false
|
return store.Held{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// replacesNothing is a resource that takes nothing found on the machine from it, so on an adopted
|
||||||
|
// node it is never held and never previewed as replacing what was found: a file written into
|
||||||
|
// (novox/hq ADR 0102), and a service whose unit's lifecycle is the machine's (novox/hq ADR 0117).
|
||||||
|
func replacesNothing(r declaration.Resource) bool {
|
||||||
|
switch res := r.(type) {
|
||||||
|
case *declaration.File:
|
||||||
|
return res.Into != ""
|
||||||
|
case *declaration.Service:
|
||||||
|
return res.Stateless()
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// holdOnAdopted decides whether a resource of an adopted node is held rather than applied, and
|
// holdOnAdopted decides whether a resource of an adopted node is held rather than applied, and
|
||||||
// holds it (novox/hq ADR 0100, ADR 0103). For a module not yet taken, what is present with no
|
// holds it (novox/hq ADR 0100, ADR 0103). For a module not yet taken, what is present with no
|
||||||
// record is kept as it is: a file or a container under its name, a directory, a service's unit,
|
// record is kept as it is: a file or a container under its name, a directory, a service's unit,
|
||||||
@@ -295,11 +308,11 @@ func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resourc
|
|||||||
}
|
}
|
||||||
|
|
||||||
// A file written into replaces nothing that was found, so it is never held (novox/hq ADR
|
// A file written into replaces nothing that was found, so it is never held (novox/hq ADR
|
||||||
// 0102) — and a hold from when it was declared whole must not keep the mesh's keys out.
|
// 0102) — and a hold from when it was declared whole must not keep the mesh's keys out. That
|
||||||
if f, ok := r.(*declaration.File); ok && f.Into != "" {
|
// hold is released by the apply once the write has worked, not here: a write that fails keeps
|
||||||
if already {
|
// it, and with it where the original was kept. A service whose lifecycle is the machine's
|
||||||
known.Release(r.Identity())
|
// replaces nothing either (novox/hq ADR 0117).
|
||||||
}
|
if replacesNothing(r) {
|
||||||
return false, false, out, nil
|
return false, false, out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -424,12 +437,21 @@ type foundContainer struct {
|
|||||||
|
|
||||||
// inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and
|
// inspectFound reads a container by name the way a hold needs it: its id, whether it runs, and
|
||||||
// whether a host made it.
|
// whether a host made it.
|
||||||
|
//
|
||||||
|
// **`container inspect`, not the bare form.** A name is not unique across object kinds — a
|
||||||
|
// module regularly names a network the same as the container that joins it (`keycloak` names
|
||||||
|
// both, and it is ordinary). The bare form resolves across every kind and returns whichever it
|
||||||
|
// finds, so a container that does not exist yet but a same-named network does answers with the
|
||||||
|
// network's JSON — no `.State` field at all — and the template below fails to execute rather
|
||||||
|
// than failing to find anything. That reads as "the runtime could not say", which this function's
|
||||||
|
// caller correctly refuses to build on (novox/hq ADR 0100) — but there was something to say, a
|
||||||
|
// question of kind, not of ambiguity that should have stopped anything.
|
||||||
func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, bool, error) {
|
func inspectFound(ctx context.Context, name string, run Runner) (foundContainer, bool, error) {
|
||||||
cri, err := containerRuntime(ctx, run)
|
cri, err := containerRuntime(ctx, run)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name)
|
return foundContainer{}, false, fmt.Errorf("%w, so nothing can be said about %q", err, name)
|
||||||
}
|
}
|
||||||
out, err := run(ctx, cri, "inspect", "--format",
|
out, err := run(ctx, cri, "container", "inspect", "--format",
|
||||||
"{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
|
"{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if absent(err) {
|
if absent(err) {
|
||||||
|
|||||||
@@ -19,6 +19,12 @@ import (
|
|||||||
type machine struct {
|
type machine struct {
|
||||||
containers map[string]*fakeContainer
|
containers map[string]*fakeContainer
|
||||||
asked []string
|
asked []string
|
||||||
|
// wgUp is what `wg show interfaces` answers: the tunnels up on the machine.
|
||||||
|
wgUp string
|
||||||
|
// handshakes is what `wg show <interface> latest-handshakes` answers, and handshakesFail the
|
||||||
|
// error it fails with instead — a machine with no `wg`, say (novox/hq ADR 0119).
|
||||||
|
handshakes string
|
||||||
|
handshakesFail error
|
||||||
|
|
||||||
// units are service units by name, as systemd would report them; volumes are the runtime's
|
// units are service units by name, as systemd would report them; volumes are the runtime's
|
||||||
// named volumes.
|
// named volumes.
|
||||||
@@ -29,6 +35,8 @@ type machine struct {
|
|||||||
|
|
||||||
type fakeUnit struct {
|
type fakeUnit struct {
|
||||||
active, enabled string
|
active, enabled string
|
||||||
|
// wontStart is a unit that accepts `start` and stays inactive — one that starts and dies.
|
||||||
|
wontStart bool
|
||||||
// fragment is where systemd loads the unit from; empty means /etc/systemd/system, where an
|
// fragment is where systemd loads the unit from; empty means /etc/systemd/system, where an
|
||||||
// administrator installs one.
|
// administrator installs one.
|
||||||
fragment string
|
fragment string
|
||||||
@@ -63,7 +71,9 @@ func (m *machine) systemctl(args []string) (string, error) {
|
|||||||
}
|
}
|
||||||
return u.enabled + "\n", nil
|
return u.enabled + "\n", nil
|
||||||
case "start":
|
case "start":
|
||||||
u.active = "active"
|
if !u.wontStart {
|
||||||
|
u.active = "active"
|
||||||
|
}
|
||||||
case "stop":
|
case "stop":
|
||||||
u.active = "inactive"
|
u.active = "inactive"
|
||||||
case "enable":
|
case "enable":
|
||||||
@@ -94,6 +104,12 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
|
|||||||
if name == "systemctl" {
|
if name == "systemctl" {
|
||||||
return m.systemctl(args)
|
return m.systemctl(args)
|
||||||
}
|
}
|
||||||
|
if name == "wg" {
|
||||||
|
if len(args) > 0 && args[len(args)-1] == "latest-handshakes" {
|
||||||
|
return m.handshakes, m.handshakesFail
|
||||||
|
}
|
||||||
|
return m.wgUp, nil
|
||||||
|
}
|
||||||
if name == "getent" {
|
if name == "getent" {
|
||||||
if m.users[args[len(args)-1]] {
|
if m.users[args[len(args)-1]] {
|
||||||
return args[len(args)-1] + ":x:1500:1500::/home/" + args[len(args)-1] + ":/bin/bash\n", nil
|
return args[len(args)-1] + ":x:1500:1500::/home/" + args[len(args)-1] + ":/bin/bash\n", nil
|
||||||
@@ -111,7 +127,10 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
|
|||||||
return "", errors.New("no such volume")
|
return "", errors.New("no such volume")
|
||||||
case "info":
|
case "info":
|
||||||
return "27.0\n", nil
|
return "27.0\n", nil
|
||||||
case "inspect":
|
case "container":
|
||||||
|
if args[1] != "inspect" {
|
||||||
|
return "", errors.New("unexpected docker container command")
|
||||||
|
}
|
||||||
c, ok := m.containers[args[len(args)-1]]
|
c, ok := m.containers[args[len(args)-1]]
|
||||||
if !ok {
|
if !ok {
|
||||||
return "", errors.New("no such container")
|
return "", errors.New("no such container")
|
||||||
@@ -120,7 +139,7 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
|
|||||||
if c.running {
|
if c.running {
|
||||||
running = "true"
|
running = "true"
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(args[2], "{{.Id}}") {
|
if strings.HasPrefix(args[3], "{{.Id}}") {
|
||||||
return c.id + "\t" + running + "\t" + c.spec + "\n", nil
|
return c.id + "\t" + running + "\t" + c.spec + "\n", nil
|
||||||
}
|
}
|
||||||
return running + "\t" + c.spec + "\n", nil
|
return running + "\t" + c.spec + "\n", nil
|
||||||
@@ -907,7 +926,7 @@ func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) {
|
|||||||
return "27.0\n", nil
|
return "27.0\n", nil
|
||||||
case name == "docker" && args[0] == "volume":
|
case name == "docker" && args[0] == "volume":
|
||||||
return "", errors.New("docker exited 1: Cannot connect to the Docker daemon")
|
return "", errors.New("docker exited 1: Cannot connect to the Docker daemon")
|
||||||
case name == "docker" && args[0] == "inspect":
|
case name == "docker" && args[0] == "container":
|
||||||
return "", errors.New("Error: No such object: hello-web")
|
return "", errors.New("Error: No such object: hello-web")
|
||||||
case name == "docker":
|
case name == "docker":
|
||||||
return "", errors.New("docker run must not happen")
|
return "", errors.New("docker run must not happen")
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A name is not unique across object kinds: a module regularly names a network the same as the
|
||||||
|
// container that joins it (keycloak does this today, ordinarily). `docker inspect <name>`, unlike
|
||||||
|
// `docker container inspect <name>`, resolves across every kind — so when the container does not
|
||||||
|
// exist yet but a same-named network does, the bare form answers with the network's JSON instead
|
||||||
|
// of reporting the container absent. containerState and inspectFound must ask by kind, or a
|
||||||
|
// same-named network makes them unable to tell "not here yet" from "the runtime is broken"
|
||||||
|
// (novox/hq ADR 0100's refusal, tripped by nothing wrong).
|
||||||
|
//
|
||||||
|
// dockerLikeByKind is Docker's real behaviour, not the bug: `container inspect` only ever
|
||||||
|
// answers from the container namespace. A fake that also answered the bare, unscoped form would
|
||||||
|
// not catch a regression back to it — this one refuses to, on purpose.
|
||||||
|
func dockerLikeByKind(containers map[string]bool) func(context.Context, string, ...string) (string, error) {
|
||||||
|
return func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
if name != "docker" {
|
||||||
|
return "", errors.New("unexpected program: " + name)
|
||||||
|
}
|
||||||
|
if len(args) > 0 && args[0] == "info" {
|
||||||
|
// containerRuntime's probe, answered so inspectFound gets past it to the check under
|
||||||
|
// test.
|
||||||
|
return "27.0\n", nil
|
||||||
|
}
|
||||||
|
if len(args) < 2 || args[0] != "container" || args[1] != "inspect" {
|
||||||
|
return "", errors.New("unexpected command: only `docker container inspect` is modelled here")
|
||||||
|
}
|
||||||
|
target := args[len(args)-1]
|
||||||
|
if !containers[target] {
|
||||||
|
return "", errors.New("Error: No such container: " + target)
|
||||||
|
}
|
||||||
|
return "false\t\n", nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContainerStateAsksTheContainerNamespaceNotTheBareForm(t *testing.T) {
|
||||||
|
// "minio" exists only as a network in this scenario — never in `containers` — matching the
|
||||||
|
// live failure this guards: a module's network and its container share a name, and the
|
||||||
|
// container does not exist yet.
|
||||||
|
run := dockerLikeByKind(map[string]bool{"keycloak": true})
|
||||||
|
|
||||||
|
if _, err := containerState(context.Background(), "minio", run); err == nil {
|
||||||
|
t.Fatal("a container that does not exist should report absent, not be mistaken for found")
|
||||||
|
}
|
||||||
|
if state, err := containerState(context.Background(), "keycloak", run); err != nil {
|
||||||
|
t.Fatalf("a container that does exist should be found: %v", err)
|
||||||
|
} else if state.Running {
|
||||||
|
t.Errorf("the fake said not running; containerState disagreed: %+v", state)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectFoundAsksTheContainerNamespaceNotTheBareForm(t *testing.T) {
|
||||||
|
run := dockerLikeByKind(map[string]bool{"keycloak": true})
|
||||||
|
|
||||||
|
if _, exists, err := inspectFound(context.Background(), "minio", run); err != nil || exists {
|
||||||
|
t.Fatalf("a container that does not exist should be reported absent cleanly, not refused: exists=%v err=%v", exists, err)
|
||||||
|
}
|
||||||
|
if _, exists, err := inspectFound(context.Background(), "keycloak", run); err != nil || !exists {
|
||||||
|
t.Fatalf("a container that does exist should be found: exists=%v err=%v", exists, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A node is adopted or converged, and a declaration says which it is for (novox/hq ADR 0100).
|
||||||
|
//
|
||||||
|
// **The two are not interchangeable, and the host knows which it is.** A converged declaration
|
||||||
|
// carries the mesh's drop-by-default filter and retires the firewall the node was found with; on
|
||||||
|
// an adopted node that closes the machine to everything the predecessor still serves — which is
|
||||||
|
// what happened when an operator ran `reconcile` on an adopted control-node and it applied the
|
||||||
|
// converged genesis bundle (novox/hq issue 104). The host reported "adopted" in every report and
|
||||||
|
// compared nothing. Now it compares, at the point of application, whichever command delivered
|
||||||
|
// the declaration.
|
||||||
|
//
|
||||||
|
// Only the mesh changes a node's mode, and it does so by sending a declaration: the flip arrives
|
||||||
|
// over the link as the first converged declaration after adopted ones (`converge` on the
|
||||||
|
// controller), and the way back as the first adopted one (`adopt`). So a declaration the link
|
||||||
|
// delivers, signed and verified, is the mode's authority and is not checked against the record —
|
||||||
|
// it becomes the record. Everything else — the carried bundle, a file, the kept declaration a
|
||||||
|
// disconnected node re-applies — is held to the mode already recorded.
|
||||||
|
|
||||||
|
// ModeOf says which mode a declaration is for.
|
||||||
|
func ModeOf(d *declaration.Declaration) string {
|
||||||
|
if d.Adoption != nil {
|
||||||
|
return store.ModeAdopted
|
||||||
|
}
|
||||||
|
return store.ModeConverged
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckMode refuses a declaration whose mode is not the one this node has recorded. A node with
|
||||||
|
// no recorded mode — a machine nothing has said a mode to yet — takes either.
|
||||||
|
func CheckMode(known store.State, d *declaration.Declaration) error {
|
||||||
|
if known.Mode == "" || known.Mode == ModeOf(d) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
act := "`converge`"
|
||||||
|
if ModeOf(d) == store.ModeAdopted {
|
||||||
|
act = "`adopt`"
|
||||||
|
}
|
||||||
|
return fmt.Errorf("this node is %s; the declaration says %s — %s declaration is not applied "+
|
||||||
|
"to %s node; %s on the controller is the act that changes it, and it sends the "+
|
||||||
|
"declaration that does", known.Mode, ModeOf(d), article(ModeOf(d)), article(known.Mode), act)
|
||||||
|
}
|
||||||
|
|
||||||
|
func article(mode string) string {
|
||||||
|
if mode == store.ModeAdopted {
|
||||||
|
return "an adopted"
|
||||||
|
}
|
||||||
|
return "a converged"
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq issue 104: a declaration is refused for the other mode than the node is in,
|
||||||
|
// naming both and the act that changes it; a node no mode has been said to takes either.
|
||||||
|
func TestADeclarationForTheOtherModeIsRefused(t *testing.T) {
|
||||||
|
converged := parse(t, `{"declaration":1,"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`)
|
||||||
|
adopted := parse(t, `{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`)
|
||||||
|
|
||||||
|
err := CheckMode(store.State{Mode: store.ModeAdopted}, converged)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "this node is adopted; the declaration says converged") ||
|
||||||
|
!strings.Contains(err.Error(), "`converge`") {
|
||||||
|
t.Errorf("a converged declaration on an adopted node: %v", err)
|
||||||
|
}
|
||||||
|
err = CheckMode(store.State{Mode: store.ModeConverged}, adopted)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "this node is converged; the declaration says adopted") ||
|
||||||
|
!strings.Contains(err.Error(), "`adopt`") {
|
||||||
|
t.Errorf("an adopted declaration on a converged node: %v", err)
|
||||||
|
}
|
||||||
|
if err := CheckMode(store.State{Mode: store.ModeAdopted}, adopted); err != nil {
|
||||||
|
t.Errorf("the node's own mode was refused: %v", err)
|
||||||
|
}
|
||||||
|
if err := CheckMode(store.State{}, converged); err != nil {
|
||||||
|
t.Errorf("a node in no mode yet refused a declaration: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -339,8 +339,18 @@ func TestReturningToAdoptedLoadsTheGuardBeforeRemovingTheFilter(t *testing.T) {
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
guard := filepath.Join(dir, "guard.nft")
|
guard := filepath.Join(dir, "guard.nft")
|
||||||
guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}`
|
guardFile := `{"id":"adoption.guard","type":"file","path":"` + guard + `","content":"table inet mesh_guard {}\n"}`
|
||||||
|
// The filter's unit file is the mesh's own, written where there was none — which is what makes
|
||||||
|
// its unit the mesh's to stop, with or without a record of what was found (novox/hq ADR 0118).
|
||||||
|
units := t.TempDir()
|
||||||
|
was := unitDir
|
||||||
|
unitDir = units
|
||||||
|
t.Cleanup(func() { unitDir = was })
|
||||||
|
filterUnit := filepath.Join(units, "mesh-filter.service")
|
||||||
for _, stopFails := range []bool{false, true} {
|
for _, stopFails := range []bool{false, true} {
|
||||||
_ = os.Remove(guard)
|
_ = os.Remove(guard)
|
||||||
|
if err := os.WriteFile(filterUnit, []byte("[Unit]\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
guardUpAtStop := false
|
guardUpAtStop := false
|
||||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
if name != "systemctl" {
|
if name != "systemctl" {
|
||||||
@@ -358,7 +368,9 @@ func TestReturningToAdoptedLoadsTheGuardBeforeRemovingTheFilter(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
|
// As a host recorded them before it kept what it found: no Found on the service.
|
||||||
converged := store.State{Resources: []store.Applied{
|
converged := store.State{Resources: []store.Applied{
|
||||||
|
{ID: "nftables.unit", Type: "file", Target: filterUnit, Origin: store.OriginDeclared},
|
||||||
{ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}}
|
{ID: "nftables.load", Type: "service", Target: "mesh-filter.service", Origin: store.OriginDeclared}}}
|
||||||
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run)
|
_, state, err := applyWith(t, adopted(t, `{"taken":[]}`, withConf(dir)+","+guardFile), converged, run)
|
||||||
if !guardUpAtStop {
|
if !guardUpAtStop {
|
||||||
|
|||||||
@@ -0,0 +1,337 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/firewall"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A declaration is said before it is done.
|
||||||
|
//
|
||||||
|
// An apply that prints what it did after it did it is a report; what an operator reaching for
|
||||||
|
// `reconcile` under pressure needs is a preview — the base filter that closed an adopted
|
||||||
|
// control-node for forty-five minutes was listed nowhere until it was on disk (novox/hq issue
|
||||||
|
// 104). Converging already previews on the controller; the host's own commands now do too, and
|
||||||
|
// `--dry-run` is the preview and nothing else.
|
||||||
|
|
||||||
|
// Step is one thing an apply would do to this machine.
|
||||||
|
type Step struct {
|
||||||
|
// Verb is create · update · check · hold · run · remove · forget · restore · disable · enable.
|
||||||
|
Verb string `json:"verb"`
|
||||||
|
Type string `json:"type,omitempty"`
|
||||||
|
ID string `json:"id,omitempty"`
|
||||||
|
Target string `json:"target,omitempty"`
|
||||||
|
Why string `json:"why,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Step) String() string {
|
||||||
|
line := fmt.Sprintf("%-8s %-10s %s", s.Verb, s.Type, s.ID)
|
||||||
|
if s.Target != "" && s.Target != s.ID {
|
||||||
|
line += " (" + s.Target + ")"
|
||||||
|
}
|
||||||
|
if s.Why != "" {
|
||||||
|
line += " — " + s.Why
|
||||||
|
}
|
||||||
|
return line
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plan says what applying a declaration would change, in the order ApplyKeeping would do it,
|
||||||
|
// before anything on the machine is touched.
|
||||||
|
//
|
||||||
|
// **Read from the declaration and the node's own record, not from the machine.** What the
|
||||||
|
// record cannot settle — whether a file recorded here has since drifted, whether a container
|
||||||
|
// runs the spec it was made from — is said as a check, because that is what the apply does: it
|
||||||
|
// reads the machine and corrects it. What the record does settle is said as it is: a resource
|
||||||
|
// with no record is created; a plain file whose declared content differs from what this host
|
||||||
|
// last wrote is updated; a hold is kept; an action is run — an action is a command, and a
|
||||||
|
// preview that folded it into "check" would hide the one kind of step that is not read back
|
||||||
|
// from state.
|
||||||
|
func Plan(d *declaration.Declaration, known store.State, origin string) []Step {
|
||||||
|
var steps []Step
|
||||||
|
|
||||||
|
declared := map[string]bool{}
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
declared[r.Identity()] = true
|
||||||
|
}
|
||||||
|
// The found tunnel's configuration is held under an id of its own, declared for as long as the
|
||||||
|
// service taking it over is — as ApplyKeeping counts it, or a plan would forget a hold the
|
||||||
|
// apply keeps (novox/hq ADR 0105).
|
||||||
|
if svc := takesOver(d); svc != nil {
|
||||||
|
declared[takeOverID(svc)] = true
|
||||||
|
}
|
||||||
|
rec := known.Firewall
|
||||||
|
ufw := rec != nil && rec.Kind == string(firewall.UFW)
|
||||||
|
|
||||||
|
if d.Adoption != nil && ufw && rec.DisabledByMesh {
|
||||||
|
steps = append(steps, Step{Verb: "enable", Type: "firewall", ID: "ufw",
|
||||||
|
Why: "this node is adopted again, so the firewall found on it is put back in force"})
|
||||||
|
}
|
||||||
|
|
||||||
|
// What is held and no longer declared is let go of on paper only (ApplyKeeping does this
|
||||||
|
// before the resources); the file or container itself is left as found.
|
||||||
|
if origin == store.OriginDeclared {
|
||||||
|
for _, h := range known.Held {
|
||||||
|
if declared[h.ID] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
steps = append(steps, Step{Verb: "forget", Type: h.Kind, ID: h.ID, Target: h.Target,
|
||||||
|
Why: "held for " + h.Module + " and no longer declared; left as found"})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var protecting, orphans []Step
|
||||||
|
made := meshMadeUnits(known)
|
||||||
|
for _, orphan := range known.Orphans(declared, origin) {
|
||||||
|
step := Step{Verb: "remove", Type: orphan.Type, ID: orphan.ID, Target: orphan.Target,
|
||||||
|
Why: "recorded here and no longer declared"}
|
||||||
|
switch {
|
||||||
|
case orphan.Stateless:
|
||||||
|
step.Verb, step.Why = "forget", "no longer declared; its unit's state was never the mesh's and is left as it is"
|
||||||
|
case orphan.Type == string(declaration.TypeService):
|
||||||
|
// What removal will do, said before it does it (novox/hq ADR 0118), in removeService's
|
||||||
|
// words. "restore" only where it may stop or disable something — the record cannot say
|
||||||
|
// whether the unit is still as the mesh left it, so "may" is as far as a preview goes —
|
||||||
|
// and a unit whose file the mesh wrote is named as the mesh's, since that one is
|
||||||
|
// stopped whatever was found.
|
||||||
|
f := orphan.Found
|
||||||
|
switch {
|
||||||
|
case made[orphan.Target]:
|
||||||
|
step.Verb, step.Why = "remove", "no longer declared; the mesh wrote its unit file, so it is "+
|
||||||
|
"stopped and disabled at boot before that file goes"
|
||||||
|
case f == nil:
|
||||||
|
step.Verb, step.Why = "forget", "no longer declared; recorded before the host kept what it "+
|
||||||
|
"found, so it is left as it is"
|
||||||
|
case f.State == "stopped" || f.Boot == "disabled":
|
||||||
|
var back []string
|
||||||
|
if f.State == "stopped" {
|
||||||
|
back = append(back, "stopped")
|
||||||
|
}
|
||||||
|
if f.Boot == "disabled" {
|
||||||
|
back = append(back, "disabled at boot")
|
||||||
|
}
|
||||||
|
step.Verb, step.Why = "restore", "no longer declared; the host found it "+
|
||||||
|
strings.Join(back, " and ")+", and it goes back to that if the mesh changed it"
|
||||||
|
default:
|
||||||
|
step.Verb, step.Why = "forget", "no longer declared; it was running before the mesh and is "+
|
||||||
|
"left as it is — nothing is started or stopped on the way out"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if d.Adoption == nil && strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) {
|
||||||
|
step.Why = "what protected this node while adopted; removed last, once everything else applied"
|
||||||
|
protecting = append(protecting, step)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
orphans = append(orphans, step)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The guard goes up before anything is removed on an adopted node; on a converged one the
|
||||||
|
// removals go first (novox/hq ADR 0103).
|
||||||
|
var guard, rest []declaration.Resource
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
if d.Adoption != nil && strings.HasPrefix(r.Identity(), guardPrefix) {
|
||||||
|
guard = append(guard, r)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rest = append(rest, r)
|
||||||
|
}
|
||||||
|
for _, r := range guard {
|
||||||
|
steps = append(steps, planned(r, d, known))
|
||||||
|
}
|
||||||
|
steps = append(steps, orphans...)
|
||||||
|
for _, r := range rest {
|
||||||
|
step := planned(r, d, known)
|
||||||
|
if svc, ok := r.(*declaration.Service); ok && svc.TakesOver != nil && d.Adoption != nil && step.Verb != "hold" {
|
||||||
|
// The take comes before the service that replaces the tunnel, as it does in the apply.
|
||||||
|
steps = append(steps, plannedTake(svc, known))
|
||||||
|
}
|
||||||
|
steps = append(steps, step)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only a declaration from the mesh converges a node; a bundle or a file never retires the
|
||||||
|
// firewall found here, and neither says so in a plan.
|
||||||
|
if d.Adoption == nil && origin == store.OriginDeclared && ufw && rec.WasActive && !rec.DisabledByMesh {
|
||||||
|
steps = append(steps, Step{Verb: "disable", Type: "firewall", ID: "ufw",
|
||||||
|
Why: "this node converges: retired once the mesh's own filter is loaded, never before; " +
|
||||||
|
"its configuration stays on disk"})
|
||||||
|
}
|
||||||
|
steps = append(steps, protecting...)
|
||||||
|
return steps
|
||||||
|
}
|
||||||
|
|
||||||
|
// planned is what one declared resource would come to.
|
||||||
|
//
|
||||||
|
// **In the order holdOnAdopted decides it**, because the one cutover ADR 0100 says must be
|
||||||
|
// previewed is the one a plan gets backwards if it looks at the record first: a resource this
|
||||||
|
// node holds for a module the declaration now says is taken is not held any longer — it is
|
||||||
|
// applied, and what was found is replaced. The declaration's word on which modules are untaken
|
||||||
|
// comes first; the record of what is held only says what that replacement replaces.
|
||||||
|
func planned(r declaration.Resource, d *declaration.Declaration, known store.State) Step {
|
||||||
|
step := Step{Type: string(r.Kind()), ID: r.Identity(), Target: r.Target()}
|
||||||
|
|
||||||
|
if d.Adoption != nil {
|
||||||
|
// Something run inside a held container is held with it, while that container's module
|
||||||
|
// is untaken; once the module is taken the container is replaced before this runs.
|
||||||
|
if in := runsIn(r); in != "" {
|
||||||
|
if container, isHeld := heldContainer(known, in); isHeld {
|
||||||
|
if _, untaken := d.Adoption.Untaken[container.Module]; untaken {
|
||||||
|
step.Verb = "hold"
|
||||||
|
step.Why = "runs in " + in + ", which is held as found; not run until " + container.Module + " is taken"
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A file written into, or a service whose lifecycle is the machine's, replaces nothing that
|
||||||
|
// was found, so it is never held (ADR 0102, ADR 0117).
|
||||||
|
into := replacesNothing(r)
|
||||||
|
h, held := known.HeldAt(r.Identity())
|
||||||
|
module, untaken := d.Adoption.UntakenModuleOf(r.Identity())
|
||||||
|
switch {
|
||||||
|
case into:
|
||||||
|
case untaken && held:
|
||||||
|
step.Verb, step.Why = "hold", "found on this machine and kept as it is until "+module+" is taken"
|
||||||
|
return step
|
||||||
|
case untaken:
|
||||||
|
step.Verb = "create"
|
||||||
|
step.Why = "unless it is found on this machine — then held as it is until " + module + " is taken"
|
||||||
|
return step
|
||||||
|
case held:
|
||||||
|
// The cutover: the module is taken, and what was held for it is replaced.
|
||||||
|
step.Verb = "create"
|
||||||
|
if _, recorded := known.Find(r.Identity()); recorded {
|
||||||
|
step.Verb = "update"
|
||||||
|
}
|
||||||
|
step.Why = h.Module + " is taken: replaces what was found and held"
|
||||||
|
if h.Kept != "" {
|
||||||
|
step.Why += "; the original stays at " + h.Kept
|
||||||
|
}
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if a, ok := r.(*declaration.Action); ok {
|
||||||
|
// Named as what it is. Its verify decides whether it runs, and that is read from the
|
||||||
|
// machine, not the record.
|
||||||
|
step.Verb = "run"
|
||||||
|
step.Target = ""
|
||||||
|
step.Why = fmt.Sprintf("an action: `%s`, unless its verify `%s` already passes; if it fails, "+
|
||||||
|
"nothing after it is attempted", strings.Join(a.Command, " "), strings.Join(a.Verify, " "))
|
||||||
|
if a.In != "" {
|
||||||
|
step.Why = "in " + a.In + ", " + step.Why
|
||||||
|
}
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
|
||||||
|
if svc, ok := r.(*declaration.Service); ok && svc.Stateless() {
|
||||||
|
// Nothing is created: the unit and whether it runs are the machine's (novox/hq ADR 0117).
|
||||||
|
step.Verb, step.Why = "check", "its lifecycle is the machine's; reloaded or restarted only if "+
|
||||||
|
"running when what it reflects changes"
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
was, recorded := known.Find(r.Identity())
|
||||||
|
if !recorded {
|
||||||
|
step.Verb, step.Why = "create", "no record of it on this node"
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
if want := wouldWrite(r); want != "" && was.Wrote != "" && want != was.Wrote {
|
||||||
|
step.Verb, step.Why = "update", "the declaration changed since this host applied it"
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
if c, ok := r.(*declaration.Container); ok {
|
||||||
|
if changed := readsChanged(c, d, known, was.Reads); len(changed) > 0 {
|
||||||
|
step.Verb = "update"
|
||||||
|
step.Why = "recreated: " + strings.Join(changed, ", ") + " changed since it was created"
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
}
|
||||||
|
step.Verb, step.Why = "check", "recorded here; corrected if this machine drifted from it"
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
|
||||||
|
// plannedTake is what the take of a found tunnel would do to its configuration (novox/hq ADR 0105,
|
||||||
|
// ADR 0119): kept as found while the take is not proven, and retired — removed from where its unit
|
||||||
|
// reads it, its original staying kept — by the first apply that finds the mesh's interface up in
|
||||||
|
// its place with a peer handshaken. Whether that is this apply is read from the machine, which a
|
||||||
|
// plan does not do, so it says when rather than whether. One the mesh retired already is said as
|
||||||
|
// retired: nothing brings it back.
|
||||||
|
func plannedTake(svc *declaration.Service, known store.State) Step {
|
||||||
|
t := svc.TakesOver
|
||||||
|
step := Step{Verb: "hold", Type: string(declaration.TypeFile), ID: takeOverID(svc), Target: t.Config}
|
||||||
|
if r, ok := known.RetiredAt(t.Config); ok {
|
||||||
|
step.Verb = "check"
|
||||||
|
step.Why = "retired once the take of " + t.Interface + " was proven; its original stays at " + r.Kept +
|
||||||
|
" and the mesh never brings it back"
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
step.Why = "the configuration of the tunnel " + t.Interface + ", kept as found while " + svc.Unit +
|
||||||
|
" takes it over (" + t.Unit + " stopped and disabled, never flushed); retired — removed from " +
|
||||||
|
t.Config + ", its original staying kept — once the take is proven by a peer handshaking on " +
|
||||||
|
strings.TrimPrefix(svc.Unit, "wg-quick@")
|
||||||
|
if h, ok := known.HeldAt(takeOverID(svc)); ok && h.Kept != "" {
|
||||||
|
step.Why += "; the original is at " + h.Kept
|
||||||
|
}
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
|
||||||
|
// readsChanged is which of the files a container was created reading the apply will hand it
|
||||||
|
// changed — the same comparison applyContainer makes (novox/hq 04-ISSUES/103), settled from the
|
||||||
|
// declaration and the record alone.
|
||||||
|
//
|
||||||
|
// A file's digest is what this apply will record for it: a plain file declared here, by its
|
||||||
|
// declared content; otherwise what this host last wrote there, under any id. A file neither
|
||||||
|
// declares nor records — an env-file a predecessor left — is read by the apply from the machine,
|
||||||
|
// which a plan does not do, so it stays a check. A container with no record of what it read was
|
||||||
|
// labelled before the host kept that record and is accepted as it is, so it is a check too.
|
||||||
|
func readsChanged(c *declaration.Container, d *declaration.Declaration, known store.State,
|
||||||
|
wasReading map[string]string) []string {
|
||||||
|
if len(wasReading) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
willWrite := map[string]string{}
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
if f, ok := r.(*declaration.File); ok {
|
||||||
|
if want := wouldWrite(f); want != "" {
|
||||||
|
willWrite[f.Path] = want
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Only what it still reads: a file it was created reading and no longer names is a changed
|
||||||
|
// declaration, not a changed file.
|
||||||
|
stillReads := map[string]bool{}
|
||||||
|
for _, path := range c.EnvFile {
|
||||||
|
stillReads[path] = true
|
||||||
|
}
|
||||||
|
for _, v := range c.Volumes {
|
||||||
|
if src := mountSource(v); strings.HasPrefix(src, "/") {
|
||||||
|
stillReads[src] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var changed []string
|
||||||
|
for _, path := range sortedKeys(wasReading) {
|
||||||
|
if !stillReads[path] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
now, settled := willWrite[path]
|
||||||
|
if !settled {
|
||||||
|
if f, recorded := known.At(string(declaration.TypeFile), path); recorded {
|
||||||
|
now, settled = f.Wrote, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if settled && now != wasReading[path] {
|
||||||
|
changed = append(changed, path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return changed
|
||||||
|
}
|
||||||
|
|
||||||
|
// wouldWrite is the digest a plain file would be recorded under, or empty where only the apply
|
||||||
|
// can know: a sealed file, one with secrets in it, one written into, one carrying bytes.
|
||||||
|
func wouldWrite(r declaration.Resource) string {
|
||||||
|
f, ok := r.(*declaration.File)
|
||||||
|
if !ok || f.Into != "" || f.Bytes != "" || f.Secret() || len(f.Secrets) > 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return digestOf(f.Content)
|
||||||
|
}
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq issue 104: what an apply would change is said before anything is, from the
|
||||||
|
// declaration and the node's record — and an action is named as the action it is.
|
||||||
|
|
||||||
|
func trusted(t *testing.T, raw string) *declaration.Declaration {
|
||||||
|
t.Helper()
|
||||||
|
d, err := declaration.ParseFileTrusted([]byte(raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("fixture is not a valid declaration: %v", err)
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
func verbs(steps []Step) string {
|
||||||
|
var out []string
|
||||||
|
for _, s := range steps {
|
||||||
|
out = append(out, s.Verb+" "+s.ID)
|
||||||
|
}
|
||||||
|
return strings.Join(out, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPlanNamesAnActionAsAnAction(t *testing.T) {
|
||||||
|
d := trusted(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"init","type":"action","command":["createdb","mesh"],"verify":["psql","-c","select 1"]}]}`)
|
||||||
|
steps := Plan(d, store.State{}, store.OriginCarried)
|
||||||
|
if len(steps) != 1 || steps[0].Verb != "run" {
|
||||||
|
t.Fatalf("an action was planned as %s", verbs(steps))
|
||||||
|
}
|
||||||
|
if !strings.Contains(steps[0].Why, "createdb mesh") || !strings.Contains(steps[0].Why, "nothing after it") {
|
||||||
|
t.Errorf("the plan does not say what the action runs and what failing it means: %q", steps[0].Why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPlanSaysWhatIsRecordedAndWhatIsNot(t *testing.T) {
|
||||||
|
d := parse(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"new","type":"file","path":"/tmp/new","content":"a\n"},
|
||||||
|
{"id":"same","type":"file","path":"/tmp/same","content":"b\n"},
|
||||||
|
{"id":"moved","type":"file","path":"/tmp/moved","content":"c\n"},
|
||||||
|
{"id":"sealed","type":"file","path":"/tmp/sealed","sealed":"AAAA","mode":"0600"}]}`)
|
||||||
|
known := store.State{}
|
||||||
|
known.Record(store.Applied{ID: "same", Type: "file", Target: "/tmp/same", Wrote: digestOf("b\n")})
|
||||||
|
known.Record(store.Applied{ID: "moved", Type: "file", Target: "/tmp/moved", Wrote: digestOf("old\n")})
|
||||||
|
known.Record(store.Applied{ID: "sealed", Type: "file", Target: "/tmp/sealed", Wrote: digestOf("secret")})
|
||||||
|
known.Record(store.Applied{ID: "gone", Type: "file", Target: "/tmp/gone"})
|
||||||
|
|
||||||
|
got := verbs(Plan(d, known, store.OriginCarried))
|
||||||
|
want := "remove gone, create new, check same, update moved, check sealed"
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("planned %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPlanSaysTheFirewallAConvergingNodeRetires(t *testing.T) {
|
||||||
|
d := parse(t, `{"declaration":1,"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`)
|
||||||
|
known := store.State{Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, FoundAt: time.Now()}}
|
||||||
|
known.Record(store.Applied{ID: "adoption.guard.table", Type: "file", Target: "/etc/guard", Origin: store.OriginDeclared})
|
||||||
|
|
||||||
|
got := verbs(Plan(d, known, store.OriginDeclared))
|
||||||
|
// The firewall goes last but for what protected the node, which goes after it.
|
||||||
|
if got != "create a, disable ufw, remove adoption.guard.table" {
|
||||||
|
t.Errorf("a converging node planned %q", got)
|
||||||
|
}
|
||||||
|
// A file or the bundle never retires the firewall found here, and says nothing about it.
|
||||||
|
if got := verbs(Plan(d, known, store.OriginCarried)); strings.Contains(got, "ufw") {
|
||||||
|
t.Errorf("a carried declaration planned to touch the firewall: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPlanHoldsWhatAnAdoptedNodeFound(t *testing.T) {
|
||||||
|
d := parse(t, `{"declaration":1,"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.page","hello-web.server"]}},
|
||||||
|
"resources":[
|
||||||
|
{"id":"hello-web.page","type":"file","path":"/srv/index.html","content":"x\n"},
|
||||||
|
{"id":"hello-web.server","type":"container","name":"hello-web","image":"example/web@sha256:0000000000000000000000000000000000000000000000000000000000000000"}]}`)
|
||||||
|
known := store.State{}
|
||||||
|
known.RecordHeld(store.Held{ID: "hello-web.page", Module: "hello-web", Kind: "file", Target: "/srv/index.html"})
|
||||||
|
|
||||||
|
steps := Plan(d, known, store.OriginDeclared)
|
||||||
|
if len(steps) != 2 || steps[0].Verb != "hold" || steps[1].Verb != "create" {
|
||||||
|
t.Fatalf("an adopted node planned %s", verbs(steps))
|
||||||
|
}
|
||||||
|
if !strings.Contains(steps[1].Why, "held as it is until hello-web is taken") {
|
||||||
|
t.Errorf("the plan does not say an untaken module's resource is held if found: %q", steps[1].Why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// both is a machine with a container runtime and ufw on it at once.
|
||||||
|
type both struct {
|
||||||
|
m *machine
|
||||||
|
u *ufwMachine
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *both) run(ctx context.Context, name string, args ...string) (string, error) {
|
||||||
|
switch name {
|
||||||
|
case "nft", "ufw", "iptables", "ip6tables", "firewall-cmd":
|
||||||
|
return b.u.run(ctx, name, args...)
|
||||||
|
}
|
||||||
|
return b.m.run(ctx, name, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ids(steps []Step) []string {
|
||||||
|
var out []string
|
||||||
|
for _, s := range steps {
|
||||||
|
if s.Type == "firewall" {
|
||||||
|
continue // said, not an outcome
|
||||||
|
}
|
||||||
|
out = append(out, s.ID)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func outcomeIDs(r Report) []string {
|
||||||
|
var out []string
|
||||||
|
for _, o := range r.Outcomes {
|
||||||
|
out = append(out, o.ID)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func write(t *testing.T, path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Defends novox/hq issue 104: the plan is the apply, said first — the same resources in the same
|
||||||
|
// order, and the one cutover ADR 0100 says must be previewed said as a cutover, not a hold.
|
||||||
|
func TestThePlanIsTheApplyInOrder(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
guard, page, keep, old := filepath.Join(dir, "guard.nft"), filepath.Join(dir, "index.html"),
|
||||||
|
filepath.Join(dir, "keep.html"), filepath.Join(dir, "old.conf")
|
||||||
|
for _, p := range []string{page, keep, old} {
|
||||||
|
write(t, p, "the predecessor's\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returning to adopted, with a guard to raise, an orphan, a hold that stays, a hold whose
|
||||||
|
// module is now taken, and a hold no longer declared.
|
||||||
|
back := adopted(t, `{"taken":["hello-web"],"untaken":{"keep":["keep.page"]}}`,
|
||||||
|
`{"id":"adoption.guard.table","type":"file","path":"`+guard+`","content":"table inet mesh-guard {}\n"},
|
||||||
|
{"id":"hello-web.page","type":"file","path":"`+page+`","content":"the mesh's page\n"},
|
||||||
|
{"id":"keep.page","type":"file","path":"`+keep+`","content":"the mesh's keep\n"}`)
|
||||||
|
known := store.State{Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, DisabledByMesh: true, FoundAt: time.Now()}}
|
||||||
|
known.Record(store.Applied{ID: "old.conf", Type: "file", Target: old, Origin: store.OriginDeclared})
|
||||||
|
for id, module := range map[string]string{"hello-web.page": "hello-web", "keep.page": "keep", "gone.page": "gone"} {
|
||||||
|
known.RecordHeld(store.Held{ID: id, Module: module, Kind: "file", Target: filepath.Join(dir, id), Since: time.Now()})
|
||||||
|
}
|
||||||
|
fake := &both{m: &machine{containers: map[string]*fakeContainer{}}, u: &ufwMachine{installed: true}}
|
||||||
|
|
||||||
|
plan := Plan(back, known, store.OriginDeclared)
|
||||||
|
report, state, err := ApplyKeeping(context.Background(), archHost(t), back, known, store.OriginDeclared,
|
||||||
|
fake.run, nil, nil, KeepIn(dir))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, want := verbs(plan), "enable ufw, forget gone.page, create adoption.guard.table, remove old.conf, "+
|
||||||
|
"create hello-web.page, hold keep.page"; got != want {
|
||||||
|
t.Errorf("planned %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if got, want := strings.Join(ids(plan), " "), strings.Join(outcomeIDs(report), " "); got != want {
|
||||||
|
t.Errorf("the plan said %q and the apply did %q", got, want)
|
||||||
|
}
|
||||||
|
taken := outcomeOf(report, "hello-web.page")
|
||||||
|
if !strings.Contains(taken.Detail, "taken") || !strings.Contains(plan[4].Why, "hello-web is taken: replaces what was found") {
|
||||||
|
t.Errorf("the cutover was applied as %q and planned as %q", taken.Detail, plan[4].Why)
|
||||||
|
}
|
||||||
|
if !fake.u.active || state.Firewall.DisabledByMesh {
|
||||||
|
t.Error("returning to adopted did not enable ufw again")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Converged, from the mesh: an orphan, a new file, ufw retired, and what protected the node
|
||||||
|
// removed last.
|
||||||
|
flip := parse(t, `{"declaration":1,"resources":[{"id":"a","type":"file","path":"`+filepath.Join(dir, "a.conf")+`","content":"a\n"}]}`)
|
||||||
|
write(t, old, "again\n")
|
||||||
|
known = store.State{Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, FoundAt: time.Now()}}
|
||||||
|
known.Record(store.Applied{ID: "adoption.guard.table", Type: "file", Target: guard, Origin: store.OriginDeclared})
|
||||||
|
known.Record(store.Applied{ID: "old.conf", Type: "file", Target: old, Origin: store.OriginDeclared})
|
||||||
|
fake = &both{m: &machine{containers: map[string]*fakeContainer{}}, u: &ufwMachine{installed: true, active: true,
|
||||||
|
ruleset: "table inet mesh {\n}\n"}}
|
||||||
|
|
||||||
|
plan = Plan(flip, known, store.OriginDeclared)
|
||||||
|
report, state, err = ApplyKeeping(context.Background(), archHost(t), flip, known, store.OriginDeclared,
|
||||||
|
fake.run, nil, nil, KeepIn(dir))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, want := verbs(plan), "remove old.conf, create a, disable ufw, remove adoption.guard.table"; got != want {
|
||||||
|
t.Errorf("planned %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if got, want := strings.Join(ids(plan), " "), strings.Join(outcomeIDs(report), " "); got != want {
|
||||||
|
t.Errorf("the plan said %q and the apply did %q", got, want)
|
||||||
|
}
|
||||||
|
if fake.u.active || !state.Firewall.DisabledByMesh {
|
||||||
|
t.Error("converging did not retire ufw")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAResourceRunInAHeldContainerIsPlannedAsItIsApplied(t *testing.T) {
|
||||||
|
// A run-once step sharing a container's namespace runs in it, as an action's `in` does.
|
||||||
|
img := "example/x@sha256:0000000000000000000000000000000000000000000000000000000000000000"
|
||||||
|
d := parse(t, `{"declaration":1,"adoption":{"taken":["web"],"untaken":{"db":["db.server"]}},"resources":[
|
||||||
|
{"id":"web.server","type":"container","name":"web","image":"`+img+`"},
|
||||||
|
{"id":"db.server","type":"container","name":"db","image":"`+img+`"},
|
||||||
|
{"id":"db.init","type":"container","name":"db-init","image":"`+img+`","run-once":true,"network":"container:db"},
|
||||||
|
{"id":"web.warm","type":"container","name":"web-warm","image":"`+img+`","run-once":true,"network":"container:web"}]}`)
|
||||||
|
known := store.State{}
|
||||||
|
known.RecordHeld(store.Held{ID: "db.server", Module: "db", Kind: "container", Target: "db", Container: "predecessor"})
|
||||||
|
known.RecordHeld(store.Held{ID: "web.server", Module: "web", Kind: "container", Target: "web", Container: "predecessor"})
|
||||||
|
got := verbs(Plan(d, known, store.OriginDeclared))
|
||||||
|
// db is untaken, so what runs in it waits; web is taken, so its held container is replaced (the
|
||||||
|
// cutover) and what runs in it runs.
|
||||||
|
if got != "create web.server, hold db.server, hold db.init, create web.warm" {
|
||||||
|
t.Errorf("planned %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPlanSaysAContainerIsRecreatedWhenAFileItReadsChanged(t *testing.T) {
|
||||||
|
// The apply recreates a container when the content of a file it reads at creation changed
|
||||||
|
// (novox/hq 04-ISSUES/103); the plan says so from the record alone — what the container was
|
||||||
|
// created reading, against what this apply will write. And a container recorded before the
|
||||||
|
// host kept that record is accepted, so it is a check, not an update.
|
||||||
|
dir := t.TempDir()
|
||||||
|
env := filepath.Join(dir, "forge.env")
|
||||||
|
declare := func(port string) *declaration.Declaration {
|
||||||
|
return trusted(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"forge.env","type":"file","path":"`+env+`","content":"DATABASE_PORT=`+port+`\n"},
|
||||||
|
{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}]}`)
|
||||||
|
}
|
||||||
|
created := digestOf("DATABASE_PORT=5432\n")
|
||||||
|
known := store.State{}
|
||||||
|
known.Record(store.Applied{ID: "forge.env", Type: "file", Target: env, Wrote: created})
|
||||||
|
known.Record(store.Applied{ID: "forge.server", Type: "container", Target: "forge",
|
||||||
|
Reads: map[string]string{env: created}})
|
||||||
|
|
||||||
|
if got := verbs(Plan(declare("5432"), known, store.OriginCarried)); got != "check forge.env, check forge.server" {
|
||||||
|
t.Errorf("nothing changed and the plan says %q", got)
|
||||||
|
}
|
||||||
|
steps := Plan(declare("5433"), known, store.OriginCarried)
|
||||||
|
if got := verbs(steps); got != "update forge.env, update forge.server" {
|
||||||
|
t.Fatalf("the env-file changes and the plan says %q", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(steps[1].Why, env+" changed") {
|
||||||
|
t.Errorf("the plan does not say which file: %+v", steps[1])
|
||||||
|
}
|
||||||
|
|
||||||
|
// No record of what it read: labelled by an earlier host, accepted as it is.
|
||||||
|
known.Record(store.Applied{ID: "forge.server", Type: "container", Target: "forge"})
|
||||||
|
if got := verbs(Plan(declare("5433"), known, store.OriginCarried)); got != "update forge.env, check forge.server" {
|
||||||
|
t.Errorf("a container with no record of what it read is planned as %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPlanSaysWhichUnitsAnUndeclareGivesBackAndWhichItLeaves(t *testing.T) {
|
||||||
|
// novox/hq ADR 0118, said before it is done: "restore" only where removal may stop or disable
|
||||||
|
// something, and a unit whose file the mesh wrote named as the mesh's.
|
||||||
|
known := store.State{}
|
||||||
|
known.Record(store.Applied{ID: "guard-unit", Type: "file", Target: "/etc/systemd/system/mesh-guard.service"})
|
||||||
|
known.Record(store.Applied{ID: "guard", Type: "service", Target: "mesh-guard.service"})
|
||||||
|
known.Record(store.Applied{ID: "filter", Type: "service", Target: "filter.service",
|
||||||
|
Found: &store.FoundUnit{State: "stopped"}})
|
||||||
|
known.Record(store.Applied{ID: "boot", Type: "service", Target: "boot.service",
|
||||||
|
Found: &store.FoundUnit{State: "running", Boot: "disabled"}})
|
||||||
|
known.Record(store.Applied{ID: "runtime", Type: "service", Target: "docker.service",
|
||||||
|
Found: &store.FoundUnit{State: "running", Boot: "enabled"}})
|
||||||
|
known.Record(store.Applied{ID: "old", Type: "service", Target: "sshd.service"})
|
||||||
|
known.Record(store.Applied{ID: "nm", Type: "service", Target: "NetworkManager.service", Stateless: true})
|
||||||
|
|
||||||
|
steps := Plan(parse(t, nothingButA(t)), known, store.OriginCarried)
|
||||||
|
got := verbs(steps)
|
||||||
|
want := "forget nm, forget old, forget runtime, restore boot, restore filter, remove guard, remove guard-unit, create other"
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("planned %s\nwant %s", got, want)
|
||||||
|
}
|
||||||
|
for _, s := range steps {
|
||||||
|
switch s.ID {
|
||||||
|
case "guard":
|
||||||
|
if !strings.Contains(s.Why, "unit file") {
|
||||||
|
t.Errorf("the mesh's own unit was not named as the mesh's: %q", s.Why)
|
||||||
|
}
|
||||||
|
case "filter":
|
||||||
|
if !strings.Contains(s.Why, "found it stopped") {
|
||||||
|
t.Errorf("what the unit goes back to went unsaid: %q", s.Why)
|
||||||
|
}
|
||||||
|
case "old":
|
||||||
|
if !strings.Contains(s.Why, "left as it is") {
|
||||||
|
t.Errorf("a unit with nothing found was not said to be left: %q", s.Why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -30,7 +30,7 @@ import (
|
|||||||
// Under the mesh's own directory rather than somewhere a distribution owns: these are files the
|
// Under the mesh's own directory rather than somewhere a distribution owns: these are files the
|
||||||
// mesh puts there and replaces, and putting them where a package manager also writes is how two
|
// mesh puts there and replaces, and putting them where a package manager also writes is how two
|
||||||
// owners end up disagreeing about one path.
|
// owners end up disagreeing about one path.
|
||||||
const daemonRoot = "/var/lib/mesh/daemons"
|
var daemonRoot = "/var/lib/mesh/daemons"
|
||||||
|
|
||||||
// unitDir is where the mesh writes the units it owns. A variable only so a test can point it at a
|
// unitDir is where the mesh writes the units it owns. A variable only so a test can point it at a
|
||||||
// directory of its own.
|
// directory of its own.
|
||||||
@@ -290,3 +290,54 @@ func unitValue(key, value string) string {
|
|||||||
).Replace(value)
|
).Replace(value)
|
||||||
return `"` + key + "=" + escaped + `"`
|
return `"` + key + "=" + escaped + `"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// removeProcess takes away a process the mesh no longer declares: its timer and unit stopped and
|
||||||
|
// disabled, their files removed, the supervisor told, and the unpacked bundle deleted.
|
||||||
|
//
|
||||||
|
// **All of it is the host's**, which is why all of it goes (novox/hq ADR 0118). Before this there
|
||||||
|
// was no way to remove a process at all, and one left undeclared failed every apply on its node
|
||||||
|
// until someone edited the host's state by hand — unassigning any module that ran its own code
|
||||||
|
// stranded the machine.
|
||||||
|
//
|
||||||
|
// Idempotent, like every removal: a unit already gone is not an error, and a record whose files
|
||||||
|
// have all vanished is forgotten rather than reported as removed.
|
||||||
|
func removeProcess(ctx context.Context, a store.Applied, run Runner) (string, string, error) {
|
||||||
|
name := a.Target
|
||||||
|
if problem := declaration.ProcessNameProblem(name); problem != "" {
|
||||||
|
// The name is a unit name and a directory under the mesh's own, and what goes is that
|
||||||
|
// directory, whole. One that could climb out of either — ".." is the mesh's own directory's
|
||||||
|
// parent — is refused rather than acted on, whatever wrote it into the record.
|
||||||
|
return "", "", fmt.Errorf("a process recorded under %q cannot be removed by name: %s", name, problem)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, unit := range []string{name + ".timer", name + ".service"} {
|
||||||
|
path := filepath.Join(unitDir, unit)
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
found = true
|
||||||
|
// The timer first, so a scheduled run cannot start the service between the two.
|
||||||
|
if _, err := run(ctx, "systemctl", "disable", "--now", unit); err != nil {
|
||||||
|
return "", "", fmt.Errorf("stopping %s: %w", unit, err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bundle := filepath.Join(daemonRoot, name)
|
||||||
|
if _, err := os.Stat(bundle); err == nil {
|
||||||
|
found = true
|
||||||
|
if err := os.RemoveAll(bundle); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return "forgotten", "no longer there", nil
|
||||||
|
}
|
||||||
|
return "removed", "stopped; its unit and its bundle removed — the mesh's own code", nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
package apply
|
package apply
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/declaration"
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
func aProcess() *declaration.Process {
|
func aProcess() *declaration.Process {
|
||||||
@@ -163,3 +167,92 @@ func TestAnEnvironmentValueMeansWhatTheDeclarationSaid(t *testing.T) {
|
|||||||
t.Fatalf("a quote was not escaped, so the value ends early: %s", line)
|
t.Fatalf("a quote was not escaped, so the value ends early: %s", line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAnUndeclaredProcessIsRemovedWithItsUnitAndBundle(t *testing.T) {
|
||||||
|
// novox/hq ADR 0118: a process's unit and bundle are the host's own, so they go with the
|
||||||
|
// declaration. Before, there was no way to remove a process at all, and one left undeclared
|
||||||
|
// failed every apply on its node.
|
||||||
|
units, bundles := t.TempDir(), t.TempDir()
|
||||||
|
wasUnits, wasBundles := unitDir, daemonRoot
|
||||||
|
unitDir, daemonRoot = units, bundles
|
||||||
|
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
||||||
|
|
||||||
|
for _, f := range []string{"mesh-job.service", "mesh-job.timer"} {
|
||||||
|
if err := os.WriteFile(filepath.Join(units, f), []byte("[Unit]\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Join(bundles, "mesh-job", "bin"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var commands []string
|
||||||
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||||
|
commands = append(commands, strings.Join(args, " "))
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
known := store.State{Resources: []store.Applied{{ID: "p", Type: "process", Target: "mesh-job"}}}
|
||||||
|
d := parse(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
|
||||||
|
]}`)
|
||||||
|
report, after, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("an undeclared process failed the apply: %v", err)
|
||||||
|
}
|
||||||
|
joined := strings.Join(commands, "; ")
|
||||||
|
timer, service := strings.Index(joined, "disable --now mesh-job.timer"), strings.Index(joined, "disable --now mesh-job.service")
|
||||||
|
if timer < 0 || service < 0 || timer > service {
|
||||||
|
t.Errorf("the timer and the unit were not stopped, timer first: %s", joined)
|
||||||
|
}
|
||||||
|
if !strings.Contains(joined, "daemon-reload") {
|
||||||
|
t.Errorf("the service manager was not told its units changed: %s", joined)
|
||||||
|
}
|
||||||
|
for _, gone := range []string{filepath.Join(units, "mesh-job.service"), filepath.Join(units, "mesh-job.timer"), filepath.Join(bundles, "mesh-job")} {
|
||||||
|
if _, err := os.Stat(gone); !os.IsNotExist(err) {
|
||||||
|
t.Errorf("%s is still there", gone)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, "p"); o.Action != "removed" {
|
||||||
|
t.Errorf("outcome %+v, want removed", o)
|
||||||
|
}
|
||||||
|
if _, still := after.Find("p"); still {
|
||||||
|
t.Error("the host still believes it owns the process")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Again, with everything already gone: forgotten, not an error.
|
||||||
|
_, _, err = Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("removing a process that is already gone failed: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAProcessRecordedUnderAPathlikeNameIsRefusedNotRemoved(t *testing.T) {
|
||||||
|
// filepath.Join(daemonRoot, "..") is the mesh's own directory, and removal deletes what that
|
||||||
|
// names, whole. A record is what some host wrote, perhaps under looser rules than today's, so
|
||||||
|
// the removal holds the name to the declaration's rule again (novox/hq ADR 0118).
|
||||||
|
root := t.TempDir()
|
||||||
|
bundles := filepath.Join(root, "daemons")
|
||||||
|
wasUnits, wasBundles := unitDir, daemonRoot
|
||||||
|
unitDir, daemonRoot = t.TempDir(), bundles
|
||||||
|
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
||||||
|
precious := filepath.Join(root, "state.json")
|
||||||
|
if err := os.MkdirAll(filepath.Join(bundles, "other"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(precious, []byte("{}"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"..", ".", "", "-x"} {
|
||||||
|
known := store.State{Resources: []store.Applied{{ID: "p", Type: "process", Target: name}}}
|
||||||
|
d := parse(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"}
|
||||||
|
]}`)
|
||||||
|
if _, _, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, noServices, nil, nil); err == nil {
|
||||||
|
t.Errorf("a process recorded as %q was removed by name", name)
|
||||||
|
}
|
||||||
|
for _, still := range []string{precious, filepath.Join(bundles, "other")} {
|
||||||
|
if _, err := os.Stat(still); err != nil {
|
||||||
|
t.Fatalf("removing a process recorded as %q took %s with it", name, still)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -113,7 +113,7 @@ func TestAFailedRunOnceStepGatesWhatFollows(t *testing.T) {
|
|||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "info":
|
case "info":
|
||||||
return "27.0\n", nil
|
return "27.0\n", nil
|
||||||
case "inspect":
|
case "container":
|
||||||
return "false\t\n", errors.New("no such container")
|
return "false\t\n", errors.New("no such container")
|
||||||
case "run":
|
case "run":
|
||||||
startedNames = append(startedNames, nameOf(args))
|
startedNames = append(startedNames, nameOf(args))
|
||||||
@@ -287,7 +287,7 @@ func TestAContainerNamingARunOnceStepIsRecreatedWhenItRan(t *testing.T) {
|
|||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "info":
|
case "info":
|
||||||
return "27.0\n", nil
|
return "27.0\n", nil
|
||||||
case "inspect":
|
case "container":
|
||||||
// The server is up, made from exactly this spec — nothing but the step's run says
|
// The server is up, made from exactly this spec — nothing but the step's run says
|
||||||
// it must be replaced.
|
// it must be replaced.
|
||||||
return "true\t" + spec, nil
|
return "true\t" + spec, nil
|
||||||
|
|||||||
@@ -208,7 +208,7 @@ func TestAScheduledStepDoesNotGateWhatFollows(t *testing.T) {
|
|||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "info":
|
case "info":
|
||||||
return "27.0\n", nil
|
return "27.0\n", nil
|
||||||
case "inspect":
|
case "container":
|
||||||
// The container name is the last argument to `docker inspect --format ... <name>`.
|
// The container name is the last argument to `docker inspect --format ... <name>`.
|
||||||
target := args[len(args)-1]
|
target := args[len(args)-1]
|
||||||
if spec, up := specs[target]; up {
|
if spec, up := specs[target]; up {
|
||||||
|
|||||||
@@ -0,0 +1,191 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq ADR 0117: a service that omits its state leaves the unit's lifecycle to the
|
||||||
|
// machine. The mesh reflects its triggers on a unit already running and does nothing else to it —
|
||||||
|
// never starts, stops, enables or disables it, and forgets it when undeclared.
|
||||||
|
|
||||||
|
// unitIn is a service manager whose one unit is active or not, recording what it is asked.
|
||||||
|
func unitIn(active bool, commands *[]string) Runner {
|
||||||
|
return func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
line := name + " " + strings.Join(args, " ")
|
||||||
|
*commands = append(*commands, line)
|
||||||
|
switch {
|
||||||
|
case strings.Contains(line, "is-enabled"):
|
||||||
|
return "enabled", nil
|
||||||
|
case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
|
||||||
|
if active {
|
||||||
|
return "LoadState=loaded\nActiveState=active\nSubState=running", nil
|
||||||
|
}
|
||||||
|
return "LoadState=loaded\nActiveState=inactive\nSubState=dead", nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func statelessDecl(path, content, triggers string) string {
|
||||||
|
return fmt.Sprintf(`{"declaration":1,"resources":[
|
||||||
|
{"id":"uplink.conf","type":"file","path":%q,"into":"block","content":%q},
|
||||||
|
{"id":"uplink.manager","type":"service","unit":"NetworkManager.service",%s}
|
||||||
|
]}`, path, content, triggers)
|
||||||
|
}
|
||||||
|
|
||||||
|
// touched is whether any command would change the unit's lifecycle.
|
||||||
|
func touched(commands []string) []string {
|
||||||
|
var changing []string
|
||||||
|
for _, c := range commands {
|
||||||
|
for _, verb := range []string{" start ", " stop ", " restart ", " enable ", " disable ", " reload "} {
|
||||||
|
if strings.Contains(c+" ", verb) {
|
||||||
|
changing = append(changing, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return changing
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStatelessServiceRunningIsReloadedForItsTrigger(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "mesh.conf")
|
||||||
|
var commands []string
|
||||||
|
report, _, err := Apply(context.Background(), archHost(t),
|
||||||
|
parse(t, statelessDecl(path, "[main]\ndns=none\n", `"reload-on":["uplink.conf"]`)),
|
||||||
|
store.State{}, store.OriginDeclared, unitIn(true, &commands), nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
joined := strings.Join(commands, "\n")
|
||||||
|
if !strings.Contains(joined, "systemctl reload NetworkManager.service") {
|
||||||
|
t.Errorf("the running manager was not reloaded; commands were %v", commands)
|
||||||
|
}
|
||||||
|
for _, c := range touched(commands) {
|
||||||
|
if !strings.Contains(c, "reload") {
|
||||||
|
t.Errorf("the manager's lifecycle was touched: %s", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, "uplink.manager"); o.Action != "updated" || !strings.Contains(o.Detail, "reloaded for uplink.conf") {
|
||||||
|
t.Errorf("reported as %q: %s", o.Action, o.Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStatelessServiceNotRunningIsLeftSo(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "mesh.conf")
|
||||||
|
for _, triggers := range []string{`"reload-on":["uplink.conf"]`, `"restart-on":["uplink.conf"]`} {
|
||||||
|
var commands []string
|
||||||
|
report, _, err := Apply(context.Background(), archHost(t),
|
||||||
|
parse(t, statelessDecl(path, fmt.Sprintf("# %s\n", triggers), triggers)),
|
||||||
|
store.State{}, store.OriginDeclared, unitIn(false, &commands), nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if changing := touched(commands); len(changing) > 0 {
|
||||||
|
t.Errorf("%s: an inactive unit was acted on: %v", triggers, changing)
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, "uplink.manager"); o.Action != "unchanged" ||
|
||||||
|
o.Detail != "not running; the change applies at its next start" {
|
||||||
|
t.Errorf("%s: reported as %q: %s", triggers, o.Action, o.Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStatelessServiceIsRestartedOnlyForItsRestartTrigger(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "mesh.conf")
|
||||||
|
var commands []string
|
||||||
|
d := parse(t, statelessDecl(path, "x\n", `"restart-on":["uplink.conf"]`))
|
||||||
|
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
|
||||||
|
unitIn(true, &commands), nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(strings.Join(commands, "\n"), "stop NetworkManager.service") {
|
||||||
|
t.Errorf("not restarted for its restart trigger; commands were %v", commands)
|
||||||
|
}
|
||||||
|
// Nothing it reflects changed: nothing is asked of the unit at all.
|
||||||
|
commands = nil
|
||||||
|
report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared,
|
||||||
|
unitIn(true, &commands), nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if changing := touched(commands); len(changing) > 0 {
|
||||||
|
t.Errorf("with nothing changed the unit was acted on: %v", changing)
|
||||||
|
}
|
||||||
|
if got := outcomeOf(report, "uplink.manager").Action; got != "unchanged" {
|
||||||
|
t.Errorf("with nothing changed it was %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStatelessServiceUndeclaredIsForgottenNotStopped(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "mesh.conf")
|
||||||
|
var commands []string
|
||||||
|
_, state, err := Apply(context.Background(), archHost(t),
|
||||||
|
parse(t, statelessDecl(path, "x\n", `"reload-on":["uplink.conf"]`)),
|
||||||
|
store.State{}, store.OriginDeclared, unitIn(true, &commands), nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rec, _ := state.Find("uplink.manager"); !rec.Stateless {
|
||||||
|
t.Fatal("the record does not say the service was stateless")
|
||||||
|
}
|
||||||
|
if steps := Plan(somethingElse(t), state, store.OriginDeclared); !hasStep(steps, "forget", "uplink.manager") {
|
||||||
|
t.Errorf("the preview does not forget it: %v", steps)
|
||||||
|
}
|
||||||
|
commands = nil
|
||||||
|
report, _, err := Apply(context.Background(), archHost(t), somethingElse(t), state, store.OriginDeclared,
|
||||||
|
unitIn(true, &commands), nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if changing := touched(commands); len(changing) > 0 {
|
||||||
|
t.Errorf("undeclaring acted on the unit: %v", changing)
|
||||||
|
}
|
||||||
|
o := outcomeOf(report, "uplink.manager")
|
||||||
|
if o.Action != "forgotten" || o.Detail != "its state was never the mesh's" {
|
||||||
|
t.Errorf("undeclaring was %q: %s", o.Action, o.Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStatelessServiceIsNeverHeldOnAnAdoptedNode(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "mesh.conf")
|
||||||
|
resources := fmt.Sprintf(`{"id":"uplink.conf","type":"file","path":%q,"into":"block","content":"x\n"},
|
||||||
|
{"id":"uplink.manager","type":"service","unit":"NetworkManager.service","reload-on":["uplink.conf"]}`, path)
|
||||||
|
d := adopted(t, `{"taken":[],"untaken":{"uplink":["uplink.conf","uplink.manager"]}}`, resources)
|
||||||
|
for _, s := range Plan(d, store.State{}, store.OriginDeclared) {
|
||||||
|
if s.ID == "uplink.manager" && (s.Verb == "hold" || s.Verb == "create" || strings.Contains(s.Why, "replaces")) {
|
||||||
|
t.Errorf("the preview holds or replaces a stateless service: %+v", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var commands []string
|
||||||
|
report, state, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{},
|
||||||
|
store.OriginDeclared, unitIn(true, &commands), nil, nil, KeepIn(t.TempDir()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := outcomeOf(report, "uplink.manager").Action; got == "held" {
|
||||||
|
t.Fatal("a stateless service was held, though it replaces nothing that was found")
|
||||||
|
}
|
||||||
|
if len(state.Held) != 0 {
|
||||||
|
t.Errorf("something was held: %+v", state.Held)
|
||||||
|
}
|
||||||
|
for _, c := range touched(commands) {
|
||||||
|
if !strings.Contains(c, "reload") {
|
||||||
|
t.Errorf("the adopted node's manager lifecycle was touched: %s", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasStep(steps []Step, verb, id string) bool {
|
||||||
|
for _, s := range steps {
|
||||||
|
if s.Verb == verb && s.ID == id {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,579 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
"github.com/novox/mesh-host/internal/system"
|
||||||
|
"github.com/novox/mesh-host/internal/tunnel"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The private network takes over the tunnel it found (novox/hq ADR 0105).
|
||||||
|
//
|
||||||
|
// The controller says so on the interface's service: `takes-over` names the found interface, the
|
||||||
|
// unit that raised it and its configuration file. Before the mesh's unit is started, the host keeps
|
||||||
|
// that file like any held file — the original recorded before anything else happens to it — and
|
||||||
|
// stops and disables the found unit. Never a flush: `wg set … peer … remove` is never run, the
|
||||||
|
// file is never written, and the found interface goes down the way its own unit takes it down.
|
||||||
|
// Then the mesh's interface comes up, with the found key the node took at enrolment, on the found
|
||||||
|
// port, with the found peers in its list — and a peer of the tunnel cannot tell it changed hands.
|
||||||
|
//
|
||||||
|
// Every apply, not once: a found unit somebody starts again would take the port back from the
|
||||||
|
// mesh's interface, so it is stopped again and said so. That is the one place an adopted node
|
||||||
|
// undoes something done by hand, and it is because the tunnel is the mesh's now.
|
||||||
|
//
|
||||||
|
// **Until the take is proven, and then the found configuration is retired** (novox/hq ADR 0119).
|
||||||
|
// Keeping it on disk was the caution the take needed: if the mesh's interface does not come up,
|
||||||
|
// the found unit is started again and the peers never notice. That caution is spent once the
|
||||||
|
// tunnel is taken — the found unit down and disabled, the mesh's interface up — and a peer has
|
||||||
|
// handshaken with the mesh's interface. From then on a configuration nothing maintains, one
|
||||||
|
// command away from raising a second way onto the network, is not a rollback path but a door
|
||||||
|
// nobody watches. So it is removed from where its unit reads it; its original, kept before
|
||||||
|
// anything happened to it (ADR 0100), stays kept; and the hold on it ends. A take never proven
|
||||||
|
// keeps it, and says so — a broken take is visible, not silently retired.
|
||||||
|
|
||||||
|
// TakenTunnel is what an apply says about a tunnel it took over, for the node's report.
|
||||||
|
type TakenTunnel struct {
|
||||||
|
Interface string
|
||||||
|
Port int
|
||||||
|
Range string
|
||||||
|
Peers int
|
||||||
|
// State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one
|
||||||
|
// down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's
|
||||||
|
// not up: the peers reach nothing). Note is what this apply did about it — and, for a taken
|
||||||
|
// tunnel, whether the take is proven and its found configuration retired (novox/hq ADR 0119).
|
||||||
|
// Kept is where the found configuration's original is, retired or not.
|
||||||
|
State string
|
||||||
|
Note string
|
||||||
|
Kept string
|
||||||
|
}
|
||||||
|
|
||||||
|
// The states, as the link says them.
|
||||||
|
const (
|
||||||
|
NotTaken = "not-taken"
|
||||||
|
Taken = "taken"
|
||||||
|
TunnelDown = "down"
|
||||||
|
)
|
||||||
|
|
||||||
|
// takeoverRecheck is how often, and takeoverRechecks how many times, a found interface still up
|
||||||
|
// after its unit stopped is looked at again before the takeover is refused: `wg-quick down` by a
|
||||||
|
// person takes a moment. Variables so a test need not wait.
|
||||||
|
var (
|
||||||
|
takeoverRecheck = 2 * time.Second
|
||||||
|
takeoverRechecks = 3
|
||||||
|
)
|
||||||
|
|
||||||
|
// takeOverID is the held record's id for the found configuration: the service's own with a suffix,
|
||||||
|
// so it is declared for as long as the service is and never mistaken for the service itself.
|
||||||
|
func takeOverID(svc *declaration.Service) string { return svc.ID + ".takes-over" }
|
||||||
|
|
||||||
|
// takeOver keeps the found tunnel's configuration and stops its unit, ahead of the service that
|
||||||
|
// replaces it. Returned is the hold's outcome, and what was found for the report.
|
||||||
|
//
|
||||||
|
// **Nothing is stopped until the mesh's interface is known to be able to replace it** (the record's
|
||||||
|
// option 2 is exactly this going wrong): the declared configuration must listen on the found port
|
||||||
|
// at the found address, and the key file it points at must hold the found key. Only then is the
|
||||||
|
// found unit stopped — and `stopped` says whether this apply did, so a mesh interface that then
|
||||||
|
// fails to start can have the found unit started again.
|
||||||
|
func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, d *declaration.Declaration,
|
||||||
|
known *store.State, run Runner, keep Keep, now time.Time) (out Outcome, facts TakenTunnel, stopped bool, err error) {
|
||||||
|
t := svc.TakesOver
|
||||||
|
id := takeOverID(svc)
|
||||||
|
module, _ := d.Adoption.UntakenModuleOf(svc.ID)
|
||||||
|
if module == "" {
|
||||||
|
module = "the private network"
|
||||||
|
}
|
||||||
|
facts = TakenTunnel{Interface: t.Interface, State: NotTaken}
|
||||||
|
|
||||||
|
// 0. What the found configuration says, before anything: the checks below are against it.
|
||||||
|
found, ferr := readFoundTunnel(t.Config)
|
||||||
|
|
||||||
|
// 1. The configuration, kept like any held file. A synthetic file resource stands for it, so
|
||||||
|
// the same code keeps its original, digests it and notices it changing.
|
||||||
|
//
|
||||||
|
// **Unless it was retired** (novox/hq ADR 0119): the take was proven and the mesh removed
|
||||||
|
// it, so there is nothing to hold and nothing missing — only where its original is, which
|
||||||
|
// the retirement recorded. A hold still standing is let go: that is what retiring it meant.
|
||||||
|
//
|
||||||
|
// **One that comes back is held again on its FIRST original** — the one kept before anything
|
||||||
|
// happened to it (ADR 0100), never whatever was put back — and retired again by the first
|
||||||
|
// apply that finds the take still proven, keeping what came back only if it differs from
|
||||||
|
// what is already kept. Put back by hand while the private network is assigned, it is not a
|
||||||
|
// rollback: that means unassigning the private network first, and the note says so.
|
||||||
|
file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config}
|
||||||
|
var held store.Held
|
||||||
|
retired, wasRetired := known.RetiredAt(t.Config)
|
||||||
|
cameBack := wasRetired && present(t.Config)
|
||||||
|
switch {
|
||||||
|
case wasRetired && !cameBack:
|
||||||
|
known.Release(id)
|
||||||
|
held = store.Held{Kept: retired.Kept}
|
||||||
|
out = begin(file)
|
||||||
|
out.Action = "unchanged"
|
||||||
|
facts.Note = "the found configuration " + t.Config + " was retired once the take was proven; " +
|
||||||
|
"its original is kept at " + retired.Kept + " and the mesh never brings it back"
|
||||||
|
default:
|
||||||
|
was, already := known.HeldAt(id)
|
||||||
|
why := "the configuration of the tunnel " + t.Interface + ", taken over by " + svc.Unit
|
||||||
|
if cameBack && !already {
|
||||||
|
digest := retired.Digest
|
||||||
|
if digest == "" {
|
||||||
|
if raw, err := os.ReadFile(retired.Kept); err == nil {
|
||||||
|
digest = digestOf(string(raw))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
was = store.Held{ID: id, Module: module, Kind: string(declaration.TypeFile), Target: t.Config,
|
||||||
|
Since: now, Why: why, Kept: retired.Kept, Digest: digest}
|
||||||
|
already = true
|
||||||
|
}
|
||||||
|
out, held, err = hold(ctx, sys, file, module, was, already, why, run, keep, now)
|
||||||
|
if err != nil {
|
||||||
|
return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err)
|
||||||
|
}
|
||||||
|
known.RecordHeld(held)
|
||||||
|
if cameBack {
|
||||||
|
facts.Note = "the found configuration " + t.Config + " came back after it was retired; while the " +
|
||||||
|
"private network is assigned the mesh retires it again, so rolling back to the found tunnel " +
|
||||||
|
"means unassigning the private network first"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
facts.Kept = held.Kept
|
||||||
|
// What the file says, for the report: from the machine, or from the kept original when the
|
||||||
|
// machine's copy is gone. The private key stays in the file; nothing here keeps it.
|
||||||
|
unread := ""
|
||||||
|
if ferr != nil && held.Kept != "" {
|
||||||
|
found, ferr = readFoundTunnel(held.Kept)
|
||||||
|
}
|
||||||
|
if ferr == nil {
|
||||||
|
facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers)
|
||||||
|
} else {
|
||||||
|
unread = ferr.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Where things stand: the found unit, and the mesh's.
|
||||||
|
foundState, unitErr := sys.ServiceState(ctx, run, t.Unit)
|
||||||
|
meshState, _ := sys.ServiceState(ctx, run, svc.Unit)
|
||||||
|
if foundState == "running" && meshState == "running" {
|
||||||
|
// Both up. On the hub this cannot last — the found unit cannot bind the port the mesh's
|
||||||
|
// holds — and on a spoke two interfaces with one key flap between them. Not stopped again
|
||||||
|
// by the mesh: what is found on an adopted node is reported, and the first takeover was
|
||||||
|
// the one act (the PR note says why). Said, so a person sees it.
|
||||||
|
facts.Note = t.Unit + " is running again beside the mesh's interface; not stopped by the mesh — " +
|
||||||
|
"`systemctl stop " + t.Unit + "` on the machine"
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Before the found unit is stopped: can the mesh's interface replace it? Its declared
|
||||||
|
// configuration must listen on the found port at the found address, and the key file it
|
||||||
|
// points at must hold the found key, or the peers would be dropped the moment it came up.
|
||||||
|
if foundState == "running" && meshState != "running" {
|
||||||
|
if ferr != nil {
|
||||||
|
return out, facts, false, fmt.Errorf("the found tunnel's configuration at %s cannot be read as a "+
|
||||||
|
"tunnel's (%v), so nothing says what the mesh's interface must match; %s is left running",
|
||||||
|
t.Config, ferr, t.Unit)
|
||||||
|
}
|
||||||
|
if err := replaces(d, svc, found); err != nil {
|
||||||
|
return out, facts, false, fmt.Errorf("%w; %s is left running", err, t.Unit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. The found unit: stopped if it runs and the mesh's does not, disabled if it starts at
|
||||||
|
// boot. A unit that is not there is not an error — the interface may have been raised
|
||||||
|
// another way, which the check below catches — and neither is one already down.
|
||||||
|
var did []string
|
||||||
|
switch {
|
||||||
|
case unitErr != nil:
|
||||||
|
did = append(did, t.Unit+" is not a unit here")
|
||||||
|
case foundState == "running" && meshState != "running":
|
||||||
|
if err := sys.SetServiceState(ctx, run, t.Unit, "stopped"); err != nil {
|
||||||
|
return out, facts, false, fmt.Errorf("stopping the found %s: %w", t.Unit, err)
|
||||||
|
}
|
||||||
|
after, err := sys.ServiceState(ctx, run, t.Unit)
|
||||||
|
if err != nil {
|
||||||
|
return out, facts, true, err
|
||||||
|
}
|
||||||
|
if after != "stopped" {
|
||||||
|
return out, facts, true, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after)
|
||||||
|
}
|
||||||
|
stopped = true
|
||||||
|
did = append(did, "stopped "+t.Unit)
|
||||||
|
}
|
||||||
|
if unitErr == nil {
|
||||||
|
if boot, err := sys.ServiceBoot(ctx, run, t.Unit); err == nil && boot == "enabled" {
|
||||||
|
if err := sys.SetServiceBoot(ctx, run, t.Unit, "disabled"); err != nil {
|
||||||
|
return out, facts, stopped, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err)
|
||||||
|
}
|
||||||
|
did = append(did, "disabled it at boot")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. The interface is gone. If it is still up, something other than its unit raised it —
|
||||||
|
// the predecessor brings its up by hand — and the mesh's interface cannot take its port
|
||||||
|
// and address while it is. Looked at again for a moment, since a person taking it down
|
||||||
|
// takes a moment; then refused, naming what to do.
|
||||||
|
if meshState != "running" {
|
||||||
|
for try := 0; ; try++ {
|
||||||
|
if !interfaceUp(ctx, run, t.Interface) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if try >= takeoverRechecks {
|
||||||
|
return out, facts, stopped, fmt.Errorf("%s is still up although its unit %s is not running: it was "+
|
||||||
|
"raised by hand, not by its unit, and the mesh's interface cannot take its port and "+
|
||||||
|
"address while it is. On the machine: `wg-quick down %s` — the next reconcile takes it "+
|
||||||
|
"over. Nothing was flushed", t.Interface, t.Unit, t.Interface)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return out, facts, stopped, ctx.Err()
|
||||||
|
case <-time.After(takeoverRecheck):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found"
|
||||||
|
switch {
|
||||||
|
case cameBack:
|
||||||
|
out.Detail = "the tunnel " + t.Interface + "'s configuration, back after it was retired; held until " +
|
||||||
|
"it is retired again"
|
||||||
|
case wasRetired:
|
||||||
|
out.Detail = "the tunnel " + t.Interface + "'s configuration, retired once the take was proven"
|
||||||
|
}
|
||||||
|
if held.Kept != "" {
|
||||||
|
out.Detail += " (original at " + held.Kept + ")"
|
||||||
|
}
|
||||||
|
if len(did) > 0 {
|
||||||
|
out.Detail += "; " + strings.Join(did, ", ") + " — never flushed"
|
||||||
|
}
|
||||||
|
if held.Changed != "" {
|
||||||
|
out.Detail += "; " + held.Changed + " by something other than the mesh since it was found"
|
||||||
|
}
|
||||||
|
if unread != "" {
|
||||||
|
// Said, not swallowed: the report would otherwise say a tunnel with no port and no
|
||||||
|
// peers was carried, which reads as a tunnel that was not one.
|
||||||
|
out.Detail += "; what it says could not be read as a tunnel's: " + unread
|
||||||
|
}
|
||||||
|
return out, facts, stopped, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readFoundTunnel is the found configuration as a tunnel.
|
||||||
|
func readFoundTunnel(path string) (tunnel.Found, error) {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return tunnel.Found{}, err
|
||||||
|
}
|
||||||
|
return tunnel.Parse(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// interfaceUp is whether a WireGuard interface is up on the machine.
|
||||||
|
func interfaceUp(ctx context.Context, run Runner, iface string) bool {
|
||||||
|
up, err := run(ctx, "wg", "show", "interfaces")
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, name := range strings.Fields(up) {
|
||||||
|
if name == iface {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// replaces holds the mesh's declared interface configuration against the found tunnel it is to
|
||||||
|
// replace: same port, same address, and a key file holding the found key. The configuration is
|
||||||
|
// the file the service restarts on; its `PostUp = wg set %i private-key <path>` names the key.
|
||||||
|
func replaces(d *declaration.Declaration, svc *declaration.Service, found tunnel.Found) error {
|
||||||
|
var conf *declaration.File
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
f, ok := r.(*declaration.File)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, id := range svc.RestartOn {
|
||||||
|
if f.ID == id {
|
||||||
|
conf = f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if conf == nil {
|
||||||
|
return fmt.Errorf("%s takes over %s and restarts on no declared file, so the interface it would "+
|
||||||
|
"raise cannot be checked against the found one", svc.Unit, found.Interface)
|
||||||
|
}
|
||||||
|
port, address, keyPath := "", "", ""
|
||||||
|
for _, line := range strings.Split(conf.Content, "\n") {
|
||||||
|
key, value, ok := strings.Cut(strings.TrimSpace(line), "=")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value)
|
||||||
|
switch key {
|
||||||
|
case "listenport":
|
||||||
|
port = value
|
||||||
|
case "address":
|
||||||
|
address = strings.TrimSpace(strings.Split(value, ",")[0])
|
||||||
|
case "postup":
|
||||||
|
if _, after, ok := strings.Cut(value, "private-key "); ok {
|
||||||
|
keyPath = strings.Fields(after)[0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var wrong []string
|
||||||
|
if port != fmt.Sprint(found.Port) {
|
||||||
|
wrong = append(wrong, fmt.Sprintf("it listens on port %q and the tunnel on %d", port, found.Port))
|
||||||
|
}
|
||||||
|
if host(address) != host(found.Address) {
|
||||||
|
wrong = append(wrong, fmt.Sprintf("its address is %q and the tunnel's %s", address, found.Address))
|
||||||
|
}
|
||||||
|
switch raw, err := os.ReadFile(keyPath); {
|
||||||
|
case keyPath == "":
|
||||||
|
wrong = append(wrong, "it names no key file")
|
||||||
|
case err != nil:
|
||||||
|
wrong = append(wrong, fmt.Sprintf("its key file %s cannot be read (%v)", keyPath, err))
|
||||||
|
default:
|
||||||
|
public, perr := tunnel.PublicKeyOf(strings.TrimSpace(string(raw)))
|
||||||
|
if perr != nil || public != found.PublicKey {
|
||||||
|
wrong = append(wrong, fmt.Sprintf("the key at %s is not the tunnel's — `mesh-host overlay take "+
|
||||||
|
"--tunnel %s` on this machine takes it, then push again", keyPath, found.Interface))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(wrong) > 0 {
|
||||||
|
return fmt.Errorf("the mesh's interface would not replace the tunnel on %s: %s — the peers would be "+
|
||||||
|
"dropped the moment it came up. Re-place the hub on the tunnel's address and port and push again",
|
||||||
|
found.Interface, strings.Join(wrong, "; "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// host is an address without its prefix length.
|
||||||
|
func host(address string) string {
|
||||||
|
if i := strings.Index(address, "/"); i >= 0 {
|
||||||
|
return address[:i]
|
||||||
|
}
|
||||||
|
return address
|
||||||
|
}
|
||||||
|
|
||||||
|
// tunnelState is where the tunnel stands, read from the machine: the found unit or interface up
|
||||||
|
// and the mesh's not is not taken; the mesh's up and the found one down is taken; neither up is
|
||||||
|
// down — the peers reach nothing.
|
||||||
|
func tunnelState(ctx context.Context, sys system.System, foundUnit, meshUnit string, run Runner) string {
|
||||||
|
foundState, _ := sys.ServiceState(ctx, run, foundUnit)
|
||||||
|
meshState, _ := sys.ServiceState(ctx, run, meshUnit)
|
||||||
|
foundUp := foundState == "running" || interfaceUp(ctx, run, strings.TrimPrefix(foundUnit, "wg-quick@"))
|
||||||
|
switch {
|
||||||
|
case meshState == "running" && !foundUp:
|
||||||
|
return Taken
|
||||||
|
case meshState == "running":
|
||||||
|
// Both up: not a takeover that holds, and said as not taken so nobody reads it as one.
|
||||||
|
return NotTaken
|
||||||
|
case foundUp:
|
||||||
|
return NotTaken
|
||||||
|
default:
|
||||||
|
return TunnelDown
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// restoreFound starts the found unit again after the mesh's interface failed to replace it, so the
|
||||||
|
// machine has the tunnel it had rather than none, and says so in the account.
|
||||||
|
func restoreFound(ctx context.Context, sys system.System, unit string, run Runner, facts *TakenTunnel) {
|
||||||
|
if err := sys.SetServiceState(ctx, run, unit, "running"); err != nil {
|
||||||
|
facts.State = TunnelDown
|
||||||
|
facts.Note += "; " + unit + " could not be started again (" + err.Error() + ") — on the machine: systemctl start " + unit
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if state, err := sys.ServiceState(ctx, run, unit); err != nil || state != "running" {
|
||||||
|
facts.State = TunnelDown
|
||||||
|
facts.Note += "; " + unit + " was started again and is not running — on the machine: systemctl start " + unit
|
||||||
|
return
|
||||||
|
}
|
||||||
|
facts.State = NotTaken
|
||||||
|
facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had"
|
||||||
|
}
|
||||||
|
|
||||||
|
// wireguardDir is where a found tunnel's configuration may be retired from: wg-quick's own, and
|
||||||
|
// nowhere else. A variable so a test can hand in a directory.
|
||||||
|
var wireguardDir = tunnel.ConfigDir
|
||||||
|
|
||||||
|
// retireFound removes the found tunnel's configuration from where its unit reads it, once the take
|
||||||
|
// is proven, and ends the hold on it (novox/hq ADR 0119). Asked after the mesh's service applied
|
||||||
|
// and the tunnel reads as taken; retired says whether this apply retired it, and out is then what
|
||||||
|
// replaces the take's outcome for the configuration.
|
||||||
|
//
|
||||||
|
// **Proven is taken and a handshake.** Taken alone — the found unit down and disabled, the mesh's
|
||||||
|
// interface up — says the mesh's interface exists, not that any peer reaches it: an interface up
|
||||||
|
// with the wrong key is taken and carries nothing. A peer that has completed a handshake with it
|
||||||
|
// has checked its key, so that is the proof, asked of the kernel through `wg`. Any handshake counts,
|
||||||
|
// however old: a change to the mesh's configuration restarts its unit, which recreates the
|
||||||
|
// interface and resets its counters, so a time that is there at all was made by this interface.
|
||||||
|
// Anything short of one — no peer yet, every time zero, `wg` missing or failing — keeps the file,
|
||||||
|
// and the account says which: a take that never proves itself is visible rather than silently
|
||||||
|
// retired.
|
||||||
|
//
|
||||||
|
// **Only what the take names, and only wg-quick's own file.** Nothing is removed unless the path
|
||||||
|
// is exactly `<wireguard dir>/<found interface>.conf`, is not a path the mesh itself writes, and is
|
||||||
|
// a file rather than a link: removing a link would leave the key-bearing file it points at where it
|
||||||
|
// is, a retirement in name only, so that one is said and left to a person.
|
||||||
|
//
|
||||||
|
// **The original must still be kept.** It is the record of what the predecessor was and a
|
||||||
|
// person's only way back (ADR 0100); a kept copy that has gone missing is said, and the file is
|
||||||
|
// not removed, since removing it then would lose the only copy. What is on disk now, if it differs
|
||||||
|
// from the first original and from what was kept at the last retirement, is kept too before it
|
||||||
|
// goes — by content, so the first original is never overwritten and a file that keeps coming back
|
||||||
|
// the same keeps nothing more.
|
||||||
|
//
|
||||||
|
// The found unit is left disabled; without its configuration it cannot raise the interface, so
|
||||||
|
// every later apply's check of it finds nothing to do. Nothing here ever writes the file back.
|
||||||
|
func retireFound(ctx context.Context, svc *declaration.Service, d *declaration.Declaration, known *store.State,
|
||||||
|
run Runner, keep Keep, facts *TakenTunnel, now time.Time) (out Outcome, retired bool) {
|
||||||
|
t := svc.TakesOver
|
||||||
|
id := takeOverID(svc)
|
||||||
|
if facts.State != Taken {
|
||||||
|
return out, false
|
||||||
|
}
|
||||||
|
held, isHeld := known.HeldAt(id)
|
||||||
|
if !isHeld {
|
||||||
|
// Retired already (takeOver let any hold go and said so), or never held: nothing to do.
|
||||||
|
return out, false
|
||||||
|
}
|
||||||
|
say := func(note string) {
|
||||||
|
if facts.Note != "" {
|
||||||
|
facts.Note += "; "
|
||||||
|
}
|
||||||
|
facts.Note += note
|
||||||
|
}
|
||||||
|
notRetired := func(why string) (Outcome, bool) {
|
||||||
|
say("the found configuration " + t.Config + " is not retired: " + why)
|
||||||
|
return Outcome{}, false
|
||||||
|
}
|
||||||
|
mesh := strings.TrimPrefix(svc.Unit, "wg-quick@")
|
||||||
|
peers, err := tunnel.Handshaken(ctx, tunnel.Runner(run), mesh)
|
||||||
|
if err != nil {
|
||||||
|
say("taken, not yet proven: " + err.Error() + "; the found configuration " + t.Config + " is kept")
|
||||||
|
return out, false
|
||||||
|
}
|
||||||
|
if peers == 0 {
|
||||||
|
say("taken, not yet proven: no peer has handshaken on " + mesh + "; the found configuration " +
|
||||||
|
t.Config + " is kept")
|
||||||
|
return out, false
|
||||||
|
}
|
||||||
|
proven := fmt.Sprintf("proven: %d peer(s) handshaken on %s", peers, mesh)
|
||||||
|
say(proven)
|
||||||
|
|
||||||
|
// What may be removed at all.
|
||||||
|
if want := filepath.Join(wireguardDir, t.Interface+".conf"); t.Config != want {
|
||||||
|
return notRetired("only " + want + ", the found interface's own wg-quick configuration, is ever " +
|
||||||
|
"retired by the mesh, and the take names " + t.Config)
|
||||||
|
}
|
||||||
|
if known.Recorded(string(declaration.TypeFile), t.Config) || declaresFile(d, t.Config) {
|
||||||
|
return notRetired("it is a path the mesh itself writes")
|
||||||
|
}
|
||||||
|
if info, err := os.Lstat(t.Config); err == nil && info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
target, _ := os.Readlink(t.Config)
|
||||||
|
return notRetired("it is a link to " + target + "; removing the link would leave the key-bearing file " +
|
||||||
|
"it points at, so it must be retired by hand — both are kept")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The kept original, read back — not just named in a record.
|
||||||
|
if held.Kept == "" {
|
||||||
|
return notRetired("no original of it was kept, so removing it would leave no record of what the " +
|
||||||
|
"predecessor was")
|
||||||
|
}
|
||||||
|
original, err := os.ReadFile(held.Kept)
|
||||||
|
if err != nil || (held.Digest != "" && digestOf(string(original)) != held.Digest) {
|
||||||
|
why := "is missing"
|
||||||
|
if err == nil {
|
||||||
|
why = "no longer holds what was found"
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
why = "cannot be read (" + err.Error() + ")"
|
||||||
|
}
|
||||||
|
return notRetired("its kept original " + held.Kept + " " + why + ", so removing it would lose the only copy")
|
||||||
|
}
|
||||||
|
|
||||||
|
before, cameBack := known.RetiredAt(t.Config)
|
||||||
|
record := store.Retired{ID: id, Path: t.Config, Kept: held.Kept, Digest: digestOf(string(original)), At: now}
|
||||||
|
if cameBack {
|
||||||
|
// The first original stays the record's, and so does what the last retirement kept.
|
||||||
|
record.Extra, record.ExtraDigest, record.Again = before.Extra, before.ExtraDigest, before.Again+1
|
||||||
|
}
|
||||||
|
newCopy := ""
|
||||||
|
gone := !present(t.Config)
|
||||||
|
if !gone {
|
||||||
|
current, err := os.ReadFile(t.Config)
|
||||||
|
if err != nil {
|
||||||
|
return notRetired("it cannot be read (" + err.Error() + ")")
|
||||||
|
}
|
||||||
|
if sum := digestOf(string(current)); sum != record.Digest && sum != record.ExtraDigest {
|
||||||
|
if keep == nil {
|
||||||
|
return notRetired("it holds something other than its kept original and this host has nowhere " +
|
||||||
|
"to keep it")
|
||||||
|
}
|
||||||
|
where, err := keep(t.Config, current, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
return notRetired("keeping what it holds now failed (" + err.Error() + ")")
|
||||||
|
}
|
||||||
|
record.Extra, record.ExtraDigest, newCopy = where, sum, where
|
||||||
|
}
|
||||||
|
if err := os.Remove(t.Config); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return notRetired("removing it failed (" + err.Error() + ")")
|
||||||
|
}
|
||||||
|
if present(t.Config) {
|
||||||
|
return notRetired("it is still there after it was removed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
known.RecordRetired(record)
|
||||||
|
known.Release(id)
|
||||||
|
facts.Kept = held.Kept
|
||||||
|
copied := ""
|
||||||
|
if newCopy != "" {
|
||||||
|
copied = "; what it held, which differed from the original, is kept at " + newCopy
|
||||||
|
}
|
||||||
|
out = Outcome{ID: id, Type: string(declaration.TypeFile), Target: t.Config, Action: "removed"}
|
||||||
|
switch {
|
||||||
|
case cameBack:
|
||||||
|
say("the found configuration came back and was retired again — its original still kept at " +
|
||||||
|
held.Kept + copied + "; rolling back to the found tunnel means unassigning the private network first")
|
||||||
|
out.Detail = "the found configuration came back and was retired again; original kept at " + held.Kept + copied
|
||||||
|
default:
|
||||||
|
say("the found configuration " + t.Config + " is retired — its original kept at " + held.Kept + copied +
|
||||||
|
", " + t.Unit + " left disabled, and the mesh never brings it back")
|
||||||
|
out.Detail = "retired: the take of " + t.Interface + " is " + proven + "; original kept at " + held.Kept + copied
|
||||||
|
}
|
||||||
|
if gone {
|
||||||
|
// Already gone — removed by something other than the mesh, or by an apply whose record was
|
||||||
|
// never saved. Nothing removed here; the hold ends all the same.
|
||||||
|
out.Action = "unchanged"
|
||||||
|
out.Detail = "retired: the take of " + t.Interface + " is " + proven + " and " + t.Config +
|
||||||
|
" was already gone; original kept at " + held.Kept
|
||||||
|
}
|
||||||
|
return out, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// declaresFile is whether a declaration writes a file at a path.
|
||||||
|
func declaresFile(d *declaration.Declaration, path string) bool {
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
if f, ok := r.(*declaration.File); ok && filepath.Clean(f.Path) == filepath.Clean(path) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since
|
||||||
|
// a machine has one private network.
|
||||||
|
func takesOver(d *declaration.Declaration) *declaration.Service {
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
if svc, ok := r.(*declaration.Service); ok && svc.TakesOver != nil {
|
||||||
|
return svc
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// errNotAdopted is a takeover on a declaration that does not say the node is adopted, which the
|
||||||
|
// parser refuses already; kept as a second line of defence at the point of acting.
|
||||||
|
var errNotAdopted = errors.New("a tunnel is taken over on an adopted node only")
|
||||||
@@ -0,0 +1,715 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/ecdh"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0105: the host raises the mesh's interface with the found key and peers, stops the
|
||||||
|
// found interface without flushing it, and keeps its configuration — and stops nothing until the
|
||||||
|
// mesh's interface is known to be able to replace it.
|
||||||
|
|
||||||
|
// foundKey is the predecessor's private key, a real one made once per run: the key is what the
|
||||||
|
// takeover must never print or copy, so it had better be one.
|
||||||
|
var foundKey = func() string {
|
||||||
|
k, err := ecdh.X25519().GenerateKey(rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return base64.StdEncoding.EncodeToString(k.Bytes())
|
||||||
|
}()
|
||||||
|
|
||||||
|
var foundConf = "[Interface]\nPrivateKey = " + foundKey + "\n" +
|
||||||
|
"ListenPort = 51900\nAddress = 192.0.2.1/24\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n" +
|
||||||
|
"\n[Peer]\nPublicKey = PEER-B=\nAllowedIPs = 192.0.2.3/32\n"
|
||||||
|
|
||||||
|
// aTakeover is the private network's declaration for an adopted hub whose interface takes over
|
||||||
|
// the found tunnel: the mesh's configuration — on the found port and address, its key set from the
|
||||||
|
// node's own key file, the found peers in its list — and the interface's service naming what it
|
||||||
|
// replaces. Port and address are parameters so a test can declare a wrong one.
|
||||||
|
func aTakeover(t *testing.T, config, mesh, keyFile, port, address string) *declaration.Declaration {
|
||||||
|
t.Helper()
|
||||||
|
return adopted(t,
|
||||||
|
`{"taken":[],"untaken":{"mesh-wireguard":["mesh-wireguard.overlay-config","mesh-wireguard.overlay-up"]}}`,
|
||||||
|
`{"id":"mesh-wireguard.overlay-config","type":"file","path":"`+mesh+`","mode":"0600",
|
||||||
|
"content":"[Interface]\nAddress = `+address+`/32\nListenPort = `+port+`\nPostUp = wg set %i private-key `+keyFile+`\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"},
|
||||||
|
{"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0","state":"running","boot":"enabled",
|
||||||
|
"restart-on":["mesh-wireguard.overlay-config"],
|
||||||
|
"takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"`+config+`"}}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// aHubInUse is a machine with the predecessor's tunnel up and the mesh's not yet: the found
|
||||||
|
// configuration on disk, and the node's key file holding the found key, as enrolment left it.
|
||||||
|
func aHubInUse(t *testing.T) (dir, config, mesh, keyFile string, m *machine) {
|
||||||
|
t.Helper()
|
||||||
|
dir = t.TempDir()
|
||||||
|
config = filepath.Join(dir, "wg0.conf")
|
||||||
|
mesh = filepath.Join(dir, "mesh0.conf")
|
||||||
|
keyFile = filepath.Join(dir, "overlay.key")
|
||||||
|
if err := os.WriteFile(config, []byte(foundConf), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(keyFile, []byte(foundKey+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m = &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{
|
||||||
|
"wg-quick@wg0": {active: "active", enabled: "enabled"},
|
||||||
|
"wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"},
|
||||||
|
}}
|
||||||
|
takeoverRecheck = 0
|
||||||
|
// The found configuration lives in this test's own wireguard directory (novox/hq ADR 0119).
|
||||||
|
was := wireguardDir
|
||||||
|
wireguardDir = dir
|
||||||
|
t.Cleanup(func() { wireguardDir = was })
|
||||||
|
return dir, config, mesh, keyFile, m
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)
|
||||||
|
|
||||||
|
// The found interface: its unit stopped and disabled, and nothing else done to it.
|
||||||
|
if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" {
|
||||||
|
t.Fatalf("the found unit was not stopped and disabled: %+v", u)
|
||||||
|
}
|
||||||
|
for _, asked := range m.asked {
|
||||||
|
// Only ever asked about: which interfaces are up, and whether a peer has handshaken with
|
||||||
|
// the mesh's own (novox/hq ADR 0119).
|
||||||
|
if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") &&
|
||||||
|
asked != "wg show mesh0 latest-handshakes" {
|
||||||
|
t.Errorf("the found interface was touched with %q; it is stopped, never flushed", asked)
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(asked, "wg-quick") || strings.Contains(asked, "peer remove") {
|
||||||
|
t.Errorf("the found interface was flushed: %q", asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Its configuration: on disk as it was, its original kept, held for the module.
|
||||||
|
if got, _ := os.ReadFile(config); string(got) != foundConf {
|
||||||
|
t.Fatalf("the found configuration was changed:\n%s", got)
|
||||||
|
}
|
||||||
|
held, ok := state.HeldAt("mesh-wireguard.overlay-up.takes-over")
|
||||||
|
if !ok || held.Kind != "file" || held.Target != config || held.Kept == "" || held.Module != "mesh-wireguard" {
|
||||||
|
t.Fatalf("the found configuration is not held: %+v", held)
|
||||||
|
}
|
||||||
|
if kept, _ := os.ReadFile(held.Kept); string(kept) != foundConf {
|
||||||
|
t.Fatalf("the original was not kept as found: %q", kept)
|
||||||
|
}
|
||||||
|
// The mesh's interface: up, enabled, with the found peers in the file the mesh wrote.
|
||||||
|
if u := m.units["wg-quick@mesh0"]; u.active != "active" || u.enabled != "enabled" {
|
||||||
|
t.Fatalf("the mesh's interface was not raised: %+v", u)
|
||||||
|
}
|
||||||
|
if got, _ := os.ReadFile(mesh); !strings.Contains(string(got), "PEER-A=") || strings.Contains(string(got), "PrivateKey") {
|
||||||
|
t.Fatalf("the mesh's configuration does not carry the found peer, or carries a key:\n%s", got)
|
||||||
|
}
|
||||||
|
// And the report says so, with what was found — port, range, peers — and never the key.
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Port != 51900 ||
|
||||||
|
report.Tunnel.Range != "192.0.2.0/24" || report.Tunnel.Peers != 2 || report.Tunnel.Kept != held.Kept {
|
||||||
|
t.Fatalf("the report does not say what was carried: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
for _, o := range report.Outcomes {
|
||||||
|
if strings.Contains(o.Detail, foundKey) {
|
||||||
|
t.Errorf("the found key was printed in an outcome: %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(report.Tunnel.Note, foundKey) {
|
||||||
|
t.Error("the found key was printed in the account")
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, "mesh-wireguard.overlay-up.takes-over"); o.Action != "held" ||
|
||||||
|
!strings.Contains(o.Detail, "stopped wg-quick@wg0") || !strings.Contains(o.Detail, "never flushed") {
|
||||||
|
t.Errorf("the takeover was not reported as a hold that stopped the found unit: %+v", o)
|
||||||
|
}
|
||||||
|
if _, recorded := state.Find("mesh-wireguard.overlay-up.takes-over"); recorded {
|
||||||
|
t.Error("the found configuration was recorded as applied, so it would be removed as an orphan")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNothingIsStoppedUntilTheMeshsInterfaceCanReplaceTheFoundOne(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
otherKey := filepath.Join(dir, "other.key")
|
||||||
|
k, _ := ecdh.X25519().GenerateKey(rand.Reader)
|
||||||
|
if err := os.WriteFile(otherKey, []byte(base64.StdEncoding.EncodeToString(k.Bytes())+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cases := map[string]*declaration.Declaration{
|
||||||
|
"another port": aTakeover(t, config, mesh, keyFile, "51821", "192.0.2.1"),
|
||||||
|
"another address": aTakeover(t, config, mesh, keyFile, "51900", "10.42.0.1"),
|
||||||
|
"another key": aTakeover(t, config, mesh, otherKey, "51900", "192.0.2.1"),
|
||||||
|
"no key file": aTakeover(t, config, mesh, filepath.Join(dir, "missing.key"), "51900", "192.0.2.1"),
|
||||||
|
}
|
||||||
|
for name, d := range cases {
|
||||||
|
m.asked = nil
|
||||||
|
report, state, err := ApplyKeeping(t.Context(), archHost(t), d, store.State{},
|
||||||
|
store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "would not replace the tunnel") {
|
||||||
|
t.Fatalf("%s: the takeover was not refused: %v", name, err)
|
||||||
|
}
|
||||||
|
if name == "another key" && !strings.Contains(err.Error(), "overlay take") {
|
||||||
|
t.Errorf("%s: the refusal does not name the remedy: %v", name, err)
|
||||||
|
}
|
||||||
|
if m.units["wg-quick@wg0"].active != "active" || m.did("systemctl stop wg-quick@wg0") {
|
||||||
|
t.Fatalf("%s: the found unit was stopped although the mesh's interface could not replace it", name)
|
||||||
|
}
|
||||||
|
if m.units["wg-quick@mesh0"].active == "active" {
|
||||||
|
t.Fatalf("%s: the mesh's interface was started on top of the found one", name)
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "would not replace") {
|
||||||
|
t.Fatalf("%s: the account does not say the tunnel is not taken and why: %+v", name, report.Tunnel)
|
||||||
|
}
|
||||||
|
if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held {
|
||||||
|
t.Errorf("%s: the found configuration was not kept before the refusal", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMeshInterfaceThatFailsToStartGivesTheFoundOneBack(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
m.units["wg-quick@mesh0"].wontStart = true
|
||||||
|
report, _, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"),
|
||||||
|
store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a mesh interface that did not come up was reported as applied")
|
||||||
|
}
|
||||||
|
if !m.did("systemctl stop wg-quick@wg0") || !m.did("systemctl start wg-quick@wg0") {
|
||||||
|
t.Fatalf("the found unit was not stopped and then started again: %v", m.asked)
|
||||||
|
}
|
||||||
|
if m.units["wg-quick@wg0"].active != "active" {
|
||||||
|
t.Fatal("the machine was left with no tunnel at all")
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != NotTaken ||
|
||||||
|
!strings.Contains(report.Tunnel.Note, "did not come up") || !strings.Contains(report.Tunnel.Note, "started again") {
|
||||||
|
t.Fatalf("the account does not say the mesh's interface failed and the found one was given back: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATakeoverIsSteadyAndAFoundUnitUpAgainIsSaidNotStopped(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
|
||||||
|
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
|
m.asked = nil
|
||||||
|
|
||||||
|
report, again := applyAdopted(t, d, state, m, dir)
|
||||||
|
if report.Changed() {
|
||||||
|
t.Errorf("a second apply moved the machine: %+v", report.Outcomes)
|
||||||
|
}
|
||||||
|
if _, still := again.HeldAt("mesh-wireguard.overlay-up.takes-over"); !still {
|
||||||
|
t.Error("the hold on the found configuration was forgotten while the service still declares it")
|
||||||
|
}
|
||||||
|
if m.did("systemctl stop wg-quick@wg0") {
|
||||||
|
t.Error("a found unit already down was stopped again")
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != Taken {
|
||||||
|
t.Errorf("a steady takeover does not read as taken: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Somebody starts the found unit again beside the mesh's interface. Not stopped by the mesh —
|
||||||
|
// on the hub it cannot hold the port, on a spoke stopping it would be a fight — but said.
|
||||||
|
m.units["wg-quick@wg0"].active = "active"
|
||||||
|
m.asked = nil
|
||||||
|
report, _ = applyAdopted(t, d, again, m, dir)
|
||||||
|
if m.did("systemctl stop wg-quick@wg0") {
|
||||||
|
t.Error("a found unit started again by hand was stopped by the mesh")
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "running again beside") {
|
||||||
|
t.Errorf("the account does not say the found unit is up again: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFoundInterfaceRaisedByHandIsRefusedNamingTheRemedy(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
// The unit is not running, yet the interface is up: the predecessor raised it by hand.
|
||||||
|
m.units["wg-quick@wg0"].active = "inactive"
|
||||||
|
m.wgUp = "wg0 mesh0\n"
|
||||||
|
report, state, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"),
|
||||||
|
store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "wg-quick down wg0") || !strings.Contains(err.Error(), "Nothing was flushed") {
|
||||||
|
t.Fatalf("an interface raised by hand was not refused naming the remedy: %v", err)
|
||||||
|
}
|
||||||
|
if m.units["wg-quick@mesh0"].active == "active" {
|
||||||
|
t.Error("the mesh's interface was started on a port the found one still holds")
|
||||||
|
}
|
||||||
|
// Looked at more than once before giving up: a person taking it down takes a moment.
|
||||||
|
shows := 0
|
||||||
|
for _, a := range m.asked {
|
||||||
|
if a == "wg show interfaces" {
|
||||||
|
shows++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if shows < takeoverRechecks+1 {
|
||||||
|
t.Errorf("the interface was looked at %d time(s) before the refusal; a person needs a moment", shows)
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != NotTaken {
|
||||||
|
t.Errorf("the account does not say the tunnel is not taken: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held {
|
||||||
|
t.Error("the found configuration was not kept before the refusal")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATakeoverIsRefusedOnAConvergedDeclaration(t *testing.T) {
|
||||||
|
_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[
|
||||||
|
{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running",
|
||||||
|
"takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}]}`))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "adopted") {
|
||||||
|
t.Fatalf("a takeover on a converged node was accepted: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0119: once the take is proven — taken, and a peer handshaken on the mesh's
|
||||||
|
// interface — the found configuration is removed from where its unit reads it, its original stays
|
||||||
|
// kept and the hold on it ends. Never before, and never brought back.
|
||||||
|
|
||||||
|
const takesOverID = "mesh-wireguard.overlay-up.takes-over"
|
||||||
|
|
||||||
|
// handshaken is `wg show mesh0 latest-handshakes` with one of the two peers through.
|
||||||
|
const handshaken = "PEER-A=\t1790000000\nPEER-B=\t0\n"
|
||||||
|
|
||||||
|
func TestAProvenTakeRetiresTheFoundConfiguration(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
m.handshakes = handshaken
|
||||||
|
report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)
|
||||||
|
|
||||||
|
if _, err := os.Lstat(config); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("a proven take left the found configuration where its unit reads it: %v", err)
|
||||||
|
}
|
||||||
|
retired, ok := state.RetiredAt(config)
|
||||||
|
if !ok || retired.Kept == "" || retired.ID != takesOverID {
|
||||||
|
t.Fatalf("the retirement was not recorded: %+v", state.Retired)
|
||||||
|
}
|
||||||
|
if kept, _ := os.ReadFile(retired.Kept); string(kept) != foundConf {
|
||||||
|
t.Fatalf("the kept original did not survive the retirement: %q", kept)
|
||||||
|
}
|
||||||
|
if _, held := state.HeldAt(takesOverID); held {
|
||||||
|
t.Error("the hold on the found configuration did not end with its retirement")
|
||||||
|
}
|
||||||
|
if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" {
|
||||||
|
t.Errorf("the found unit is not left down and disabled: %+v", u)
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Kept != retired.Kept ||
|
||||||
|
!strings.Contains(report.Tunnel.Note, "proven: 1 peer(s) handshaken on mesh0") ||
|
||||||
|
!strings.Contains(report.Tunnel.Note, "is retired") {
|
||||||
|
t.Fatalf("the account does not say the take is proven and the configuration retired: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, takesOverID); o.Action != "removed" || !strings.Contains(o.Detail, "retired") {
|
||||||
|
t.Errorf("the retirement is not what the apply says it did to the file: %+v", o)
|
||||||
|
}
|
||||||
|
// And the account still carries what was found, read from the kept original.
|
||||||
|
if report.Tunnel.Port != 51900 || report.Tunnel.Peers != 2 {
|
||||||
|
t.Errorf("the account lost what the tunnel was: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATakeNotProvenKeepsTheFoundConfigurationAndSaysSo(t *testing.T) {
|
||||||
|
cases := map[string]struct {
|
||||||
|
handshakes string
|
||||||
|
fail error
|
||||||
|
says string
|
||||||
|
}{
|
||||||
|
"no peer at all": {"", nil, "no peer has handshaken on mesh0"},
|
||||||
|
"every handshake at zero": {"PEER-A=\t0\nPEER-B=\t0\n", nil, "no peer has handshaken on mesh0"},
|
||||||
|
"wg is not there": {"", errors.New(`exec: "wg": executable file not found in $PATH`), "executable file not found"},
|
||||||
|
"the answer is nonsense": {"unable to access interface\n", nil, "not a peer and a time"},
|
||||||
|
}
|
||||||
|
for name, c := range cases {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
m.handshakes, m.handshakesFail = c.handshakes, c.fail
|
||||||
|
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
|
||||||
|
report, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
|
|
||||||
|
if got, _ := os.ReadFile(config); string(got) != foundConf {
|
||||||
|
t.Fatalf("%s: a take not proven lost the found configuration", name)
|
||||||
|
}
|
||||||
|
if _, held := state.HeldAt(takesOverID); !held {
|
||||||
|
t.Errorf("%s: the hold ended although the take is not proven", name)
|
||||||
|
}
|
||||||
|
if _, retired := state.RetiredAt(config); retired {
|
||||||
|
t.Errorf("%s: recorded as retired", name)
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != Taken ||
|
||||||
|
!strings.Contains(report.Tunnel.Note, "taken, not yet proven") ||
|
||||||
|
!strings.Contains(report.Tunnel.Note, c.says) || !strings.Contains(report.Tunnel.Note, "is kept") {
|
||||||
|
t.Errorf("%s: the account does not say the take is not proven and why: %+v", name, report.Tunnel)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A later apply that finds a peer through retires it: the take itself need not be the one.
|
||||||
|
m.handshakes, m.handshakesFail = handshaken, nil
|
||||||
|
_, state = applyAdopted(t, d, state, m, dir)
|
||||||
|
if _, err := os.Lstat(config); !os.IsNotExist(err) {
|
||||||
|
t.Errorf("%s: the apply after the take was proven kept the found configuration", name)
|
||||||
|
}
|
||||||
|
if _, retired := state.RetiredAt(config); !retired {
|
||||||
|
t.Errorf("%s: the later retirement was not recorded", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFoundConfigurationWhoseKeptOriginalIsMissingIsNotRetired(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
|
||||||
|
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
|
held, _ := state.HeldAt(takesOverID)
|
||||||
|
if err := os.Remove(held.Kept); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m.handshakes = handshaken
|
||||||
|
report, state := applyAdopted(t, d, state, m, dir)
|
||||||
|
if got, _ := os.ReadFile(config); string(got) != foundConf {
|
||||||
|
t.Fatal("the found configuration was removed with no kept original left of it")
|
||||||
|
}
|
||||||
|
if _, still := state.HeldAt(takesOverID); !still {
|
||||||
|
t.Error("the hold ended although nothing was retired")
|
||||||
|
}
|
||||||
|
if _, retired := state.RetiredAt(config); retired {
|
||||||
|
t.Error("recorded as retired")
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || !strings.Contains(report.Tunnel.Note, "is not retired") ||
|
||||||
|
!strings.Contains(report.Tunnel.Note, held.Kept+" is missing") {
|
||||||
|
t.Errorf("the account does not say the kept original is missing: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFoundConfigurationRewrittenSinceItWasFoundIsKeptAgainBeforeItGoes(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
|
||||||
|
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
|
rewritten := foundConf + "\n[Peer]\nPublicKey = PEER-C=\nAllowedIPs = 192.0.2.4/32\n"
|
||||||
|
if err := os.WriteFile(config, []byte(rewritten), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m.handshakes = handshaken
|
||||||
|
_, state = applyAdopted(t, d, state, m, dir)
|
||||||
|
if _, err := os.Lstat(config); !os.IsNotExist(err) {
|
||||||
|
t.Fatal("a proven take kept a rewritten configuration")
|
||||||
|
}
|
||||||
|
retired, _ := state.RetiredAt(config)
|
||||||
|
if first, _ := os.ReadFile(retired.Kept); string(first) != foundConf {
|
||||||
|
t.Errorf("the first original was overwritten: %q", first)
|
||||||
|
}
|
||||||
|
kept, _ := filepath.Glob(filepath.Join(dir, "kept", "*-wg0.conf"))
|
||||||
|
var found bool
|
||||||
|
for _, k := range kept {
|
||||||
|
if got, _ := os.ReadFile(k); string(got) == rewritten {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Errorf("what the file held when it was retired was not kept: %v", kept)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARetiredTakeIsSteadyAndItsFoundUnitFindsNothingToDo(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
m.handshakes = handshaken
|
||||||
|
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
|
||||||
|
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
|
retired, _ := state.RetiredAt(config)
|
||||||
|
|
||||||
|
// wg-quick@wg0 with no configuration: inactive, and disabled — the check of it every apply
|
||||||
|
// makes must find nothing to do and fail on nothing.
|
||||||
|
m.asked = nil
|
||||||
|
report, again := applyAdopted(t, d, state, m, dir)
|
||||||
|
if report.Changed() {
|
||||||
|
t.Errorf("an apply after the retirement moved the machine: %+v", report.Outcomes)
|
||||||
|
}
|
||||||
|
if m.did("systemctl stop wg-quick@wg0") || m.did("systemctl start wg-quick@wg0") {
|
||||||
|
t.Errorf("the retired tunnel's unit was acted on: %v", m.asked)
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(config); !os.IsNotExist(err) {
|
||||||
|
t.Error("the found configuration came back")
|
||||||
|
}
|
||||||
|
if _, held := again.HeldAt(takesOverID); held {
|
||||||
|
t.Error("a retired configuration is held again")
|
||||||
|
}
|
||||||
|
if r, ok := again.RetiredAt(config); !ok || r != retired {
|
||||||
|
t.Errorf("the retirement was not kept as it was: %+v", again.Retired)
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, takesOverID); o.Action != "unchanged" || !strings.Contains(o.Detail, "retired") {
|
||||||
|
t.Errorf("the retired configuration is not said as retired: %+v", o)
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Kept != retired.Kept ||
|
||||||
|
!strings.Contains(report.Tunnel.Note, "retired") || report.Tunnel.Port != 51900 {
|
||||||
|
t.Errorf("the account of a retired take does not say so: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enabled at boot again by a person: disabled again, as any take does, and still no error.
|
||||||
|
m.units["wg-quick@wg0"].enabled = "enabled"
|
||||||
|
_, _ = applyAdopted(t, d, again, m, dir)
|
||||||
|
if m.units["wg-quick@wg0"].enabled != "disabled" {
|
||||||
|
t.Error("the found unit enabled again by hand was left to start at boot")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUndeclaringThePrivateNetworkAfterRetirementBringsNothingBack(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
m.handshakes = handshaken
|
||||||
|
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
|
||||||
|
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
|
|
||||||
|
// The private network unassigned: only something else is declared.
|
||||||
|
other := adopted(t, `{"taken":[],"untaken":{}}`,
|
||||||
|
`{"id":"other.file","type":"file","path":"`+filepath.Join(dir, "other.conf")+`","content":"x\n"}`)
|
||||||
|
m.asked = nil
|
||||||
|
_, after := applyAdopted(t, other, state, m, dir)
|
||||||
|
if _, err := os.Lstat(config); !os.IsNotExist(err) {
|
||||||
|
t.Fatal("undeclaring the private network brought the found configuration back")
|
||||||
|
}
|
||||||
|
if m.did("systemctl start wg-quick@wg0") || m.did("systemctl enable wg-quick@wg0") {
|
||||||
|
t.Errorf("undeclaring the private network started the found tunnel: %v", m.asked)
|
||||||
|
}
|
||||||
|
if _, ok := after.RetiredAt(config); !ok {
|
||||||
|
t.Error("the retirement was forgotten with the private network")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Assigned again, it finds the configuration retired rather than missing, and raises the
|
||||||
|
// mesh's interface.
|
||||||
|
report, _ := applyAdopted(t, d, after, m, dir)
|
||||||
|
if report.Tunnel == nil || report.Tunnel.State != Taken {
|
||||||
|
t.Errorf("the private network assigned again did not take the tunnel: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(config); !os.IsNotExist(err) {
|
||||||
|
t.Error("assigning the private network again brought the found configuration back")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPlanSaysTheFoundConfigurationIsRetiredWhenTheTakeIsProven(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
|
||||||
|
|
||||||
|
plan := Plan(d, store.State{}, store.OriginDeclared)
|
||||||
|
report, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
|
if got, want := strings.Join(ids(plan), " "), strings.Join(outcomeIDs(report), " "); got != want {
|
||||||
|
t.Errorf("the plan said %q and the apply did %q", got, want)
|
||||||
|
}
|
||||||
|
var take Step
|
||||||
|
for _, s := range plan {
|
||||||
|
if s.ID == takesOverID {
|
||||||
|
take = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if take.Verb != "hold" || take.Target != config || !strings.Contains(take.Why, "retired — removed from "+config) ||
|
||||||
|
!strings.Contains(take.Why, "handshaking on mesh0") {
|
||||||
|
t.Errorf("the plan does not say the found configuration is retired once proven: %+v", take)
|
||||||
|
}
|
||||||
|
// Held and still declared: never planned as forgotten.
|
||||||
|
if strings.Contains(verbs(Plan(d, state, store.OriginDeclared)), "forget "+takesOverID) {
|
||||||
|
t.Error("the plan forgets a hold the apply keeps")
|
||||||
|
}
|
||||||
|
|
||||||
|
m.handshakes = handshaken
|
||||||
|
_, state = applyAdopted(t, d, state, m, dir)
|
||||||
|
for _, s := range Plan(d, state, store.OriginDeclared) {
|
||||||
|
if s.ID == takesOverID && (s.Verb != "check" || !strings.Contains(s.Why, "retired once the take")) {
|
||||||
|
t.Errorf("a retired configuration is planned as %+v", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptCopies is every copy kept of the found configuration.
|
||||||
|
func keptCopies(t *testing.T, dir string) []string {
|
||||||
|
t.Helper()
|
||||||
|
kept, err := filepath.Glob(filepath.Join(dir, "kept", "*-wg0.conf"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return kept
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAConfigurationPutBackIsRetiredAgainOnItsFirstOriginalAndSettles(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
m.handshakes = handshaken
|
||||||
|
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
|
||||||
|
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
|
first, _ := state.RetiredAt(config)
|
||||||
|
|
||||||
|
// Put back by hand with the original, while no peer is through yet: held on the first
|
||||||
|
// original, and the account says what a rollback takes.
|
||||||
|
write(t, config, foundConf)
|
||||||
|
m.handshakes = ""
|
||||||
|
report, state := applyAdopted(t, d, state, m, dir)
|
||||||
|
if held, ok := state.HeldAt(takesOverID); !ok || held.Kept != first.Kept {
|
||||||
|
t.Fatalf("what came back is not held on the first original: %+v", held)
|
||||||
|
}
|
||||||
|
if report.Tunnel == nil || !strings.Contains(report.Tunnel.Note, "came back after it was retired") ||
|
||||||
|
!strings.Contains(report.Tunnel.Note, "unassigning the private network first") {
|
||||||
|
t.Errorf("the account does not say a rollback means unassigning the private network: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Proven: retired again, nothing more kept, said once.
|
||||||
|
m.handshakes = handshaken
|
||||||
|
report, state = applyAdopted(t, d, state, m, dir)
|
||||||
|
again, _ := state.RetiredAt(config)
|
||||||
|
if _, err := os.Lstat(config); !os.IsNotExist(err) || again.Kept != first.Kept || again.Extra != "" {
|
||||||
|
t.Fatalf("put back as it was, it was not retired again on the first original: %+v", again)
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, takesOverID); o.Action != "removed" ||
|
||||||
|
!strings.HasPrefix(o.Detail, "the found configuration came back and was retired again") ||
|
||||||
|
strings.Contains(o.Detail, "differed") {
|
||||||
|
t.Errorf("the second retirement is not said as one: %+v", o)
|
||||||
|
}
|
||||||
|
if !strings.Contains(report.Tunnel.Note, "unassigning the private network first") {
|
||||||
|
t.Errorf("the account does not say what a rollback takes: %q", report.Tunnel.Note)
|
||||||
|
}
|
||||||
|
if n := len(keptCopies(t, dir)); n != 1 {
|
||||||
|
t.Errorf("%d copies kept of one content", n)
|
||||||
|
}
|
||||||
|
if report, _ := applyAdopted(t, d, state, m, dir); report.Changed() {
|
||||||
|
t.Errorf("a steady machine moved after the second retirement: %+v", report.Outcomes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put back with something else: that is kept beside the first original, which stays the record's.
|
||||||
|
other := strings.Replace(foundConf, "PEER-B=", "PEER-Z=", 1)
|
||||||
|
write(t, config, other)
|
||||||
|
report, state = applyAdopted(t, d, state, m, dir)
|
||||||
|
third, _ := state.RetiredAt(config)
|
||||||
|
if third.Kept != first.Kept || third.Extra == "" || third.Extra == first.Kept {
|
||||||
|
t.Fatalf("other content was not kept apart from the first original: %+v", third)
|
||||||
|
}
|
||||||
|
if got, _ := os.ReadFile(third.Extra); string(got) != other {
|
||||||
|
t.Errorf("the extra copy does not hold what was put back: %q", got)
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, takesOverID); !strings.Contains(o.Detail, third.Extra) ||
|
||||||
|
!strings.Contains(report.Tunnel.Note, third.Extra) {
|
||||||
|
t.Errorf("where the extra copy is was not said: %+v / %q", o, report.Tunnel.Note)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the same other content again: nothing more kept, the record as it was.
|
||||||
|
write(t, config, other)
|
||||||
|
report, state = applyAdopted(t, d, state, m, dir)
|
||||||
|
fourth, _ := state.RetiredAt(config)
|
||||||
|
if fourth.Kept != first.Kept || fourth.Extra != third.Extra || len(keptCopies(t, dir)) != 2 {
|
||||||
|
t.Errorf("the same content put back again grew the copies: %+v, %v", fourth, keptCopies(t, dir))
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, takesOverID); strings.Contains(o.Detail, "differed") {
|
||||||
|
t.Errorf("a copy already kept was said as new: %+v", o)
|
||||||
|
}
|
||||||
|
if report, _ := applyAdopted(t, d, state, m, dir); report.Changed() {
|
||||||
|
t.Errorf("a steady machine moved: %+v", report.Outcomes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOnlyWgQuicksOwnConfigurationIsRetired(t *testing.T) {
|
||||||
|
// Not under the wireguard directory.
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
wireguardDir = filepath.Join(dir, "elsewhere")
|
||||||
|
m.handshakes = handshaken
|
||||||
|
report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)
|
||||||
|
if got, _ := os.ReadFile(config); string(got) != foundConf {
|
||||||
|
t.Fatal("a configuration outside wg-quick's directory was removed")
|
||||||
|
}
|
||||||
|
if _, held := state.HeldAt(takesOverID); !held || !strings.Contains(report.Tunnel.Note, "is not retired: only ") {
|
||||||
|
t.Errorf("the refusal is not said, or the hold ended: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A path the mesh itself writes.
|
||||||
|
dir, config, mesh, keyFile, m = aHubInUse(t)
|
||||||
|
m.handshakes = handshaken
|
||||||
|
known := store.State{}
|
||||||
|
known.Record(store.Applied{ID: "bundle.wg0", Type: "file", Target: config, Origin: store.OriginCarried})
|
||||||
|
report, _ = applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), known, m, dir)
|
||||||
|
if got, _ := os.ReadFile(config); string(got) != foundConf {
|
||||||
|
t.Fatal("a path the mesh writes was retired")
|
||||||
|
}
|
||||||
|
if !strings.Contains(report.Tunnel.Note, "a path the mesh itself writes") {
|
||||||
|
t.Errorf("the refusal is not said: %+v", report.Tunnel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFoundConfigurationThatIsALinkIsKeptAndLeftToAPerson(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
target := filepath.Join(dir, "predecessor", "hub.conf")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
write(t, target, foundConf)
|
||||||
|
if err := os.Remove(config); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(target, config); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m.handshakes = handshaken
|
||||||
|
report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)
|
||||||
|
if _, err := os.Lstat(config); err != nil {
|
||||||
|
t.Fatal("the link was removed, leaving the key-bearing file it points at")
|
||||||
|
}
|
||||||
|
if got, _ := os.ReadFile(target); string(got) != foundConf {
|
||||||
|
t.Fatal("the file the link points at was touched")
|
||||||
|
}
|
||||||
|
held, ok := state.HeldAt(takesOverID)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("the hold ended")
|
||||||
|
}
|
||||||
|
if kept, _ := os.ReadFile(held.Kept); string(kept) != foundConf {
|
||||||
|
t.Errorf("what was kept is not what the link points at: %q", kept)
|
||||||
|
}
|
||||||
|
if !strings.Contains(report.Tunnel.Note, "is a link to "+target) || !strings.Contains(report.Tunnel.Note, "by hand") {
|
||||||
|
t.Errorf("the account does not say the link must be retired by hand: %q", report.Tunnel.Note)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARetirementWhoseRecordWasNeverSavedIsRecordedByTheNextApply(t *testing.T) {
|
||||||
|
dir, config, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
|
||||||
|
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
|
held, _ := state.HeldAt(takesOverID)
|
||||||
|
// An apply removed the file and stopped before its state was saved.
|
||||||
|
if err := os.Remove(config); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m.handshakes = handshaken
|
||||||
|
report, state := applyAdopted(t, d, state, m, dir)
|
||||||
|
if r, ok := state.RetiredAt(config); !ok || r.Kept != held.Kept {
|
||||||
|
t.Fatalf("the retirement was not recorded: %+v", state.Retired)
|
||||||
|
}
|
||||||
|
if _, still := state.HeldAt(takesOverID); still {
|
||||||
|
t.Error("the hold did not end")
|
||||||
|
}
|
||||||
|
if o := outcomeOf(report, takesOverID); o.Action != "unchanged" || !strings.Contains(o.Detail, "already gone") {
|
||||||
|
t.Errorf("a file already gone is not said as such: %+v", o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARemovalThatFailsKeepsTheFileAndTheHold(t *testing.T) {
|
||||||
|
if os.Geteuid() == 0 {
|
||||||
|
t.Skip("root removes from a directory it may not write to")
|
||||||
|
}
|
||||||
|
dir, _, mesh, keyFile, m := aHubInUse(t)
|
||||||
|
wg := filepath.Join(dir, "wireguard")
|
||||||
|
if err := os.MkdirAll(wg, 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
config := filepath.Join(wg, "wg0.conf")
|
||||||
|
write(t, config, foundConf)
|
||||||
|
wireguardDir = wg
|
||||||
|
d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
|
||||||
|
_, state := applyAdopted(t, d, store.State{}, m, dir)
|
||||||
|
|
||||||
|
if err := os.Chmod(wg, 0o500); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = os.Chmod(wg, 0o700) })
|
||||||
|
m.handshakes = handshaken
|
||||||
|
report, state := applyAdopted(t, d, state, m, dir)
|
||||||
|
if got, _ := os.ReadFile(config); string(got) != foundConf {
|
||||||
|
t.Fatal("the found configuration is gone although it could not be removed")
|
||||||
|
}
|
||||||
|
if _, held := state.HeldAt(takesOverID); !held {
|
||||||
|
t.Error("the hold ended although nothing was retired")
|
||||||
|
}
|
||||||
|
if _, retired := state.RetiredAt(config); retired {
|
||||||
|
t.Error("recorded as retired")
|
||||||
|
}
|
||||||
|
if !strings.Contains(report.Tunnel.Note, "removing it failed") || !strings.Contains(report.Tunnel.Note, "permission denied") {
|
||||||
|
t.Errorf("the failed removal is not said: %q", report.Tunnel.Note)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -376,7 +376,7 @@ func TestAContainerIsGivenItsEnvironmentFiles(t *testing.T) {
|
|||||||
var ran []string
|
var ran []string
|
||||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||||
if len(args) > 0 && args[0] == "inspect" {
|
if len(args) > 0 && args[0] == "container" {
|
||||||
return "", fmt.Errorf("no such container")
|
return "", fmt.Errorf("no such container")
|
||||||
}
|
}
|
||||||
return "", nil
|
return "", nil
|
||||||
@@ -403,3 +403,51 @@ func TestAContainerIsGivenItsEnvironmentFiles(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A container may name its resolvers and its own address — the shape a module shipping its own
|
||||||
|
// validating DNS needs: the resolver pinned where its siblings can find it, the siblings pointed
|
||||||
|
// at it. Both flags take addresses, so both reach the runtime verbatim.
|
||||||
|
func TestAContainerIsGivenItsResolverAndItsAddress(t *testing.T) {
|
||||||
|
var ran []string
|
||||||
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||||
|
if len(args) > 0 && args[0] == "container" {
|
||||||
|
return "", fmt.Errorf("no such container")
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
d := declare(t, `{"id":"imap","type":"container","name":"mailu-imap",`+
|
||||||
|
`"image":"dovecot@sha256:0000000000000000000000000000000000000000000000000000000000000000",`+
|
||||||
|
`"network":"mailu","dns":["192.168.203.254"],"ip":"192.168.203.7"}`)
|
||||||
|
|
||||||
|
_, _, _ = Apply(context.Background(), archHost(t), d, store.State{},
|
||||||
|
store.OriginDeclared, run, nil, nil)
|
||||||
|
|
||||||
|
var started string
|
||||||
|
for _, line := range ran {
|
||||||
|
if strings.Contains(line, "run ") {
|
||||||
|
started = line
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, want := range []string{"--dns 192.168.203.254", "--ip 192.168.203.7"} {
|
||||||
|
if !strings.Contains(started, want) {
|
||||||
|
t.Errorf("the container was started without %q:\n%s", want, started)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The resolver and address are part of the spec — a container whose dns or ip moved is a
|
||||||
|
// different container, or the fields can never reach one that already runs. That is not
|
||||||
|
// hypothetical: their first deployment compared equal and changed nothing.
|
||||||
|
func TestAChangedResolverOrAddressIsAChangedContainer(t *testing.T) {
|
||||||
|
base := &declaration.Container{ID: "c", Name: "x", Image: "a@sha256:00"}
|
||||||
|
withDns := &declaration.Container{ID: "c", Name: "x", Image: "a@sha256:00", Dns: []string{"192.168.203.254"}}
|
||||||
|
withIP := &declaration.Container{ID: "c", Name: "x", Image: "a@sha256:00", IP: "192.168.203.254"}
|
||||||
|
plain := containerSpecReading(base, nil, nil)
|
||||||
|
if containerSpecReading(withDns, nil, nil) == plain {
|
||||||
|
t.Error("adding a resolver did not change the spec, so it can never reach a running container")
|
||||||
|
}
|
||||||
|
if containerSpecReading(withIP, nil, nil) == plain {
|
||||||
|
t.Error("adding an address did not change the spec, so it can never reach a running container")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ func (l *labelled) run(_ context.Context, _ string, args ...string) (string, err
|
|||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "info":
|
case "info":
|
||||||
return "27.0\n", nil
|
return "27.0\n", nil
|
||||||
case "inspect":
|
case "container":
|
||||||
spec, ok := l.spec[args[len(args)-1]]
|
spec, ok := l.spec[args[len(args)-1]]
|
||||||
if !ok {
|
if !ok {
|
||||||
return "", errors.New("no such container")
|
return "", errors.New("no such container")
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/apply"
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
"github.com/novox/mesh-host/internal/declaration"
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
@@ -33,8 +34,16 @@ type Runner = apply.Runner
|
|||||||
// What it does not do is the host's own lifecycle bookkeeping — recording a known-good version,
|
// What it does not do is the host's own lifecycle bookkeeping — recording a known-good version,
|
||||||
// clearing the launcher's start counter. Those are facts about a running `mesh-host`, and this is
|
// clearing the launcher's start counter. Those are facts about a running `mesh-host`, and this is
|
||||||
// not one.
|
// not one.
|
||||||
|
//
|
||||||
|
// What it does record is that the bundle was consumed, and in which mode the operator raised
|
||||||
|
// the machine. `raw` is the exact bytes of what is applied — the bundle as rewritten for this
|
||||||
|
// machine — and its digest is what `mesh-host reconcile` later holds the carried bundle against,
|
||||||
|
// by digest: a host built from the carried template does not match it, since genesis rewrote the
|
||||||
|
// ports, root credentials and adoption; a host built from the lock genesis wrote out does.
|
||||||
|
// Applying the unrewritten template on a raised node recreated the store and loaded the converged
|
||||||
|
// filter on an adopted one (novox/hq issue 104).
|
||||||
func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declaration.Declaration,
|
func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declaration.Declaration,
|
||||||
run Runner, say func(string)) (apply.Report, error) {
|
raw []byte, run Runner, say func(string)) (apply.Report, error) {
|
||||||
|
|
||||||
// Refuse a shape this host cannot apply before anything is applied, exactly as `mesh-host`
|
// Refuse a shape this host cannot apply before anything is applied, exactly as `mesh-host`
|
||||||
// does: finding out half way through is the half-configured machine tier 0 exists to prevent.
|
// does: finding out half way through is the half-configured machine tier 0 exists to prevent.
|
||||||
@@ -42,6 +51,12 @@ func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declarati
|
|||||||
return apply.Report{}, err
|
return apply.Report{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One apply at a time on this machine: a host already running here applies too.
|
||||||
|
unlock, err := store.Lock(o.State, func() { say(" waiting another apply holds this node's state") })
|
||||||
|
if err != nil {
|
||||||
|
return apply.Report{}, err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
known, err := store.Load(o.State)
|
known, err := store.Load(o.State)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return apply.Report{}, err
|
return apply.Report{}, err
|
||||||
@@ -55,6 +70,20 @@ func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declarati
|
|||||||
func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed,
|
func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed,
|
||||||
apply.KeepIn(filepath.Dir(o.State)))
|
apply.KeepIn(filepath.Dir(o.State)))
|
||||||
|
|
||||||
|
// The bundle is consumed, whichever way the apply went: what is on the machine came from these
|
||||||
|
// bytes, and the carried ones must not be applied over it. The mode is the operator's word at
|
||||||
|
// genesis, and only at genesis: the controller records the same and says it in every
|
||||||
|
// declaration from then on (novox/hq ADR 0100), so a node the mesh has spoken to — this
|
||||||
|
// installer re-run on it, or finishing a pivot — keeps the mode it has, which may since have
|
||||||
|
// been flipped.
|
||||||
|
updated.Genesis = &store.Genesis{Digest: apply.DigestOf(raw), At: time.Now().UTC(), Rewritten: true}
|
||||||
|
if updated.Mode == "" {
|
||||||
|
updated.Mode = store.ModeConverged
|
||||||
|
if o.Adopted {
|
||||||
|
updated.Mode = store.ModeAdopted
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Saved whichever way it went, for the reason `mesh-host` gives: what was applied before a
|
// Saved whichever way it went, for the reason `mesh-host` gives: what was applied before a
|
||||||
// failure is on the machine either way, and a host that did not record it would believe it
|
// failure is on the machine either way, and a host that did not record it would believe it
|
||||||
// owns less than it does and leave that behind for ever.
|
// owns less than it does and leave that behind for ever.
|
||||||
|
|||||||
@@ -3,12 +3,15 @@ package bootstrap
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
"github.com/novox/mesh-host/internal/declaration"
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
"github.com/novox/mesh-host/internal/system"
|
"github.com/novox/mesh-host/internal/system"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -113,7 +116,7 @@ func TestTheBundleKeepsTheOriginalOfWhatItWritesOver(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
o := Options{State: filepath.Join(dir, "state.json")}
|
o := Options{State: filepath.Join(dir, "state.json")}
|
||||||
report, err := ApplyBundle(context.Background(), o, sys, d, nil, quietly)
|
report, err := ApplyBundle(context.Background(), o, sys, d, nil, nil, quietly)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -127,3 +130,53 @@ func TestTheBundleKeepsTheOriginalOfWhatItWritesOver(t *testing.T) {
|
|||||||
t.Errorf("the kept original is %q (%v)", got, err)
|
t.Errorf("the kept original is %q (%v)", got, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Defends novox/hq issue 104: genesis consumes the bundle, recording the digest of what it applied
|
||||||
|
// and the mode the operator raised the machine in, so the host's own `reconcile` never applies the
|
||||||
|
// carried bytes over it.
|
||||||
|
func TestGenesisConsumesTheBundleAndRecordsTheMode(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[
|
||||||
|
{"id":"a","type":"file","path":"` + filepath.Join(dir, "a.conf") + `","content":"x\n"}]}`)
|
||||||
|
d, err := declaration.ParseFileTrusted(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sys, err := system.For("arch")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, adopted := range []bool{false, true} {
|
||||||
|
o := Options{State: filepath.Join(dir, fmt.Sprintf("state-%v.json", adopted)), Adopted: adopted}
|
||||||
|
if _, err := ApplyBundle(context.Background(), o, sys, d, raw, nil, quietly); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
known, err := store.Load(o.State)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if known.Genesis == nil || known.Genesis.Digest != apply.DigestOf(raw) || !known.Genesis.Rewritten {
|
||||||
|
t.Errorf("adopted=%v: genesis did not record the bundle it consumed: %+v", adopted, known.Genesis)
|
||||||
|
}
|
||||||
|
want := store.ModeConverged
|
||||||
|
if adopted {
|
||||||
|
want = store.ModeAdopted
|
||||||
|
}
|
||||||
|
if known.Mode != want {
|
||||||
|
t.Errorf("adopted=%v: genesis recorded the mode as %q, want %q", adopted, known.Mode, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-run on a node the mesh has spoken to since — and converged — genesis leaves the mode
|
||||||
|
// alone: it is the operator's word at genesis, and the controller's from then on.
|
||||||
|
o := Options{State: filepath.Join(dir, "state-flipped.json"), Adopted: true}
|
||||||
|
if err := store.Save(o.State, store.State{Mode: store.ModeConverged}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := ApplyBundle(context.Background(), o, sys, d, raw, nil, quietly); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if known, _ := store.Load(o.State); known.Mode != store.ModeConverged {
|
||||||
|
t.Errorf("a genesis re-run set the mode back to %q over the mesh's converged", known.Mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/firewall"
|
"github.com/novox/mesh-host/internal/firewall"
|
||||||
|
"github.com/novox/mesh-host/internal/tunnel"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence
|
// Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence
|
||||||
@@ -201,6 +202,11 @@ type Options struct {
|
|||||||
// until each module is taken, its firewall stays in force, and the mesh guards its own ports
|
// until each module is taken, its firewall stays in force, and the mesh guards its own ports
|
||||||
// in a table that only refuses. Without it, a machine in use is refused.
|
// in a table that only refuses. Without it, a machine in use is refused.
|
||||||
Adopted bool
|
Adopted bool
|
||||||
|
// Tunnel names the found tunnel's interface an adopted hub takes over (novox/hq ADR 0105), when
|
||||||
|
// more than one is up and the machine cannot say which. Empty finds the one that is up. Once
|
||||||
|
// found, the tunnel's port is the hub's and its range the private network's; --hub-port and
|
||||||
|
// --overlay-range may agree with it or be left unsaid.
|
||||||
|
Tunnel string
|
||||||
}
|
}
|
||||||
|
|
||||||
// pivots reports whether this run goes past the foundation.
|
// pivots reports whether this run goes past the foundation.
|
||||||
@@ -311,6 +317,9 @@ type Result struct {
|
|||||||
// Filter is the packet filter chosen for when the node converges; an adopted genesis loads
|
// Filter is the packet filter chosen for when the node converges; an adopted genesis loads
|
||||||
// none, and the flip assigns this one.
|
// none, and the flip assigns this one.
|
||||||
Filter string `json:"filter-on-converge,omitempty"`
|
Filter string `json:"filter-on-converge,omitempty"`
|
||||||
|
// Tunnel is the found tunnel an adopted genesis takes over (novox/hq ADR 0105): what was read
|
||||||
|
// from it, never its key.
|
||||||
|
Tunnel *tunnel.Found `json:"tunnel,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run performs the bootstrap, saying what it is doing as it goes.
|
// Run performs the bootstrap, saying what it is doing as it goes.
|
||||||
@@ -394,6 +403,23 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
|||||||
}
|
}
|
||||||
result.Firewall = string(kind)
|
result.Firewall = string(kind)
|
||||||
say(" adopted what is on this machine is kept; its firewall (" + string(kind) + ") stays in force")
|
say(" adopted what is on this machine is kept; its firewall (" + string(kind) + ") stays in force")
|
||||||
|
|
||||||
|
// The tunnel the predecessor left, which the private network takes over (novox/hq ADR
|
||||||
|
// 0105): its port is the hub's and its range is the mesh's from here on, so both are
|
||||||
|
// settled before the ports are checked free and the bundle rewritten.
|
||||||
|
found, err := TakeTheTunnel(&o, d.Run)
|
||||||
|
if err != nil {
|
||||||
|
return result, failed(StepPreflight, err)
|
||||||
|
}
|
||||||
|
if found != nil {
|
||||||
|
result.Tunnel = found
|
||||||
|
result.Ports = o.Ports
|
||||||
|
say(fmt.Sprintf(" tunnel %s — the private network takes it over: its port %d is "+
|
||||||
|
"the hub's, its range %s the mesh's, and its %d peer(s) are carried until they enrol",
|
||||||
|
found.Interface, found.Port, found.Range, len(found.Peers)))
|
||||||
|
} else {
|
||||||
|
say(" tunnel none up on this machine; the private network is raised on its own port and range")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
sys, err := WorkOutSystem(ctx, d.Run, o.System)
|
sys, err := WorkOutSystem(ctx, d.Run, o.System)
|
||||||
@@ -566,7 +592,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
|||||||
|
|
||||||
// ---- 4. apply -----------------------------------------------------------------------
|
// ---- 4. apply -----------------------------------------------------------------------
|
||||||
say("apply — raising the foundation")
|
say("apply — raising the foundation")
|
||||||
report, err := ApplyBundle(ctx, o, sys, rewritten.Declaration, d.Run, say)
|
report, err := ApplyBundle(ctx, o, sys, rewritten.Declaration, rewritten.Bundle, d.Run, say)
|
||||||
result.Applied, result.Changed = len(report.Outcomes), report.Changed()
|
result.Applied, result.Changed = len(report.Outcomes), report.Changed()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, failed(StepApply, err)
|
return result, failed(StepApply, err)
|
||||||
|
|||||||
@@ -254,7 +254,7 @@ func TestAManifestWantingNoStoresIsRefusedWithTheShapeItShouldHave(t *testing.T)
|
|||||||
// a reply proves the sealed connections it was given are the ones the foundation made.
|
// a reply proves the sealed connections it was given are the ones the foundation made.
|
||||||
func TestThePermanentControlPlaneIsAskedTheSameQuestion(t *testing.T) {
|
func TestThePermanentControlPlaneIsAskedTheSameQuestion(t *testing.T) {
|
||||||
runtime := &asked{answer: aMeshThatAgrees(map[string]string{
|
runtime := &asked{answer: aMeshThatAgrees(map[string]string{
|
||||||
"module list": "",
|
"module list": "",
|
||||||
"exec mesh-controller /mesh-controller": "1 node, 0 waiting\n",
|
"exec mesh-controller /mesh-controller": "1 node, 0 waiting\n",
|
||||||
})}
|
})}
|
||||||
control := controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second}
|
control := controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second}
|
||||||
|
|||||||
@@ -112,7 +112,14 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
|
|||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
joining, cancel := context.WithTimeout(ctx, o.Wait)
|
joining, cancel := context.WithTimeout(ctx, o.Wait)
|
||||||
joined, err := control.run(joining, o.Host, "enrol", "--token", token, "--state", o.State)
|
args := []string{"enrol", "--token", token, "--state", o.State}
|
||||||
|
if o.Tunnel != "" {
|
||||||
|
// The found tunnel's key becomes this node's overlay key, and the tunnel travels with
|
||||||
|
// the enrolment (novox/hq ADR 0105). Named, so the host takes the one genesis settled
|
||||||
|
// its ports and range on and not another that came up since.
|
||||||
|
args = append(args, "--tunnel", o.Tunnel)
|
||||||
|
}
|
||||||
|
joined, err := control.run(joining, o.Host, args...)
|
||||||
cancel()
|
cancel()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return out, fmt.Errorf(
|
return out, fmt.Errorf(
|
||||||
|
|||||||
@@ -230,8 +230,9 @@ func raiseGiteaServer(ctx context.Context, run Runner, timeout time.Duration, db
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
// Already there: a re-run does not raise a second one. `docker start` is a no-op on a running
|
// Already there: a re-run does not raise a second one. `docker start` is a no-op on a running
|
||||||
// container and revives a stopped one.
|
// container and revives a stopped one. `container inspect`, not the bare form: a name is not
|
||||||
if out, _ := run(asking, "docker", "inspect", "--format", "{{.Id}}", giteaBootstrap); strings.TrimSpace(out) != "" {
|
// unique across object kinds, and `.Id` resolves on a network or volume too.
|
||||||
|
if out, _ := run(asking, "docker", "container", "inspect", "--format", "{{.Id}}", giteaBootstrap); strings.TrimSpace(out) != "" {
|
||||||
_, _ = run(asking, "docker", "start", giteaBootstrap)
|
_, _ = run(asking, "docker", "start", giteaBootstrap)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"sort"
|
"sort"
|
||||||
@@ -13,6 +14,7 @@ import (
|
|||||||
"github.com/novox/mesh-host/internal/declaration"
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
"github.com/novox/mesh-host/internal/reachable"
|
"github.com/novox/mesh-host/internal/reachable"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
"github.com/novox/mesh-host/internal/tunnel"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FoundationPorts are the machine's ports the foundation binds (novox/hq ADR 0100).
|
// FoundationPorts are the machine's ports the foundation binds (novox/hq ADR 0100).
|
||||||
@@ -321,6 +323,36 @@ func PortsFree(ctx context.Context, run Runner, p FoundationPorts, ours func(rea
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TakeTheTunnel finds the tunnel an adopted machine's private network takes over (novox/hq ADR
|
||||||
|
// 0105) and settles the options on it: the hub's port is the tunnel's, the mesh's range is the
|
||||||
|
// tunnel's, and the interface is named for the enrolment that takes its key. Nil when no tunnel is
|
||||||
|
// up, which is an ordinary machine. A --hub-port or --overlay-range that disagrees with the
|
||||||
|
// tunnel is refused: the peers dial the tunnel's port and live in its range, and a mesh raised
|
||||||
|
// beside them on other numbers is the two-tunnel shape the record rejects.
|
||||||
|
func TakeTheTunnel(o *Options, run Runner) (*tunnel.Found, error) {
|
||||||
|
found, err := tunnel.Find(context.Background(), tunnel.Runner(run), o.Tunnel)
|
||||||
|
if errors.Is(err, tunnel.ErrNone) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w. An adopted hub takes over the tunnel it finds; nothing was changed", err)
|
||||||
|
}
|
||||||
|
if o.Ports.Hub != 0 && o.Ports.Hub != DefaultPorts().Hub && o.Ports.Hub != found.Port {
|
||||||
|
return nil, fmt.Errorf("--hub-port %d disagrees with the tunnel %s, which listens on %d: the "+
|
||||||
|
"private network takes over that tunnel on its own port, so leave --hub-port unsaid or "+
|
||||||
|
"say %d", o.Ports.Hub, found.Interface, found.Port, found.Port)
|
||||||
|
}
|
||||||
|
if o.OverlayRange != "" && o.OverlayRange != DefaultOverlayRange && o.OverlayRange != found.Range {
|
||||||
|
return nil, fmt.Errorf("--overlay-range %s disagrees with the tunnel %s, whose range is %s: the "+
|
||||||
|
"private network takes over that tunnel with its range, so leave --overlay-range unsaid "+
|
||||||
|
"or say %s", o.OverlayRange, found.Interface, found.Range, found.Range)
|
||||||
|
}
|
||||||
|
o.Tunnel = found.Interface
|
||||||
|
o.Ports.Hub = found.Port
|
||||||
|
o.OverlayRange = found.Range
|
||||||
|
return &found, nil
|
||||||
|
}
|
||||||
|
|
||||||
// OverlayClear refuses a private-network range that overlaps an address or a route the machine
|
// OverlayClear refuses a private-network range that overlaps an address or a route the machine
|
||||||
// already has — a predecessor's tunnel still running — naming the interface. The mesh's own
|
// already has — a predecessor's tunnel still running — naming the interface. The mesh's own
|
||||||
// interface is not counted.
|
// interface is not counted.
|
||||||
@@ -399,7 +431,9 @@ func NamesFree(ctx context.Context, run Runner, names []string, known store.Stat
|
|||||||
sorted := append([]string{}, names...)
|
sorted := append([]string{}, names...)
|
||||||
sort.Strings(sorted)
|
sort.Strings(sorted)
|
||||||
for _, name := range sorted {
|
for _, name := range sorted {
|
||||||
out, err := run(ctx, "docker", "inspect", "--format",
|
// `container inspect`: a name is not unique across object kinds, and the bare form can
|
||||||
|
// resolve to a same-named network or volume instead of reporting the container absent.
|
||||||
|
out, err := run(ctx, "docker", "container", "inspect", "--format",
|
||||||
"{{index .Config.Labels \"mesh-host.spec\"}}", name)
|
"{{index .Config.Labels \"mesh-host.spec\"}}", name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue // no such container
|
continue // no such container
|
||||||
@@ -459,12 +493,26 @@ func CheckTheMachine(ctx context.Context, o Options, run Runner, bundle *declara
|
|||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
if o.Tunnel != "" {
|
||||||
|
// The hub's port is the found tunnel's, held by that tunnel until the mesh's interface
|
||||||
|
// takes it over (novox/hq ADR 0105): held by design, not by something else.
|
||||||
|
inner := ours
|
||||||
|
ours = func(r reachable.Reach) bool {
|
||||||
|
return inner(r) || (r.Protocol == "udp" && r.Port == p.Hub)
|
||||||
|
}
|
||||||
|
}
|
||||||
if err := PortsFree(ctx, run, p, ours); err != nil {
|
if err := PortsFree(ctx, run, p, ours); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
say(fmt.Sprintf(" ports free store %d, bus %d, amqp %d, management %d, registry %d, packages %d, hub %d/udp",
|
say(fmt.Sprintf(" ports free store %d, bus %d, amqp %d, management %d, registry %d, packages %d, hub %d/udp",
|
||||||
p.Store, p.Bus, p.AMQP, p.Management, p.Registry, p.Packages, p.Hub))
|
p.Store, p.Bus, p.AMQP, p.Management, p.Registry, p.Packages, p.Hub))
|
||||||
if err := OverlayClear(ctx, run, o.OverlayRange); err != nil {
|
if o.Tunnel != "" {
|
||||||
|
// One tunnel and one range: the mesh's range IS the found tunnel's, so the rule that the
|
||||||
|
// two must not overlap applies only where a found tunnel is left running beside the mesh's
|
||||||
|
// (ADR 0100, narrowed by ADR 0105).
|
||||||
|
say(fmt.Sprintf(" range %s is the tunnel %s's, taken over; not checked against it",
|
||||||
|
o.OverlayRange, o.Tunnel))
|
||||||
|
} else if err := OverlayClear(ctx, run, o.OverlayRange); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := NamesFree(ctx, run, names, known); err != nil {
|
if err := NamesFree(ctx, run, names, known); err != nil {
|
||||||
|
|||||||
@@ -2,6 +2,9 @@ package bootstrap
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/ecdh"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -12,6 +15,7 @@ import (
|
|||||||
"github.com/novox/mesh-host/internal/declaration"
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
"github.com/novox/mesh-host/internal/reachable"
|
"github.com/novox/mesh-host/internal/reachable"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
"github.com/novox/mesh-host/internal/tunnel"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Defends novox/hq ADR 0100: the foundation's ports are the node's — inputs to genesis, checked free,
|
// Defends novox/hq ADR 0100: the foundation's ports are the node's — inputs to genesis, checked free,
|
||||||
@@ -130,9 +134,9 @@ func TestTwoThingsOnOnePortAreRefused(t *testing.T) {
|
|||||||
|
|
||||||
// machineRunner answers ss, docker ps, docker inspect and ip from fixtures.
|
// machineRunner answers ss, docker ps, docker inspect and ip from fixtures.
|
||||||
type machineRunner struct {
|
type machineRunner struct {
|
||||||
ss, ps, addrs, routes string
|
ss, ps, addrs, routes, wg string
|
||||||
unlabelled map[string]bool
|
unlabelled map[string]bool
|
||||||
labelled map[string]bool
|
labelled map[string]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m machineRunner) run(_ context.Context, name string, args ...string) (string, error) {
|
func (m machineRunner) run(_ context.Context, name string, args ...string) (string, error) {
|
||||||
@@ -141,7 +145,7 @@ func (m machineRunner) run(_ context.Context, name string, args ...string) (stri
|
|||||||
return m.ss, nil
|
return m.ss, nil
|
||||||
case name == "docker" && args[0] == "ps":
|
case name == "docker" && args[0] == "ps":
|
||||||
return m.ps, nil
|
return m.ps, nil
|
||||||
case name == "docker" && args[0] == "inspect":
|
case name == "docker" && args[0] == "container":
|
||||||
n := args[len(args)-1]
|
n := args[len(args)-1]
|
||||||
if m.labelled[n] {
|
if m.labelled[n] {
|
||||||
return "abc\n", nil
|
return "abc\n", nil
|
||||||
@@ -154,6 +158,8 @@ func (m machineRunner) run(_ context.Context, name string, args ...string) (stri
|
|||||||
return m.addrs, nil
|
return m.addrs, nil
|
||||||
case name == "ip" && args[1] == "route":
|
case name == "ip" && args[1] == "route":
|
||||||
return m.routes, nil
|
return m.routes, nil
|
||||||
|
case name == "wg":
|
||||||
|
return m.wg, nil
|
||||||
}
|
}
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
@@ -295,3 +301,76 @@ func TestARerunOfGenesisFindsItsOwnPackageRegistry(t *testing.T) {
|
|||||||
t.Error("a container under the package registry's name on a fresh machine was not refused")
|
t.Error("a container under the package registry's name on a fresh machine was not refused")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0105: an adopted genesis takes over the tunnel it finds — its port is the hub's,
|
||||||
|
// its range the mesh's, and neither is refused for being held by it.
|
||||||
|
func TestAnAdoptedGenesisSettlesOnTheTunnelItFinds(t *testing.T) {
|
||||||
|
private, err := aFoundKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
conf := "[Interface]\nPrivateKey = " + private + "\nListenPort = 51900\nAddress = 192.0.2.1/24\n" +
|
||||||
|
"[Peer]\nPublicKey = PEER=\nAllowedIPs = 192.0.2.2/32\n"
|
||||||
|
tunnel.ReadFile = func(path string) ([]byte, error) {
|
||||||
|
if path == tunnel.ConfigDir+"/wg0.conf" {
|
||||||
|
return []byte(conf), nil
|
||||||
|
}
|
||||||
|
return nil, errors.New("no such file")
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { tunnel.ReadFile = os.ReadFile })
|
||||||
|
m := machineRunner{
|
||||||
|
wg: "wg0\n",
|
||||||
|
ss: "udp UNCONN 0 0 0.0.0.0:51900 0.0.0.0:*\n",
|
||||||
|
addrs: "5: wg0 inet 192.0.2.1/24 scope global wg0\n",
|
||||||
|
routes: "192.0.2.0/24 dev wg0 proto kernel scope link src 192.0.2.1\n",
|
||||||
|
}
|
||||||
|
|
||||||
|
o := Options{Adopted: true, Ports: DefaultPorts(), OverlayRange: DefaultOverlayRange, State: filepath.Join(t.TempDir(), "state.json")}
|
||||||
|
found, err := TakeTheTunnel(&o, m.run)
|
||||||
|
if err != nil || found == nil {
|
||||||
|
t.Fatalf("the tunnel was not found and taken: %+v %v", found, err)
|
||||||
|
}
|
||||||
|
if o.Tunnel != "wg0" || o.Ports.Hub != 51900 || o.OverlayRange != "192.0.2.0/24" {
|
||||||
|
t.Fatalf("genesis did not settle on the tunnel's port and range: %+v", o)
|
||||||
|
}
|
||||||
|
// Its port is held by the tunnel and its range overlaps the tunnel's — by design, not refused.
|
||||||
|
if err := CheckTheMachine(context.Background(), o, m.run, producedBundle(t).Declaration, func(string) {}); err != nil {
|
||||||
|
t.Fatalf("the machine was refused for the tunnel it takes over: %v", err)
|
||||||
|
}
|
||||||
|
// Whereas the same machine not taking it over is refused on both counts (ADR 0100).
|
||||||
|
plain := o
|
||||||
|
plain.Tunnel = ""
|
||||||
|
if err := CheckTheMachine(context.Background(), plain, m.run, producedBundle(t).Declaration, func(string) {}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "51900") {
|
||||||
|
t.Fatalf("a tunnel not taken over stopped being refused for holding the hub's port: %v", err)
|
||||||
|
}
|
||||||
|
plain.Ports.Hub = 51821
|
||||||
|
if err := CheckTheMachine(context.Background(), plain, m.run, producedBundle(t).Declaration, func(string) {}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "wg0") {
|
||||||
|
t.Fatalf("a tunnel not taken over stopped being refused for overlapping the range: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Numbers that disagree with the tunnel are refused, naming the tunnel's.
|
||||||
|
for name, given := range map[string]Options{
|
||||||
|
"--hub-port": {Adopted: true, Ports: FoundationPorts{Hub: 51821}, OverlayRange: DefaultOverlayRange},
|
||||||
|
"--overlay-range": {Adopted: true, Ports: DefaultPorts(), OverlayRange: "10.77.0.0/16"},
|
||||||
|
} {
|
||||||
|
if _, err := TakeTheTunnel(&given, m.run); err == nil || !strings.Contains(err.Error(), name) {
|
||||||
|
t.Errorf("a %s disagreeing with the tunnel was accepted: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And no tunnel up is an ordinary machine.
|
||||||
|
m.wg = "mesh0\n"
|
||||||
|
none := Options{Adopted: true, Ports: DefaultPorts(), OverlayRange: DefaultOverlayRange}
|
||||||
|
if found, err := TakeTheTunnel(&none, m.run); err != nil || found != nil || none.Ports.Hub != DefaultPorts().Hub {
|
||||||
|
t.Errorf("a machine with no tunnel was not left as it was: %+v %v", found, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func aFoundKey() (string, error) {
|
||||||
|
k, err := ecdh.X25519().GenerateKey(rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return base64.StdEncoding.EncodeToString(k.Bytes()), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -150,7 +150,7 @@ func digestOf(ctx context.Context, o Options, d Deps, remote string) (string, er
|
|||||||
|
|
||||||
// The registry has it. What digest, according to the runtime that pushed it.
|
// The registry has it. What digest, according to the runtime that pushed it.
|
||||||
reading, cancel := context.WithTimeout(ctx, o.Timeout)
|
reading, cancel := context.WithTimeout(ctx, o.Timeout)
|
||||||
out, err := d.Run(reading, "docker", "inspect", "--format", "{{json .RepoDigests}}",
|
out, err := d.Run(reading, "docker", "image", "inspect", "--format", "{{json .RepoDigests}}",
|
||||||
remote+":"+genesisTag)
|
remote+":"+genesisTag)
|
||||||
cancel()
|
cancel()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ func TestAnImageNoRegistryHasEverHeldIsPushed(t *testing.T) {
|
|||||||
case "push":
|
case "push":
|
||||||
pushed = true
|
pushed = true
|
||||||
return "", nil
|
return "", nil
|
||||||
case "inspect":
|
case "image":
|
||||||
return `["127.0.0.1:5000/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil
|
return `["127.0.0.1:5000/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("unexpected: %v", args)
|
return "", fmt.Errorf("unexpected: %v", args)
|
||||||
@@ -80,7 +80,7 @@ func TestAnImageTheRegistryAlreadyServesIsNotPushedAgain(t *testing.T) {
|
|||||||
return http.StatusOK, `{"name":"mesh-controller","tags":["genesis"]}`, nil
|
return http.StatusOK, `{"name":"mesh-controller","tags":["genesis"]}`, nil
|
||||||
},
|
},
|
||||||
func(_ string, args []string) (string, error) {
|
func(_ string, args []string) (string, error) {
|
||||||
if args[0] == "inspect" {
|
if args[0] == "image" {
|
||||||
return `["127.0.0.1:5000/mesh-controller@sha256:` + strings.Repeat("b", 64) + `"]`, nil
|
return `["127.0.0.1:5000/mesh-controller@sha256:` + strings.Repeat("b", 64) + `"]`, nil
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("unexpected: %v", args)
|
return "", fmt.Errorf("unexpected: %v", args)
|
||||||
@@ -111,7 +111,7 @@ func TestTheDigestComesFromThisMeshsOwnRegistry(t *testing.T) {
|
|||||||
return http.StatusOK, `{"tags":["genesis"]}`, nil
|
return http.StatusOK, `{"tags":["genesis"]}`, nil
|
||||||
},
|
},
|
||||||
func(_ string, args []string) (string, error) {
|
func(_ string, args []string) (string, error) {
|
||||||
if args[0] == "inspect" {
|
if args[0] == "image" {
|
||||||
return `["` + elsewhere + `","` + ours + `"]`, nil
|
return `["` + elsewhere + `","` + ours + `"]`, nil
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("unexpected: %v", args)
|
return "", fmt.Errorf("unexpected: %v", args)
|
||||||
@@ -166,7 +166,7 @@ func TestATagIsNotAPin(t *testing.T) {
|
|||||||
return http.StatusOK, `{"tags":["genesis"]}`, nil
|
return http.StatusOK, `{"tags":["genesis"]}`, nil
|
||||||
},
|
},
|
||||||
func(_ string, args []string) (string, error) {
|
func(_ string, args []string) (string, error) {
|
||||||
if args[0] == "inspect" {
|
if args[0] == "image" {
|
||||||
return `["127.0.0.1:5000/mesh-controller:genesis"]`, nil
|
return `["127.0.0.1:5000/mesh-controller:genesis"]`, nil
|
||||||
}
|
}
|
||||||
return "", nil
|
return "", nil
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ func aMeshThatAgrees(answers map[string]string) func(string, []string) (string,
|
|||||||
return "", fmt.Errorf("unexpected program %q", name)
|
return "", fmt.Errorf("unexpected program %q", name)
|
||||||
case args[0] == "cp":
|
case args[0] == "cp":
|
||||||
return "", nil
|
return "", nil
|
||||||
case args[0] == "inspect":
|
case args[0] == "container":
|
||||||
return "true running\n", nil
|
return "true running\n", nil
|
||||||
case args[0] == "exec":
|
case args[0] == "exec":
|
||||||
return "", nil
|
return "", nil
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ func RetireTheTemporaryControlPlane(ctx context.Context, o Options, sys system.S
|
|||||||
// same state file. What makes this a removal rather than a no-op is that the state file
|
// same state file. What makes this a removal rather than a no-op is that the state file
|
||||||
// records the container as something this installer applied, and the declaration no longer
|
// records the container as something this installer applied, and the declaration no longer
|
||||||
// asks for it.
|
// asks for it.
|
||||||
report, err := ApplyBundle(ctx, o, sys, without, run, say)
|
report, err := ApplyBundle(ctx, o, sys, without, bundle, run, say)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return out, fmt.Errorf(
|
return out, fmt.Errorf(
|
||||||
"%w\n\nThe permanent control plane is running and the temporary one is still here. "+
|
"%w\n\nThe permanent control plane is running and the temporary one is still here. "+
|
||||||
|
|||||||
@@ -129,8 +129,9 @@ func containerRunning(ctx context.Context, run Runner, probe time.Duration, name
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
// Both facts in one answer, so a container that is not running is reported with what it IS
|
// Both facts in one answer, so a container that is not running is reported with what it IS
|
||||||
// rather than with the absence of what it should be.
|
// rather than with the absence of what it should be. `container inspect`, not the bare form:
|
||||||
out, err := run(asking, "docker", "inspect", "--format", "{{.State.Running}} {{.State.Status}}", name)
|
// a same-named network or volume would otherwise answer in the container's place.
|
||||||
|
out, err := run(asking, "docker", "container", "inspect", "--format", "{{.State.Running}} {{.State.Status}}", name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return containerState{}, fmt.Errorf(
|
return containerState{}, fmt.Errorf(
|
||||||
"the container %q is not there at all, and the apply reported it applied: %w", name, err)
|
"the container %q is not there at all, and the apply reported it applied: %w", name, err)
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ func TestAContainerThatIsUpIsNotAControlPlaneThatReplies(t *testing.T) {
|
|||||||
|
|
||||||
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "inspect":
|
case "container":
|
||||||
return "true running\n", nil
|
return "true running\n", nil
|
||||||
case "exec":
|
case "exec":
|
||||||
// Up, and saying nothing. The program inside is not answering.
|
// Up, and saying nothing. The program inside is not answering.
|
||||||
@@ -59,7 +59,7 @@ func TestAControlPlaneThatSaysNothingHasNotAnswered(t *testing.T) {
|
|||||||
defer func() { answerEvery = previous }()
|
defer func() { answerEvery = previous }()
|
||||||
|
|
||||||
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
||||||
if args[0] == "inspect" {
|
if args[0] == "container" {
|
||||||
return "true running\n", nil
|
return "true running\n", nil
|
||||||
}
|
}
|
||||||
return " \n", nil
|
return " \n", nil
|
||||||
@@ -74,7 +74,7 @@ func TestAControlPlaneThatSaysNothingHasNotAnswered(t *testing.T) {
|
|||||||
// The foundation answering is the whole point, and what it said is reported rather than asserted.
|
// The foundation answering is the whole point, and what it said is reported rather than asserted.
|
||||||
func TestAFoundationThatIsUpAndAnsweringIsAccepted(t *testing.T) {
|
func TestAFoundationThatIsUpAndAnsweringIsAccepted(t *testing.T) {
|
||||||
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
||||||
if args[0] == "inspect" {
|
if args[0] == "container" {
|
||||||
return "true running\n", nil
|
return "true running\n", nil
|
||||||
}
|
}
|
||||||
return "1 node, 0 waiting\n", nil
|
return "1 node, 0 waiting\n", nil
|
||||||
@@ -112,7 +112,7 @@ func TestAControlPlaneThatIsStillStartingIsWaitedFor(t *testing.T) {
|
|||||||
|
|
||||||
attempts := 0
|
attempts := 0
|
||||||
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
||||||
if args[0] == "inspect" {
|
if args[0] == "container" {
|
||||||
return "true running\n", nil
|
return "true running\n", nil
|
||||||
}
|
}
|
||||||
attempts++
|
attempts++
|
||||||
@@ -132,10 +132,10 @@ func TestAControlPlaneThatIsStillStartingIsWaitedFor(t *testing.T) {
|
|||||||
// than being told only that something is not what it should be.
|
// than being told only that something is not what it should be.
|
||||||
func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) {
|
func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) {
|
||||||
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
||||||
if args[0] == "inspect" && args[len(args)-1] == "mesh-broker" {
|
if args[0] == "container" && args[len(args)-1] == "mesh-broker" {
|
||||||
return "false exited\n", nil
|
return "false exited\n", nil
|
||||||
}
|
}
|
||||||
if args[0] == "inspect" {
|
if args[0] == "container" {
|
||||||
return "true running\n", nil
|
return "true running\n", nil
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("unexpected command: %v", args)
|
return "", fmt.Errorf("unexpected command: %v", args)
|
||||||
@@ -155,7 +155,7 @@ func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) {
|
|||||||
// `FROM scratch` and has no shell for a command line to be interpreted by.
|
// `FROM scratch` and has no shell for a command line to be interpreted by.
|
||||||
func TestTheControlPlaneIsAskedByRunningItsOwnBinary(t *testing.T) {
|
func TestTheControlPlaneIsAskedByRunningItsOwnBinary(t *testing.T) {
|
||||||
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
||||||
if args[0] == "inspect" {
|
if args[0] == "container" {
|
||||||
return "true running\n", nil
|
return "true running\n", nil
|
||||||
}
|
}
|
||||||
return "1 node\n", nil
|
return "1 node\n", nil
|
||||||
|
|||||||
@@ -105,3 +105,28 @@ func TestACarriedBundleCannotSayTheNodeIsAdopted(t *testing.T) {
|
|||||||
t.Fatalf("a bundle claiming adoption was not refused: %v", err)
|
t.Fatalf("a bundle claiming adoption was not refused: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0105: a service may take over a found tunnel, said whole and on an adopted node.
|
||||||
|
func TestTakingOverATunnelIsSaidWholeAndForARunningService(t *testing.T) {
|
||||||
|
adoptedWith := func(service string) error {
|
||||||
|
_, err := Parse([]byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[` + service + `]}`))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
good := `{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running",
|
||||||
|
"takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}`
|
||||||
|
if err := adoptedWith(good); err != nil {
|
||||||
|
t.Fatalf("a whole takeover on an adopted node was refused: %v", err)
|
||||||
|
}
|
||||||
|
for name, bad := range map[string]string{
|
||||||
|
"its own unit": `{"id":"up","type":"service","unit":"wg-quick@wg0","state":"running",
|
||||||
|
"takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}`,
|
||||||
|
"no config": `{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running",
|
||||||
|
"takes-over":{"interface":"wg0","unit":"wg-quick@wg0"}}`,
|
||||||
|
"a stopped service": `{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"stopped",
|
||||||
|
"takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}`,
|
||||||
|
} {
|
||||||
|
if err := adoptedWith(bad); err == nil {
|
||||||
|
t.Errorf("a takeover naming %s was accepted", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
package declaration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq issue 128: a file written into a block carries only its lines, in content,
|
||||||
|
// never a line the host keeps as its own marker, and says where a new region goes only as a block.
|
||||||
|
|
||||||
|
func TestAFileWrittenIntoABlockIsRefusedUnlessItIsOnlyItsLines(t *testing.T) {
|
||||||
|
for name, c := range map[string]struct{ resource, refusal string }{
|
||||||
|
"a begin marker in content": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a\n# BEGIN mesh f\nb\n"}`, "# BEGIN mesh f"},
|
||||||
|
"an end marker in content": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a\n# END mesh other\n"}`, "# END mesh other"},
|
||||||
|
"a marker with a CR": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"# BEGIN mesh x\r\n"}`, "marker"},
|
||||||
|
"sealed": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","sealed":"abc"}`, "not sealed"},
|
||||||
|
"bytes": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","bytes":"YQ=="}`, "not sealed, bytes"},
|
||||||
|
"secrets": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"${secret:s}","secrets":{"s":"abc"}}`, "secrets"},
|
||||||
|
"create-once": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a","create-once":true}`, "create-once"},
|
||||||
|
"at on a whole file": {`{"id":"f","type":"file","path":"/etc/hosts","content":"a","at":"start"}`, `at "start"`},
|
||||||
|
"at on a JSON file": {`{"id":"f","type":"file","path":"/etc/x.json","into":"json","content":"{}","at":"end"}`, `at "end"`},
|
||||||
|
"at somewhere else": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a","at":"middle"}`, `"start" or "end"`},
|
||||||
|
"an id ending in a space": {`{"id":"f ","type":"file","path":"/etc/hosts","into":"block","content":"a"}`, "whitespace"},
|
||||||
|
"an unknown format": {`{"id":"f","type":"file","path":"/etc/hosts","into":"lines","content":"a"}`, `"json" or "block"`},
|
||||||
|
} {
|
||||||
|
_, err := Parse([]byte(`{"declaration":1,"resources":[` + c.resource + `]}`))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), c.refusal) {
|
||||||
|
t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFileWrittenIntoABlockIsRead(t *testing.T) {
|
||||||
|
d, err := Parse([]byte(`{"declaration":1,"resources":[
|
||||||
|
{"id":"mesh-wireguard.fact-node-names","type":"file","path":"/etc/hosts","into":"block","content":"10.42.0.1 ace\n# BEGIN devtool x\n"},
|
||||||
|
{"id":"dhcpcd.options","type":"file","path":"/etc/dhcpcd.conf","into":"block","content":"nohook resolv.conf\n","at":"start"},
|
||||||
|
{"id":"empty","type":"file","path":"/etc/x","into":"block","content":"","at":"end"}
|
||||||
|
]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f := d.Resources[0].(*File); f.Into != IntoBlock || f.At != "" {
|
||||||
|
t.Errorf("read as into %q at %q", f.Into, f.At)
|
||||||
|
}
|
||||||
|
if f := d.Resources[1].(*File); f.At != AtStart {
|
||||||
|
t.Errorf("at was read as %q", f.At)
|
||||||
|
}
|
||||||
|
if begin, end := BlockMarkers("mesh-wireguard.fact-node-names"); begin != "# BEGIN mesh mesh-wireguard.fact-node-names" ||
|
||||||
|
end != "# END mesh mesh-wireguard.fact-node-names" {
|
||||||
|
t.Errorf("the markers are %q and %q", begin, end)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,10 +11,12 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net"
|
||||||
"reflect"
|
"reflect"
|
||||||
"regexp"
|
"regexp"
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -159,8 +161,30 @@ type File struct {
|
|||||||
// "json" is spoken — the content is a JSON object whose keys the host sets in the file's
|
// "json" is spoken — the content is a JSON object whose keys the host sets in the file's
|
||||||
// object, keeping every other key as it found it and recording what each of its keys held
|
// object, keeping every other key as it found it and recording what each of its keys held
|
||||||
// before, so undeclaring the file gives those back.
|
// before, so undeclaring the file gives those back.
|
||||||
|
//
|
||||||
|
// "block" is the same idea for a file that is not structured (novox/hq issue 128): the
|
||||||
|
// content is the mesh's lines, and the host owns only the region between `# BEGIN mesh <id>`
|
||||||
|
// and `# END mesh <id>`, keeping every line outside it byte for byte. The machine's hosts file
|
||||||
|
// is the case that needed it — on a workstation the distribution, a local development tool and
|
||||||
|
// the operator all write into it, and the mesh writing it whole took their lines away at the
|
||||||
|
// next change to the mesh's names, silently. Marked blocks are the shape the other tools in
|
||||||
|
// that file already use, and `#` is the comment character of every file this serves.
|
||||||
|
//
|
||||||
|
// Written into, in either format, the file's mode and owner are the machine's: a declared mode
|
||||||
|
// and owner apply only to a file the host creates, and a file that was there keeps its own.
|
||||||
Into string `json:"into,omitempty"`
|
Into string `json:"into,omitempty"`
|
||||||
|
|
||||||
|
// At is where a file written into a block has its region added when the file does not hold
|
||||||
|
// one yet: "end", the default, or "start". A region already there stays where it is, whatever
|
||||||
|
// this says — moving it would move the lines around it, and those are the machine's.
|
||||||
|
//
|
||||||
|
// **Some files give a line its meaning by what stands above it.** dhcpcd's configuration scopes
|
||||||
|
// every line after `interface X` to that interface, and a real one ends with exactly that — an
|
||||||
|
// interface and its static address. A region added at the end would make the mesh's global
|
||||||
|
// options (`nohook resolv.conf`, `denyinterfaces mesh0`) options of one interface, and dhcpcd
|
||||||
|
// would read them without complaint. At the start, nothing stands above them.
|
||||||
|
At string `json:"at,omitempty"`
|
||||||
|
|
||||||
// Sealed is content encrypted to this node's sealing key, for a file the mesh must deliver
|
// Sealed is content encrypted to this node's sealing key, for a file the mesh must deliver
|
||||||
// without being able to read.
|
// without being able to read.
|
||||||
//
|
//
|
||||||
@@ -239,16 +263,50 @@ func (f *File) validate(where string, _ bool) []string {
|
|||||||
problems = append(problems, where+
|
problems = append(problems, where+
|
||||||
": a file written into JSON carries a JSON object of the keys it sets")
|
": a file written into JSON carries a JSON object of the keys it sets")
|
||||||
}
|
}
|
||||||
if f.Sealed != "" || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce {
|
case IntoBlock:
|
||||||
|
// The markers are how the host finds its region again. A marker in the content would be
|
||||||
|
// a second region, or the end of this one, the next time the file is read — and the host
|
||||||
|
// would then rewrite, or on undeclare take out, lines that were never the mesh's.
|
||||||
|
for _, line := range strings.Split(f.Content, "\n") {
|
||||||
|
if strings.HasPrefix(line, BlockBegin) || strings.HasPrefix(line, BlockEnd) {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s: a file written into a block carries the mesh's lines, and %q is a marker the "+
|
||||||
|
"host keeps for itself", where, strings.TrimRight(line, "\r")))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The id is written into the markers, so it has to stay on one line — and whitespace at
|
||||||
|
// either end of it is whitespace the host would have to match exactly in a line some
|
||||||
|
// editor may trim.
|
||||||
|
if strings.ContainsAny(f.ID, "\r\n") {
|
||||||
problems = append(problems, where+
|
problems = append(problems, where+
|
||||||
": a file written into says only its keys, in content — not sealed, bytes, "+
|
": a file written into a block names its region by its id, and this id spans lines")
|
||||||
"secrets or create-once")
|
} else if strings.TrimSpace(f.ID) != f.ID {
|
||||||
|
problems = append(problems, where+
|
||||||
|
": a file written into a block names its region by its id, and this id begins or ends in whitespace")
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
problems = append(problems, fmt.Sprintf(
|
problems = append(problems, fmt.Sprintf(
|
||||||
"%s: into %q; a file is written into \"json\", or omits it to be written whole",
|
"%s: into %q; a file is written into \"json\" or \"block\", or omits it to be written whole",
|
||||||
where, f.Into))
|
where, f.Into))
|
||||||
}
|
}
|
||||||
|
switch {
|
||||||
|
case f.At == "":
|
||||||
|
case f.Into != IntoBlock:
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s: at %q; only a file written into a block has a place its region is added", where, f.At))
|
||||||
|
case f.At != AtStart && f.At != AtEnd:
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s: at %q; a block is added at \"start\" or \"end\", or omits it to be added at the end",
|
||||||
|
where, f.At))
|
||||||
|
}
|
||||||
|
if f.Into == IntoJSON || f.Into == IntoBlock {
|
||||||
|
if f.Sealed != "" || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce {
|
||||||
|
problems = append(problems, where+
|
||||||
|
": a file written into says only its part, in content — not sealed, bytes, "+
|
||||||
|
"secrets or create-once")
|
||||||
|
}
|
||||||
|
}
|
||||||
var said []string
|
var said []string
|
||||||
for name, value := range map[string]string{
|
for name, value := range map[string]string{
|
||||||
"content": f.Content, "sealed": f.Sealed, "bytes": f.Bytes,
|
"content": f.Content, "sealed": f.Sealed, "bytes": f.Bytes,
|
||||||
@@ -517,16 +575,34 @@ func (d *Process) Identity() string { return d.ID }
|
|||||||
func (d *Process) Kind() Type { return TypeProcess }
|
func (d *Process) Kind() Type { return TypeProcess }
|
||||||
func (d *Process) Target() string { return d.Name }
|
func (d *Process) Target() string { return d.Name }
|
||||||
|
|
||||||
|
// ProcessNameProblem says what is wrong with a process name, or nothing.
|
||||||
|
//
|
||||||
|
// The name becomes a unit name, a file under the unit directory and a directory under the mesh's
|
||||||
|
// own — the one removing the process deletes, whole (novox/hq ADR 0118). So a name that is not one
|
||||||
|
// plain path element is refused: with a separator it writes somewhere nobody meant, and "." or
|
||||||
|
// ".." IS the mesh's directory or its parent — removing a process named ".." would delete every
|
||||||
|
// bundle the mesh has, and more. One with a leading dash is read by the service manager as an
|
||||||
|
// option, not a unit. Exported because the removal checks the recorded name again: a record is
|
||||||
|
// what the host wrote, and a host of an older version wrote it under looser rules.
|
||||||
|
func ProcessNameProblem(name string) string {
|
||||||
|
switch {
|
||||||
|
case name == "":
|
||||||
|
return "a process needs a name, which is what its unit is called"
|
||||||
|
case strings.ContainsAny(name, "/ \t"):
|
||||||
|
return "a process name becomes a unit name, so it cannot contain a path separator or a space"
|
||||||
|
case name == "." || name == "..":
|
||||||
|
return fmt.Sprintf("a process name becomes a directory under the mesh's own, and %q would be "+
|
||||||
|
"that directory or its parent", name)
|
||||||
|
case strings.HasPrefix(name, "-"):
|
||||||
|
return "a process name cannot begin with a dash: the service manager would read it as an option"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
func (d *Process) validate(where string, _ bool) []string {
|
func (d *Process) validate(where string, _ bool) []string {
|
||||||
var problems []string
|
var problems []string
|
||||||
if d.Name == "" {
|
if problem := ProcessNameProblem(d.Name); problem != "" {
|
||||||
problems = append(problems, where+": a process needs a name, which is what its unit is called")
|
problems = append(problems, where+": "+problem)
|
||||||
}
|
|
||||||
if strings.ContainsAny(d.Name, "/ \t") {
|
|
||||||
// It becomes a unit name and a file on disk. A name with a separator in it would write
|
|
||||||
// somewhere nobody meant.
|
|
||||||
problems = append(problems, where+": a process name becomes a unit name, so it cannot "+
|
|
||||||
"contain a path separator or a space")
|
|
||||||
}
|
}
|
||||||
if d.Source == "" {
|
if d.Source == "" {
|
||||||
problems = append(problems, where+": a process needs somewhere to fetch its bundle from")
|
problems = append(problems, where+": a process needs somewhere to fetch its bundle from")
|
||||||
@@ -587,10 +663,20 @@ func (d *Process) validate(where string, _ bool) []string {
|
|||||||
// (it will come back at boot), or disabled and running (started by hand, gone after a reboot).
|
// (it will come back at boot), or disabled and running (started by hand, gone after a reboot).
|
||||||
// Folding them into one field would make the second expressible only by accident.
|
// Folding them into one field would make the second expressible only by accident.
|
||||||
type Service struct {
|
type Service struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Type Type `json:"type"`
|
Type Type `json:"type"`
|
||||||
Unit string `json:"unit"`
|
Unit string `json:"unit"`
|
||||||
State string `json:"state"`
|
// State is "running" or "stopped" — or absent, and then **the unit's lifecycle is the
|
||||||
|
// machine's; the mesh only reflects its triggers** (novox/hq ADR 0117). The uplink modules
|
||||||
|
// declare the machine's own network manager this way: the mesh writes into its configuration
|
||||||
|
// and needs it to read that again, and nothing more. Stated, the host would start the manager
|
||||||
|
// on a machine that uses another one — two managers fighting over the same links — and, when
|
||||||
|
// the module was unassigned, stop it: the machine's network, the channel the mesh itself
|
||||||
|
// arrives on, gone at the moment of a routine change. So a service without a state is never
|
||||||
|
// started, stopped, enabled or disabled, is reloaded or restarted only when a trigger changed
|
||||||
|
// and it is already running, and undeclared is simply forgotten. It says nothing unless it
|
||||||
|
// names a trigger, and it may not say boot or takes-over, which are both lifecycle.
|
||||||
|
State string `json:"state,omitempty"`
|
||||||
// Boot is "enabled" or "disabled" — whether the unit starts at boot. Optional: absent means
|
// Boot is "enabled" or "disabled" — whether the unit starts at boot. Optional: absent means
|
||||||
// the host asserts nothing about it and leaves whatever is there.
|
// the host asserts nothing about it and leaves whatever is there.
|
||||||
//
|
//
|
||||||
@@ -617,8 +703,28 @@ type Service struct {
|
|||||||
// container runtime, whose restart stops every container on the machine (novox/hq ADR 0102).
|
// container runtime, whose restart stops every container on the machine (novox/hq ADR 0102).
|
||||||
// A change that is also in RestartOn restarts it, which covers a reload.
|
// A change that is also in RestartOn restarts it, which covers a reload.
|
||||||
ReloadOn []string `json:"reload-on,omitempty"`
|
ReloadOn []string `json:"reload-on,omitempty"`
|
||||||
|
|
||||||
|
// TakesOver names the found tunnel this service replaces (novox/hq ADR 0105): before this unit
|
||||||
|
// is started, the named unit is stopped and disabled — never flushed — and its configuration
|
||||||
|
// file is kept like any held file — until the take is proven by a peer's handshake, and then
|
||||||
|
// retired, its original staying kept (novox/hq ADR 0119). Only on an adopted node, and only
|
||||||
|
// said by the controller, which knows the found tunnel's key is this node's own: without that,
|
||||||
|
// starting this unit on the found one's port would drop every peer's packets.
|
||||||
|
TakesOver *TakeOver `json:"takes-over,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TakeOver is a found tunnel a service replaces: its interface, the unit that raised it, and its
|
||||||
|
// configuration file.
|
||||||
|
type TakeOver struct {
|
||||||
|
Interface string `json:"interface"`
|
||||||
|
Unit string `json:"unit"`
|
||||||
|
Config string `json:"config"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stateless reports whether the unit's lifecycle is the machine's, and the mesh only reflects the
|
||||||
|
// service's triggers (novox/hq ADR 0117).
|
||||||
|
func (s *Service) Stateless() bool { return s.State == "" }
|
||||||
|
|
||||||
func (s *Service) Identity() string { return s.ID }
|
func (s *Service) Identity() string { return s.ID }
|
||||||
func (s *Service) Kind() Type { return TypeService }
|
func (s *Service) Kind() Type { return TypeService }
|
||||||
func (s *Service) Target() string { return s.Unit }
|
func (s *Service) Target() string { return s.Unit }
|
||||||
@@ -628,20 +734,65 @@ func (s *Service) validate(where string, _ bool) []string {
|
|||||||
if s.Unit == "" {
|
if s.Unit == "" {
|
||||||
problems = append(problems, where+": a service needs a unit")
|
problems = append(problems, where+": a service needs a unit")
|
||||||
}
|
}
|
||||||
if s.State != "running" && s.State != "stopped" {
|
switch {
|
||||||
|
case s.State == "running" || s.State == "stopped":
|
||||||
|
case s.State != "":
|
||||||
problems = append(problems, fmt.Sprintf(
|
problems = append(problems, fmt.Sprintf(
|
||||||
"%s: state %q; a service is \"running\" or \"stopped\"", where, s.State))
|
"%s: state %q; a service is \"running\" or \"stopped\", or omits state to leave the "+
|
||||||
|
"unit's lifecycle to the machine", where, s.State))
|
||||||
|
case s.Boot != "" || s.TakesOver != nil:
|
||||||
|
problems = append(problems, where+": a service that omits state leaves the unit's lifecycle "+
|
||||||
|
"to the machine, and boot and takes-over are both its lifecycle")
|
||||||
|
case len(s.RestartOn) == 0 && len(s.ReloadOn) == 0:
|
||||||
|
problems = append(problems, where+": a service that omits state leaves the unit's lifecycle "+
|
||||||
|
"to the machine, and names no restart-on or reload-on — it declares nothing")
|
||||||
}
|
}
|
||||||
if s.Boot != "" && s.Boot != "enabled" && s.Boot != "disabled" {
|
if s.Boot != "" && s.Boot != "enabled" && s.Boot != "disabled" {
|
||||||
problems = append(problems, fmt.Sprintf(
|
problems = append(problems, fmt.Sprintf(
|
||||||
"%s: boot %q; a service is \"enabled\" or \"disabled\" at boot, or omits it to "+
|
"%s: boot %q; a service is \"enabled\" or \"disabled\" at boot, or omits it to "+
|
||||||
"leave the machine's own setting alone", where, s.Boot))
|
"leave the machine's own setting alone", where, s.Boot))
|
||||||
}
|
}
|
||||||
|
if t := s.TakesOver; t != nil {
|
||||||
|
switch {
|
||||||
|
case t.Unit == "" || t.Config == "" || t.Interface == "":
|
||||||
|
problems = append(problems, where+": takes-over names the found tunnel's interface, unit "+
|
||||||
|
"and config, and this leaves one out")
|
||||||
|
case t.Unit == s.Unit:
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: takes-over names %s, which is this service's own unit",
|
||||||
|
where, t.Unit))
|
||||||
|
case s.State != "running" && s.State != "":
|
||||||
|
problems = append(problems, where+": a service that takes over a tunnel is running — stopping "+
|
||||||
|
"the found one for a service that will not run would leave the peers with nothing")
|
||||||
|
}
|
||||||
|
}
|
||||||
return problems
|
return problems
|
||||||
}
|
}
|
||||||
|
|
||||||
// IntoJSON is the one structured format a file is written into.
|
// What a file is written into (novox/hq ADR 0102): a JSON object whose keys the mesh sets, or a
|
||||||
const IntoJSON = "json"
|
// text file in which the mesh owns one marked block of lines (novox/hq issue 128).
|
||||||
|
const (
|
||||||
|
IntoJSON = "json"
|
||||||
|
IntoBlock = "block"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The lines that delimit the mesh's region in a file written into a block, each followed by the
|
||||||
|
// resource's id. Exact lines, never patterns: another tool's `# BEGIN …` block in the same file is
|
||||||
|
// that tool's, and a marker that merely resembled the mesh's must not be taken for it.
|
||||||
|
const (
|
||||||
|
BlockBegin = "# BEGIN mesh "
|
||||||
|
BlockEnd = "# END mesh "
|
||||||
|
)
|
||||||
|
|
||||||
|
// Where a file written into a block has its region added, when it has none yet.
|
||||||
|
const (
|
||||||
|
AtStart = "start"
|
||||||
|
AtEnd = "end"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BlockMarkers are the two lines, without their line ends, that delimit a resource's region.
|
||||||
|
func BlockMarkers(id string) (begin, end string) {
|
||||||
|
return BlockBegin + id, BlockEnd + id
|
||||||
|
}
|
||||||
|
|
||||||
// Opening is a port reachable on an adopted node, from where, and on which path.
|
// Opening is a port reachable on an adopted node, from where, and on which path.
|
||||||
//
|
//
|
||||||
@@ -779,6 +930,23 @@ type Container struct {
|
|||||||
// worse failure mode.
|
// worse failure mode.
|
||||||
Network string `json:"network,omitempty"`
|
Network string `json:"network,omitempty"`
|
||||||
|
|
||||||
|
// Dns is the resolvers this container asks, passed to the runtime unchanged.
|
||||||
|
//
|
||||||
|
// **Because some software refuses to run behind the runtime's forwarding resolver.** A mail
|
||||||
|
// server's admin demands a DNSSEC-validating resolver, and the runtime's own (127.0.0.11)
|
||||||
|
// forwards to whatever the machine has — so a module that ships its own validating resolver
|
||||||
|
// must be able to point its other containers at it. Addresses, not names: the runtime's flag
|
||||||
|
// takes only addresses, which is also why IP below exists — the resolver has to be somewhere
|
||||||
|
// its siblings can name before any of them can resolve anything.
|
||||||
|
Dns []string `json:"dns,omitempty"`
|
||||||
|
|
||||||
|
// IP is this container's address on its network, passed to the runtime unchanged.
|
||||||
|
//
|
||||||
|
// Only meaningful on a user-defined network, and refused by the runtime elsewhere. Exists for
|
||||||
|
// exactly one shape: a container others must reach *before* name resolution works — a
|
||||||
|
// module's own DNS resolver being the case that forced it (see Dns).
|
||||||
|
IP string `json:"ip,omitempty"`
|
||||||
|
|
||||||
// RestartOn names resources whose change means this container must be recreated — the same
|
// RestartOn names resources whose change means this container must be recreated — the same
|
||||||
// field a service has, for the same reason (novox/hq 04-ISSUES/009). A container reads a
|
// field a service has, for the same reason (novox/hq 04-ISSUES/009). A container reads a
|
||||||
// mounted file once at start; a changed file leaves the running process holding the old value,
|
// mounted file once at start; a changed file leaves the running process holding the old value,
|
||||||
@@ -847,6 +1015,23 @@ func (c *Container) validate(where string, _ bool) []string {
|
|||||||
problems = append(problems, where+": "+err.Error())
|
problems = append(problems, where+": "+err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The runtime's flags take addresses, and a name here would be handed to it verbatim and
|
||||||
|
// refused at create — after the old container was already removed. Refused on arrival instead.
|
||||||
|
for _, d := range c.Dns {
|
||||||
|
if net.ParseIP(d) == nil {
|
||||||
|
problems = append(problems, where+": dns "+strconv.Quote(d)+" is not an address; "+
|
||||||
|
"the runtime's resolver flag takes only addresses")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.IP != "" {
|
||||||
|
if net.ParseIP(c.IP) == nil {
|
||||||
|
problems = append(problems, where+": ip "+strconv.Quote(c.IP)+" is not an address")
|
||||||
|
}
|
||||||
|
if c.Network == "" {
|
||||||
|
problems = append(problems, where+": an ip needs a network; the runtime refuses a "+
|
||||||
|
"static address anywhere but a user-defined one")
|
||||||
|
}
|
||||||
|
}
|
||||||
return append(problems, checkImage(where, c.Image)...)
|
return append(problems, checkImage(where, c.Image)...)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1078,10 +1263,17 @@ func ParseTrusted(raw []byte) (*Declaration, error) { return parse(raw, true) }
|
|||||||
// first pass takes the envelope and each resource's bytes; the second decodes each one into
|
// first pass takes the envelope and each resource's bytes; the second decodes each one into
|
||||||
// the struct for its kind, strictly.
|
// the struct for its kind, strictly.
|
||||||
type envelope struct {
|
type envelope struct {
|
||||||
Version int `json:"declaration"`
|
Version int `json:"declaration"`
|
||||||
For string `json:"for,omitempty"`
|
For string `json:"for,omitempty"`
|
||||||
Adoption *Adoption `json:"adoption,omitempty"`
|
Adoption *Adoption `json:"adoption,omitempty"`
|
||||||
Resources []json.RawMessage `json:"resources"`
|
// OwnsNothing is the control plane saying, explicitly, that this node's declaration is empty
|
||||||
|
// on purpose — it owns nothing the mesh put there (novox/hq issue 127). Without it an empty
|
||||||
|
// resources list is refused as a likely mistake; with it the node applies the empty
|
||||||
|
// declaration and drops what it last held. The two are distinguished because a truncated or
|
||||||
|
// mis-composed body arrives as empty too, and a host that could not tell them apart would let
|
||||||
|
// a bug quietly strip a machine.
|
||||||
|
OwnsNothing bool `json:"owns_nothing,omitempty"`
|
||||||
|
Resources []json.RawMessage `json:"resources"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
||||||
@@ -1101,7 +1293,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
|||||||
d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption}
|
d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption}
|
||||||
var problems []string
|
var problems []string
|
||||||
|
|
||||||
if len(env.Resources) == 0 {
|
if len(env.Resources) == 0 && !env.OwnsNothing {
|
||||||
problems = append(problems, "no resources. An empty declaration is a mistake, not a "+
|
problems = append(problems, "no resources. An empty declaration is a mistake, not a "+
|
||||||
"machine with nothing on it — say so with an explicit empty list if that is meant")
|
"machine with nothing on it — say so with an explicit empty list if that is meant")
|
||||||
}
|
}
|
||||||
@@ -1172,6 +1364,14 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
|||||||
"resource %q: an opening is for an adopted node, and this declaration does not "+
|
"resource %q: an opening is for an adopted node, and this declaration does not "+
|
||||||
"say the node is adopted", r.Identity()))
|
"say the node is adopted", r.Identity()))
|
||||||
}
|
}
|
||||||
|
if svc, ok := r.(*Service); ok && svc.TakesOver != nil {
|
||||||
|
// A tunnel is taken over on an adopted node, where what is found is kept: on a
|
||||||
|
// converged one there is nothing found to take over, and stopping a unit the
|
||||||
|
// mesh did not declare would be the host deciding (novox/hq ADR 0105).
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"resource %q: taking over a tunnel is for an adopted node, and this declaration "+
|
||||||
|
"does not say the node is adopted", r.Identity()))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -427,3 +427,40 @@ func TestASecretTheContentNeverUsesIsRefused(t *testing.T) {
|
|||||||
t.Fatal("a secret the content never mentions was accepted")
|
t.Fatal("a secret the content never mentions was accepted")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The runtime's resolver and address flags take only addresses; a name would be refused at
|
||||||
|
// create, after the old container was already gone. Refused on arrival instead — and an address
|
||||||
|
// without a user-defined network is refused for the same reason.
|
||||||
|
func TestAContainersResolverAndAddressAreAddressesOrRefused(t *testing.T) {
|
||||||
|
refused := func(body string) []string {
|
||||||
|
_, err := Parse([]byte(`{"declaration":1,"resources":[` + body + `]}`))
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []string{err.Error()}
|
||||||
|
}
|
||||||
|
base := `"id":"c","type":"container","name":"x",` +
|
||||||
|
`"image":"a@sha256:0000000000000000000000000000000000000000000000000000000000000000"`
|
||||||
|
if p := refused(`{` + base + `,"network":"m","dns":["resolver.local"]}`); len(p) == 0 {
|
||||||
|
t.Error("a resolver named by name was accepted; the runtime takes only addresses")
|
||||||
|
}
|
||||||
|
if p := refused(`{` + base + `,"ip":"192.168.203.7"}`); len(p) == 0 {
|
||||||
|
t.Error("a static address with no network was accepted; the runtime refuses it")
|
||||||
|
}
|
||||||
|
if p := refused(`{` + base + `,"network":"m","dns":["192.168.203.254"],"ip":"192.168.203.7"}`); len(p) != 0 {
|
||||||
|
t.Errorf("a well-formed resolver and address were refused: %v", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnExplicitlyEmptyDeclarationIsAccepted(t *testing.T) {
|
||||||
|
// A deliberately-empty declaration (novox/hq issue 127) says owns_nothing, and is applied so
|
||||||
|
// the node drops what it last held — distinct from an accidental empty body, which is refused.
|
||||||
|
if _, err := Parse([]byte(`{"declaration":1,"owns_nothing":true,"resources":[]}`)); err != nil {
|
||||||
|
t.Fatalf("an explicitly-empty declaration must be accepted: %v", err)
|
||||||
|
}
|
||||||
|
// Without the marker, an empty declaration is still refused as a likely mistake.
|
||||||
|
_, err := Parse([]byte(`{"declaration":1,"resources":[]}`))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "no resources") {
|
||||||
|
t.Fatalf("an unmarked empty declaration must still be refused; got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -55,7 +55,10 @@ func TestAProcessMustSayWhatToRun(t *testing.T) {
|
|||||||
|
|
||||||
// Its name becomes a unit name and a path, so a separator in it would write somewhere nobody meant.
|
// Its name becomes a unit name and a path, so a separator in it would write somewhere nobody meant.
|
||||||
func TestAProcesssNameCannotEscapeItsUnit(t *testing.T) {
|
func TestAProcesssNameCannotEscapeItsUnit(t *testing.T) {
|
||||||
for _, bad := range []string{"", "../escape", "two words", "a/b"} {
|
// "." and ".." are one path element each, and the mesh's own bundle directory and its parent:
|
||||||
|
// removing a process named ".." would delete every bundle the mesh has, and more (novox/hq ADR
|
||||||
|
// 0118). A leading dash is an option to the service manager, not a unit.
|
||||||
|
for _, bad := range []string{"", "../escape", "two words", "a/b", ".", "..", "-", "--now"} {
|
||||||
d := aProcess()
|
d := aProcess()
|
||||||
d.Name = bad
|
d.Name = bad
|
||||||
if problems := d.validate("a process", false); len(problems) == 0 {
|
if problems := d.validate("a process", false); len(problems) == 0 {
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
package declaration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq ADR 0117: a service may leave its unit's lifecycle to the machine, and then
|
||||||
|
// says nothing but its triggers.
|
||||||
|
func TestAServiceWithoutAStateSaysOnlyItsTriggers(t *testing.T) {
|
||||||
|
for name, c := range map[string]struct{ resource, refusal string }{
|
||||||
|
"no trigger": {`{"id":"s","type":"service","unit":"NetworkManager.service"}`, "declares nothing"},
|
||||||
|
"with boot": {`{"id":"s","type":"service","unit":"NetworkManager.service","boot":"enabled","reload-on":["f"]}`, "boot and takes-over"},
|
||||||
|
"with takes-over": {`{"id":"s","type":"service","unit":"a.service","reload-on":["f"],"takes-over":{"interface":"wg0","unit":"b.service","config":"/etc/x"}}`, "boot and takes-over"},
|
||||||
|
"an unknown state": {`{"id":"s","type":"service","unit":"a.service","state":"paused"}`, "omits state to leave the unit's lifecycle to the machine"},
|
||||||
|
} {
|
||||||
|
_, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"f","type":"file","path":"/etc/x","content":"x"},` + c.resource + `]}`))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), c.refusal) {
|
||||||
|
t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"f","type":"file","path":"/etc/x","content":"x"},
|
||||||
|
{"id":"s","type":"service","unit":"NetworkManager.service","restart-on":["f"]}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if s := d.Resources[1].(*Service); !s.Stateless() {
|
||||||
|
t.Error("a service without a state was not read as stateless")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -49,8 +49,13 @@ type Identity struct {
|
|||||||
Membership Membership `json:"membership"`
|
Membership Membership `json:"membership"`
|
||||||
|
|
||||||
// Overlay is this node's key on the private network. Generated here, like the identity above,
|
// Overlay is this node's key on the private network. Generated here, like the identity above,
|
||||||
// and for the same reason: the mesh computes a graph it cannot impersonate.
|
// and for the same reason: the mesh computes a graph it cannot impersonate — or, on an adopted
|
||||||
|
// node that took a found tunnel over, that tunnel's key (novox/hq ADR 0105).
|
||||||
Overlay OverlayKey `json:"overlay"`
|
Overlay OverlayKey `json:"overlay"`
|
||||||
|
// OverlayBefore is the public half of the overlay key this node held before it took a found
|
||||||
|
// tunnel's, so a take run again names the key the mesh still records. Empty on a node that
|
||||||
|
// never took one.
|
||||||
|
OverlayBefore string `json:"overlay_before,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Membership is how this node reaches the mesh it belongs to, and who it believes.
|
// Membership is how this node reaches the mesh it belongs to, and who it believes.
|
||||||
@@ -71,6 +76,11 @@ type Membership struct {
|
|||||||
// Not the token's secret: that is spent, and a credential that lives for ever should not be
|
// Not the token's secret: that is spent, and a credential that lives for ever should not be
|
||||||
// the same string as one that was meant to be used once.
|
// the same string as one that was meant to be used once.
|
||||||
Password string `json:"password"`
|
Password string `json:"password"`
|
||||||
|
|
||||||
|
// Transport is which bus this membership is for. Empty is the bus the mesh ran on before
|
||||||
|
// the move — so every membership written before this field existed reads as correct, not as
|
||||||
|
// unset — and "nats" is the one being moved to (novox/hq design 28, task 5.2).
|
||||||
|
Transport string `json:"transport,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Queue is where this node listens. Its account may read this and nothing else.
|
// Queue is where this node listens. Its account may read this and nothing else.
|
||||||
|
|||||||
@@ -43,6 +43,27 @@ func GenerateOverlayKey() (OverlayKey, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OverlayKeyFrom makes this node's overlay key from a private key it did not generate: the found
|
||||||
|
// tunnel's, on an adopted node whose private network takes that tunnel over (novox/hq ADR 0105).
|
||||||
|
// The one case where the mesh takes a credential it did not mint. From here on it is stored and
|
||||||
|
// sealed exactly as a generated one — in the identity file and the key file, readable by root
|
||||||
|
// alone — and the mesh receives only the public half, derived here from the private one so the
|
||||||
|
// two cannot disagree.
|
||||||
|
func OverlayKeyFrom(privateBase64 string) (OverlayKey, error) {
|
||||||
|
raw, err := base64.StdEncoding.DecodeString(privateBase64)
|
||||||
|
if err != nil {
|
||||||
|
return OverlayKey{}, fmt.Errorf("the found tunnel's private key is not base64: %w", err)
|
||||||
|
}
|
||||||
|
private, err := ecdh.X25519().NewPrivateKey(raw)
|
||||||
|
if err != nil {
|
||||||
|
return OverlayKey{}, fmt.Errorf("the found tunnel's private key is not a Curve25519 key: %w", err)
|
||||||
|
}
|
||||||
|
return OverlayKey{
|
||||||
|
Public: base64.StdEncoding.EncodeToString(private.PublicKey().Bytes()),
|
||||||
|
Private: base64.StdEncoding.EncodeToString(private.Bytes()),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// OverlayKeyPath is where the private half lives: a file of its own, referenced by the interface
|
// OverlayKeyPath is where the private half lives: a file of its own, referenced by the interface
|
||||||
// configuration rather than embedded in it.
|
// configuration rather than embedded in it.
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
package identity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0105: the found tunnel's private key becomes the node's overlay key, stored as a
|
||||||
|
// generated one is, and the public half the mesh records is derived from it — so the peers that
|
||||||
|
// know the tunnel by that key keep reaching it.
|
||||||
|
func TestAnOverlayKeyTakenFromAFoundTunnelIsTheSameKey(t *testing.T) {
|
||||||
|
generated, err := GenerateOverlayKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
taken, err := OverlayKeyFrom(generated.Private)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if taken.Public != generated.Public || taken.Private != generated.Private {
|
||||||
|
t.Fatalf("a key taken from a private half is not that key: %+v vs %+v", taken, generated)
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"", "not base64!", "c2hvcnQ="} {
|
||||||
|
if _, err := OverlayKeyFrom(bad); err == nil || !strings.Contains(err.Error(), "found tunnel") {
|
||||||
|
t.Errorf("%q was taken as a key: %v", bad, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The enrolment conversation, as the host's own words for it.
|
||||||
|
//
|
||||||
|
// **Its own seam rather than part of Link**, because almost nothing about it is the same. The
|
||||||
|
// credential is a one-time secret rather than this node's own; there is no declaration to hear; the
|
||||||
|
// whole exchange is a single question asked and possibly asked again. And the stakes differ: a node
|
||||||
|
// that fails here is not in the mesh at all, where a node that fails in Link has merely lost touch
|
||||||
|
// with one it belongs to.
|
||||||
|
|
||||||
|
// Approach is how a node reaches a mesh it does not yet belong to.
|
||||||
|
//
|
||||||
|
// The three things a token carries about where to go, and nothing about who is asking: the address,
|
||||||
|
// the certificate that address must present, and which bus is at the other end. **Every token names
|
||||||
|
// the bus the mesh runs on today until the rollout** (novox/hq ADR 0116 step 5), so an empty
|
||||||
|
// Transport is the ordinary case rather than something missing.
|
||||||
|
type Approach struct {
|
||||||
|
Address string
|
||||||
|
Fingerprint string
|
||||||
|
Transport string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Asking is one open enrolment conversation.
|
||||||
|
type Asking interface {
|
||||||
|
// Ask puts the request to the mesh and waits for one answer, or says why none came.
|
||||||
|
//
|
||||||
|
// Called again, with the same bytes, while the mesh says "try again": the keys this node
|
||||||
|
// generated are the ones it keeps, so the same request is the same enrolment and the mesh holds
|
||||||
|
// the token for it (novox/hq issue 083).
|
||||||
|
Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error)
|
||||||
|
|
||||||
|
// Close lets go of the connection made with the token.
|
||||||
|
Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Present opens an enrolment conversation with the mesh.
|
||||||
|
//
|
||||||
|
// The connection is made before anything is sent, and the certificate is checked while it is being
|
||||||
|
// made — so a node pointed at the wrong bus finds out before its token has left the machine (ADR
|
||||||
|
// 0004).
|
||||||
|
func Present(ctx context.Context, to Approach, node, secret string,
|
||||||
|
timeout time.Duration) (Asking, error) {
|
||||||
|
|
||||||
|
switch to.Transport {
|
||||||
|
case OnNATS:
|
||||||
|
return presentNats(ctx, to, node, secret, timeout)
|
||||||
|
default:
|
||||||
|
return presentCurrent(ctx, to, node, secret, timeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
amqp "github.com/rabbitmq/amqp091-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The enrolment conversation on the bus the mesh runs on today.
|
||||||
|
//
|
||||||
|
// Moved out of Enrol rather than changed. The reply travels on this node's own queue and is picked
|
||||||
|
// out by the correlation id the request carried, which is what this transport's reply field means
|
||||||
|
// and has always meant.
|
||||||
|
|
||||||
|
type currentAsking struct {
|
||||||
|
conn *amqp.Connection
|
||||||
|
channel *amqp.Channel
|
||||||
|
queue string
|
||||||
|
node string
|
||||||
|
replies <-chan amqp.Delivery
|
||||||
|
closed chan *amqp.Error
|
||||||
|
}
|
||||||
|
|
||||||
|
func presentCurrent(_ context.Context, to Approach, node, secret string,
|
||||||
|
timeout time.Duration) (Asking, error) {
|
||||||
|
|
||||||
|
config, err := PinnedConfig(to.Fingerprint)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The account name is the node's, and the password is the token's secret. Escaped because a name
|
||||||
|
// or secret containing a colon or an at-sign would otherwise change which host this connects to
|
||||||
|
// — a credential silently redirecting a connection is the worst shape this could take.
|
||||||
|
dsn := fmt.Sprintf("amqps://%s:%s@%s/",
|
||||||
|
url.QueryEscape(node), url.QueryEscape(secret), to.Address)
|
||||||
|
|
||||||
|
conn, err := amqp.DialConfig(dsn, amqp.Config{
|
||||||
|
TLSClientConfig: config,
|
||||||
|
Dial: amqp.DefaultDial(timeout),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrWrongCertificate) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Not quoted back: the DSN carries the one-time secret.
|
||||||
|
return nil, fmt.Errorf("cannot reach the broker at %s as %s: %w", to.Address, node, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
channel, err := conn.Channel()
|
||||||
|
if err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// This node's own queue, which its account is scoped to and nothing else may read.
|
||||||
|
queue, err := channel.QueueDeclare(QueueFor(node), true, false, false, false, nil)
|
||||||
|
if err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, fmt.Errorf("cannot declare this node's queue %s: %w", QueueFor(node), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
replies, err := channel.Consume(queue.Name, "", true, false, false, false, nil)
|
||||||
|
if err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ¤tAsking{
|
||||||
|
conn: conn, channel: channel, queue: queue.Name, node: node, replies: replies,
|
||||||
|
closed: conn.NotifyClose(make(chan *amqp.Error, 1)),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *currentAsking) Close() {
|
||||||
|
if a.channel != nil {
|
||||||
|
_ = a.channel.Close()
|
||||||
|
}
|
||||||
|
if a.conn != nil {
|
||||||
|
_ = a.conn.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *currentAsking) Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error) {
|
||||||
|
correlation := fmt.Sprintf("%s-%d", a.node, time.Now().UnixNano())
|
||||||
|
publish, cancel := context.WithTimeout(ctx, wait)
|
||||||
|
defer cancel()
|
||||||
|
if err := a.channel.PublishWithContext(publish, Exchange, KeyEnrol, false, false,
|
||||||
|
amqp.Publishing{
|
||||||
|
ContentType: "application/json",
|
||||||
|
CorrelationId: correlation,
|
||||||
|
ReplyTo: a.queue,
|
||||||
|
Body: request,
|
||||||
|
}); err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot publish to the %s exchange: %w", Exchange, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Waited for rather than assumed. A published message that nothing answers means the control
|
||||||
|
// plane is not running, and a node that carried on regardless would believe it had joined a mesh
|
||||||
|
// that has never heard of it.
|
||||||
|
deadline := time.NewTimer(wait)
|
||||||
|
defer deadline.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case reason := <-a.closed:
|
||||||
|
return nil, fmt.Errorf("the broker closed the connection: %v", reason)
|
||||||
|
case <-deadline.C:
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"the broker accepted this node's connection and nothing answered within %s. The "+
|
||||||
|
"mesh's broker is running and its control plane is not", wait)
|
||||||
|
case delivery, ok := <-a.replies:
|
||||||
|
if !ok {
|
||||||
|
return nil, errors.New("the broker stopped delivering")
|
||||||
|
}
|
||||||
|
// Anything else on this queue is not the answer to this question.
|
||||||
|
if delivery.CorrelationId != correlation {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return delivery.Body, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The enrolment conversation on the bus being built.
|
||||||
|
//
|
||||||
|
// **The reply address is the whole of what changes**, and it changes for a reason the transport
|
||||||
|
// forces rather than a preference. Core NATS request/reply puts the caller's inbox in the message's
|
||||||
|
// reply field and a plain responder answers it — but this request goes into a stream, and a message
|
||||||
|
// a JetStream consumer delivers has had that field claimed for the consumer's own ack address. So by
|
||||||
|
// the time the controller reads the request, the transport's reply field names where the
|
||||||
|
// *controller* must acknowledge. Verified against a running server (design 25 §2).
|
||||||
|
//
|
||||||
|
// The address therefore travels as a field of the request, and this subscribes it before publishing:
|
||||||
|
// a node that published first could miss an answer to a question nobody was listening for.
|
||||||
|
|
||||||
|
// enrolInbox is where a node enrolling waits.
|
||||||
|
//
|
||||||
|
// Under `_INBOX.enrol.<node>.`, which is exactly what its enrolment user may subscribe and no
|
||||||
|
// wider — so an answer sealed to one machine cannot be read by another enrolling beside it. The
|
||||||
|
// random tail is this attempt's own: a reply left over from an attempt that timed out is not the
|
||||||
|
// answer to this question, which is what the correlation id does on the other transport.
|
||||||
|
func enrolInbox(node string) (string, error) {
|
||||||
|
tail := make([]byte, 8)
|
||||||
|
if _, err := rand.Read(tail); err != nil {
|
||||||
|
return "", fmt.Errorf("cannot make a reply address: %w", err)
|
||||||
|
}
|
||||||
|
return "_INBOX.enrol." + node + "." + hex.EncodeToString(tail), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type natsAsking struct {
|
||||||
|
conn *nats.Conn
|
||||||
|
js nats.JetStreamContext
|
||||||
|
inbox string
|
||||||
|
answers *nats.Subscription
|
||||||
|
lost chan error
|
||||||
|
}
|
||||||
|
|
||||||
|
func presentNats(_ context.Context, to Approach, node, secret string,
|
||||||
|
timeout time.Duration) (Asking, error) {
|
||||||
|
|
||||||
|
config, err := PinnedConfig(to.Fingerprint)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
inbox, err := enrolInbox(node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
lost := make(chan error, 1)
|
||||||
|
// The user is this token's own — `enrol.<node>`, which may publish the enrolment subject and
|
||||||
|
// subscribe its own inbox and nothing else (design 25 §6). The secret is its password, the same
|
||||||
|
// string the request claims, so the server proves somebody holds the token and the request
|
||||||
|
// proves the same thing to the controller without it having to ask the server who connected.
|
||||||
|
conn, err := nats.Connect(natsURL(to.Address),
|
||||||
|
nats.Secure(config),
|
||||||
|
nats.UserInfo("enrol."+node, secret),
|
||||||
|
nats.Name("mesh-host/enrol/"+node),
|
||||||
|
nats.Timeout(timeout),
|
||||||
|
nats.NoReconnect(),
|
||||||
|
nats.DisconnectErrHandler(func(_ *nats.Conn, err error) {
|
||||||
|
select {
|
||||||
|
case lost <- fmt.Errorf("the bus closed the connection: %w", err):
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrWrongCertificate) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Not quoted back with the credential: the secret is one-time and still a secret.
|
||||||
|
return nil, fmt.Errorf("cannot reach the bus at %s as %s: %w", to.Address, node, err)
|
||||||
|
}
|
||||||
|
js, err := conn.JetStream()
|
||||||
|
if err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, fmt.Errorf("the bus at %s has no JetStream: %w", to.Address, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Subscribed before anything is published, so an answer cannot arrive before there is anywhere
|
||||||
|
// for it to land.
|
||||||
|
answers, err := conn.SubscribeSync(inbox)
|
||||||
|
if err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, fmt.Errorf("this node cannot listen for the mesh's answer: %w", err)
|
||||||
|
}
|
||||||
|
if err := conn.Flush(); err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, fmt.Errorf("this node's reply address did not reach the bus: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &natsAsking{conn: conn, js: js, inbox: inbox, answers: answers, lost: lost}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *natsAsking) Close() {
|
||||||
|
if a.answers != nil {
|
||||||
|
_ = a.answers.Unsubscribe()
|
||||||
|
}
|
||||||
|
if a.conn != nil {
|
||||||
|
a.conn.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ask publishes the request with this attempt's reply address written into it, and waits there.
|
||||||
|
func (a *natsAsking) Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error) {
|
||||||
|
addressed, err := withReplyTo(request, a.inbox)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
publish, cancel := context.WithTimeout(ctx, wait)
|
||||||
|
defer cancel()
|
||||||
|
// Into the stream and awaited: an enrolment the bus never accepted must fail here rather than be
|
||||||
|
// assumed, because the node has nothing else to go on.
|
||||||
|
if _, err := a.js.Publish(EnrolSubject, addressed, nats.Context(publish)); err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot ask the mesh to enrol this node: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Waited for rather than assumed. A published message that nothing answers means the controller
|
||||||
|
// is not running, and a node that carried on regardless would believe it had joined a mesh that
|
||||||
|
// has never heard of it.
|
||||||
|
//
|
||||||
|
// **The wait may legitimately be several store-window cycles long**: the controller naks the
|
||||||
|
// request with a delay while its store is restarting, and the node is waiting on the other side
|
||||||
|
// of that — which is exactly the combination that would have delivered the answer to a caller
|
||||||
|
// who had given up, had the address travelled in the transport's field.
|
||||||
|
answered, cancelAnswer := context.WithTimeout(ctx, wait)
|
||||||
|
defer cancelAnswer()
|
||||||
|
for {
|
||||||
|
msg, err := a.answers.NextMsgWithContext(answered)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
return msg.Data, nil
|
||||||
|
case errors.Is(err, context.DeadlineExceeded):
|
||||||
|
select {
|
||||||
|
case reason := <-a.lost:
|
||||||
|
return nil, reason
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"the bus accepted this node's connection and nothing answered within %s. The mesh's "+
|
||||||
|
"bus is running and its controller is not", wait)
|
||||||
|
case errors.Is(err, context.Canceled):
|
||||||
|
return nil, ctx.Err()
|
||||||
|
default:
|
||||||
|
select {
|
||||||
|
case reason := <-a.lost:
|
||||||
|
return nil, reason
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("waiting for the mesh's answer: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The enrolment round trip against a real server.
|
||||||
|
//
|
||||||
|
// **This is the test that keeps a reason from becoming folklore.** The reply address travels in the
|
||||||
|
// request's payload because a JetStream consumer's delivery has had the transport's reply field
|
||||||
|
// claimed for its own ack address — which is a fact about a server, not a rule anybody can check by
|
||||||
|
// reading. Both halves are asserted here: that the field really is eaten, and that the answer
|
||||||
|
// reaches the node anyway.
|
||||||
|
//
|
||||||
|
// docker run -d --rm --name t -p 14223:4222 nats:2.10-alpine -js
|
||||||
|
// MESH_TEST_NATS=nats://127.0.0.1:14223 go test ./internal/link/ -run TestNatsAnEnrolment
|
||||||
|
|
||||||
|
// asking is a conversation on a bus with no TLS. Built directly rather than through Present because
|
||||||
|
// the pin is what Present adds and PinnedConfig's own tests cover it; what is under test here is the
|
||||||
|
// address the answer comes back on.
|
||||||
|
func asking(t *testing.T, conn *nats.Conn, js nats.JetStreamContext, node string) *natsAsking {
|
||||||
|
t.Helper()
|
||||||
|
inbox, err := enrolInbox(node)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
answers, err := conn.SubscribeSync(inbox)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := conn.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a := &natsAsking{conn: conn, js: js, inbox: inbox, answers: answers, lost: make(chan error, 1)}
|
||||||
|
t.Cleanup(a.Close)
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
// theMeshAnswers stands in for the controller: it consumes the enrolment off the stream, reads the
|
||||||
|
// reply address out of the payload — never from the transport field — and answers there. It reports
|
||||||
|
// what the transport field actually held, which is the claim design 25 §2 rests on.
|
||||||
|
func theMeshAnswers(t *testing.T, conn *nats.Conn, js nats.JetStreamContext,
|
||||||
|
reply EnrolReply) <-chan string {
|
||||||
|
t.Helper()
|
||||||
|
sawReplyField := make(chan string, 1)
|
||||||
|
sub, err := js.Subscribe(EnrolSubject, func(msg *nats.Msg) {
|
||||||
|
select {
|
||||||
|
case sawReplyField <- msg.Reply:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
var addressed struct {
|
||||||
|
ReplyTo string `json:"reply_to"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(msg.Data, &addressed); err != nil || addressed.ReplyTo == "" {
|
||||||
|
_ = msg.Ack()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(reply)
|
||||||
|
// Published explicitly to the address the payload named, never msg.Respond — which would
|
||||||
|
// send it to whatever the transport's reply field holds, and that is the point.
|
||||||
|
_ = conn.Publish(addressed.ReplyTo, body)
|
||||||
|
_ = msg.Ack()
|
||||||
|
}, nats.Durable("controller-standin"), nats.ManualAck())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||||
|
return sawReplyField
|
||||||
|
}
|
||||||
|
|
||||||
|
// An enrolment is answered on the address the request carried, and the transport's own reply field
|
||||||
|
// held something else entirely.
|
||||||
|
func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) {
|
||||||
|
conn, js := aBus(t)
|
||||||
|
const node = "joining"
|
||||||
|
|
||||||
|
sawReplyField := theMeshAnswers(t, conn, js, EnrolReply{Accepted: true, Node: node, Password: "p"})
|
||||||
|
a := asking(t, conn, js, node)
|
||||||
|
|
||||||
|
request, _ := json.Marshal(EnrolRequest{Node: node, Secret: "t"})
|
||||||
|
answer, err := a.Ask(context.Background(), request, 8*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("no answer reached the node: %v", err)
|
||||||
|
}
|
||||||
|
var reply EnrolReply
|
||||||
|
if err := json.Unmarshal(answer, &reply); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reply.Accepted || reply.Node != node {
|
||||||
|
t.Fatalf("the answer was not the mesh's: %+v", reply)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the field the answer would have gone to, had it used the transport's: the consumer's own
|
||||||
|
// ack address. If a future server stopped doing this, the payload-borne address would still
|
||||||
|
// work and this line is what would say the reason had changed.
|
||||||
|
select {
|
||||||
|
case field := <-sawReplyField:
|
||||||
|
if field == a.inbox {
|
||||||
|
t.Fatalf("the transport's reply field held this node's inbox (%s), so the payload "+
|
||||||
|
"address is no longer load-bearing — check design 25 §2 before relying on it", field)
|
||||||
|
}
|
||||||
|
if field == "" {
|
||||||
|
t.Fatal("the transport's reply field was empty rather than claimed, which is a third " +
|
||||||
|
"behaviour from the two design 25 §2 describes")
|
||||||
|
}
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("the stand-in never saw the request")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A request that names no reply address is not answered, and the node says so as a mesh that is not
|
||||||
|
// running rather than hanging. The controller has nowhere to send an answer, which is the failure
|
||||||
|
// the payload field exists to make impossible — asserted so that a request built without it fails
|
||||||
|
// loudly here rather than quietly on a machine.
|
||||||
|
func TestNatsAnEnrolmentWithNoReplyAddressIsNotAnswered(t *testing.T) {
|
||||||
|
conn, js := aBus(t)
|
||||||
|
const node = "silent"
|
||||||
|
|
||||||
|
theMeshAnswers(t, conn, js, EnrolReply{Accepted: true, Node: node})
|
||||||
|
a := asking(t, conn, js, node)
|
||||||
|
|
||||||
|
// Published without going through Ask, so the reply address is genuinely absent.
|
||||||
|
request, _ := json.Marshal(EnrolRequest{Node: node, Secret: "t"})
|
||||||
|
if _, err := js.Publish(EnrolSubject, request); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := a.Ask(context.Background(), []byte(`{"node":"`+node+`"}`), 0); err == nil {
|
||||||
|
t.Fatal("a node with no answer coming was told it had one")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
amqp "github.com/rabbitmq/amqp091-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Bus is what a host needs of the mesh's bus, in the mesh's own words.
|
||||||
|
//
|
||||||
|
// A host says exactly two things unprompted: what it applied, and that it is here. They are not
|
||||||
|
// the same kind of statement and the difference is the whole of this interface — one must arrive
|
||||||
|
// and one must not be insisted on.
|
||||||
|
//
|
||||||
|
// **The host still imports nothing of the mesh's own** (novox/hq ADR 0005): this is its own
|
||||||
|
// interface over its own client libraries, not a contract shared with the controller. The two
|
||||||
|
// agree because a conformance fixture holds them to one envelope, which is the only kind of
|
||||||
|
// agreement that survives being in different repositories.
|
||||||
|
type Bus interface {
|
||||||
|
// Report says what this node applied. **It must arrive.** A report that fails leaves the
|
||||||
|
// mesh believing the node never answered while the node believes it did, and the two go on
|
||||||
|
// disagreeing with nothing anywhere saying so — the shape of fault this project keeps
|
||||||
|
// finding. Returns false when it could not be delivered, so the caller can say so.
|
||||||
|
Report(ctx context.Context, node string, body []byte) error
|
||||||
|
|
||||||
|
// Alive says this node is here, and nothing else. **Losing one is nothing**: the next is a
|
||||||
|
// minute away and the mesh reads a gap rather than counting arrivals. Insisting on delivery
|
||||||
|
// would turn a harmless miss into a logged failure every minute.
|
||||||
|
Alive(ctx context.Context, node string, body []byte) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- The bus the mesh runs on today -----------------------------------------------------------
|
||||||
|
|
||||||
|
// OverCurrent is the bus as a channel, until the rollout.
|
||||||
|
type OverCurrent struct{ Channel *amqp.Channel }
|
||||||
|
|
||||||
|
func (b OverCurrent) Report(ctx context.Context, node string, body []byte) error {
|
||||||
|
// Mandatory: an unroutable report comes back rather than disappearing.
|
||||||
|
return b.Channel.PublishWithContext(ctx, Exchange, KeyReport, true, false,
|
||||||
|
amqp.Publishing{ContentType: "application/json", Body: body})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b OverCurrent) Alive(ctx context.Context, node string, body []byte) error {
|
||||||
|
return b.Channel.PublishWithContext(ctx, Exchange, KeyAlive, false, false,
|
||||||
|
amqp.Publishing{ContentType: "application/json", Body: body})
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- NATS ---------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// OverNATS is the bus as a connection. A report goes through JetStream because it must survive
|
||||||
|
// the controller's store restarting; a heartbeat does not, because it must not.
|
||||||
|
type OverNATS struct {
|
||||||
|
Conn *nats.Conn
|
||||||
|
JS nats.JetStreamContext
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReportSubject and AliveSubject are this node's own, and no other node's: a host's account may
|
||||||
|
// publish `mesh.control.<its own node>.>` and nothing wider, so the subject is the authority on
|
||||||
|
// which node a report is about.
|
||||||
|
func ReportSubject(node string) string { return "mesh.control." + node + ".report" }
|
||||||
|
func AliveSubject(node string) string { return "mesh.control." + node + ".alive" }
|
||||||
|
|
||||||
|
func (b OverNATS) Report(ctx context.Context, node string, body []byte) error {
|
||||||
|
// Into the CONTROL stream and awaited: this is the message the store-window guarantee is
|
||||||
|
// about (novox/hq ADR 0083). The controller naks with a delay while its store is away and
|
||||||
|
// the message is redelivered; a publish the bus never accepted must fail here rather than
|
||||||
|
// be assumed.
|
||||||
|
if _, err := b.JS.Publish(ReportSubject(node), body, nats.Context(ctx)); err != nil {
|
||||||
|
return fmt.Errorf("reporting: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b OverNATS) Alive(ctx context.Context, node string, body []byte) error {
|
||||||
|
// Core, deliberately: a heartbeat in a stream is the mesh's least valuable message competing
|
||||||
|
// for retention with its most valuable, and a lost one is the next one.
|
||||||
|
if err := b.Conn.Publish(AliveSubject(node), body); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Flushed rather than fired and forgotten, so "could not tell the mesh" means the write
|
||||||
|
// failed rather than that nobody has looked yet.
|
||||||
|
flush, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
return b.Conn.FlushWithContext(flush)
|
||||||
|
}
|
||||||
+76
-115
@@ -6,10 +6,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/url"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
amqp "github.com/rabbitmq/amqp091-go"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The wire format shared with the control plane, which defines it separately because this binary
|
// The wire format shared with the control plane, which defines it separately because this binary
|
||||||
@@ -52,6 +49,41 @@ type EnrolRequest struct {
|
|||||||
// holds the private half of PublicKey. The mesh asks for it before letting an enrolment finish
|
// holds the private half of PublicKey. The mesh asks for it before letting an enrolment finish
|
||||||
// on a token this key already spent (novox/hq issue 083).
|
// on a token this key already spent (novox/hq issue 083).
|
||||||
Proof []byte `json:"proof,omitempty"`
|
Proof []byte `json:"proof,omitempty"`
|
||||||
|
|
||||||
|
// ReplyTo is where the mesh's answer goes, as a field of the request rather than the
|
||||||
|
// transport's own reply address (design 25 §2). Written by the transport that needs it —
|
||||||
|
// withReplyTo, once per attempt — because a request going into a stream has had the transport's
|
||||||
|
// reply field claimed for the consumer's ack address before the controller ever reads it.
|
||||||
|
//
|
||||||
|
// Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the
|
||||||
|
// field means what it has always meant. Named here so both sides of the wire hold the same
|
||||||
|
// field name, which is what the shape test on each side is for.
|
||||||
|
ReplyTo string `json:"reply_to,omitempty"`
|
||||||
|
|
||||||
|
// Tunnel is the tunnel this node found and whose key it took as its overlay key (novox/hq ADR
|
||||||
|
// 0105): everything about it but that key. Sent with the keys because it is one of them —
|
||||||
|
// OverlayKey above IS this tunnel's public key when this is set — and the mesh composes the
|
||||||
|
// hub's address, the range and the carried peers from it before the first declaration.
|
||||||
|
Tunnel *Tunnel `json:"tunnel,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tunnel is a found tunnel as it travels: no private key.
|
||||||
|
type Tunnel struct {
|
||||||
|
Interface string `json:"interface"`
|
||||||
|
Unit string `json:"unit"`
|
||||||
|
Config string `json:"config"`
|
||||||
|
Port int `json:"port"`
|
||||||
|
Address string `json:"address"`
|
||||||
|
Range string `json:"range"`
|
||||||
|
MTU int `json:"mtu,omitempty"`
|
||||||
|
PublicKey string `json:"public_key"`
|
||||||
|
Peers []TunnelPeer `json:"peers,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TunnelPeer is one peer of a found tunnel: its key, and the address the tunnel routes to it.
|
||||||
|
type TunnelPeer struct {
|
||||||
|
PublicKey string `json:"public_key"`
|
||||||
|
Address string `json:"address"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// EnrolReply is what the mesh says back.
|
// EnrolReply is what the mesh says back.
|
||||||
@@ -123,136 +155,65 @@ func answered(reply EnrolReply, asking time.Duration) (again bool, err error) {
|
|||||||
// was issued and the secret is its password. So this is not how the node gets in — it is what it
|
// was issued and the secret is its password. So this is not how the node gets in — it is what it
|
||||||
// says once it is in, and the secret travels again because the control plane must not have to ask
|
// says once it is in, and the secret travels again because the control plane must not have to ask
|
||||||
// the broker who connected.
|
// the broker who connected.
|
||||||
func Enrol(ctx context.Context, address, pin, node, secret string, public []byte,
|
func Enrol(ctx context.Context, to Approach, node, secret string, public []byte,
|
||||||
overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte,
|
overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte,
|
||||||
timeout time.Duration) (EnrolReply, error) {
|
tunnel *Tunnel, timeout time.Duration) (EnrolReply, error) {
|
||||||
|
|
||||||
config, err := PinnedConfig(pin)
|
asking, err := Present(ctx, to, node, secret, timeout)
|
||||||
if err != nil {
|
|
||||||
return EnrolReply{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// The account name is the node's, and the password is the token's secret. Escaped because a
|
|
||||||
// name or secret containing a colon or an at-sign would otherwise change which host this
|
|
||||||
// connects to — a credential silently redirecting a connection is the worst shape this could
|
|
||||||
// take.
|
|
||||||
dsn := fmt.Sprintf("amqps://%s:%s@%s/",
|
|
||||||
url.QueryEscape(node), url.QueryEscape(secret), address)
|
|
||||||
|
|
||||||
conn, err := amqp.DialConfig(dsn, amqp.Config{
|
|
||||||
TLSClientConfig: config,
|
|
||||||
Dial: amqp.DefaultDial(timeout),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, ErrWrongCertificate) {
|
|
||||||
return EnrolReply{}, err
|
|
||||||
}
|
|
||||||
// Not quoted back: the DSN carries the one-time secret.
|
|
||||||
return EnrolReply{}, fmt.Errorf("cannot reach the broker at %s as %s: %w", address, node, err)
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
|
|
||||||
channel, err := conn.Channel()
|
|
||||||
if err != nil {
|
|
||||||
return EnrolReply{}, err
|
|
||||||
}
|
|
||||||
defer channel.Close()
|
|
||||||
|
|
||||||
// This node's own queue, which its account is scoped to and nothing else may read.
|
|
||||||
queue, err := channel.QueueDeclare(QueueFor(node), true, false, false, false, nil)
|
|
||||||
if err != nil {
|
|
||||||
return EnrolReply{}, fmt.Errorf(
|
|
||||||
"cannot declare this node's queue %s: %w", QueueFor(node), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
replies, err := channel.Consume(queue.Name, "", true, false, false, false, nil)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return EnrolReply{}, err
|
return EnrolReply{}, err
|
||||||
}
|
}
|
||||||
|
defer asking.Close()
|
||||||
|
|
||||||
request := EnrolRequest{Node: node, Secret: secret, PublicKey: public,
|
request := EnrolRequest{Node: node, Secret: secret, PublicKey: public,
|
||||||
OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile,
|
OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile,
|
||||||
Proof: proof}
|
Proof: proof, Tunnel: tunnel}
|
||||||
body, err := json.Marshal(request)
|
body, err := json.Marshal(request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return EnrolReply{}, err
|
return EnrolReply{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Asked, and asked again with the same request while the mesh says "try again": the keys
|
// Asked, and asked again with the same request while the mesh says "try again": the keys this
|
||||||
// this node generated are the ones it keeps, so the same request is the same enrolment, and
|
// node generated are the ones it keeps, so the same request is the same enrolment, and the mesh
|
||||||
// the mesh holds the token for it (novox/hq issue 083).
|
// holds the token for it (novox/hq issue 083).
|
||||||
ask := func() (string, error) {
|
|
||||||
correlation := fmt.Sprintf("%s-%d", node, time.Now().UnixNano())
|
|
||||||
publish, cancel := context.WithTimeout(ctx, timeout)
|
|
||||||
defer cancel()
|
|
||||||
if err := channel.PublishWithContext(publish, Exchange, KeyEnrol, false, false,
|
|
||||||
amqp.Publishing{
|
|
||||||
ContentType: "application/json",
|
|
||||||
CorrelationId: correlation,
|
|
||||||
ReplyTo: queue.Name,
|
|
||||||
Body: body,
|
|
||||||
}); err != nil {
|
|
||||||
return "", fmt.Errorf("cannot publish to the %s exchange: %w", Exchange, err)
|
|
||||||
}
|
|
||||||
return correlation, nil
|
|
||||||
}
|
|
||||||
correlation, err := ask()
|
|
||||||
if err != nil {
|
|
||||||
return EnrolReply{}, err
|
|
||||||
}
|
|
||||||
began := time.Now()
|
began := time.Now()
|
||||||
|
|
||||||
// Waited for rather than assumed. A published message that nothing answers means the control
|
|
||||||
// plane is not running, and a node that carried on regardless would believe it had joined a
|
|
||||||
// mesh that has never heard of it.
|
|
||||||
deadline := time.NewTimer(timeout)
|
|
||||||
defer deadline.Stop()
|
|
||||||
closed := conn.NotifyClose(make(chan *amqp.Error, 1))
|
|
||||||
|
|
||||||
for {
|
for {
|
||||||
|
answer, err := asking.Ask(ctx, body, timeout)
|
||||||
|
if err != nil {
|
||||||
|
return EnrolReply{}, err
|
||||||
|
}
|
||||||
|
var reply EnrolReply
|
||||||
|
if err := json.Unmarshal(answer, &reply); err != nil {
|
||||||
|
return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err)
|
||||||
|
}
|
||||||
|
again, err := answered(reply, time.Since(began))
|
||||||
|
if err != nil {
|
||||||
|
return reply, err
|
||||||
|
}
|
||||||
|
if !again {
|
||||||
|
return reply, nil
|
||||||
|
}
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return EnrolReply{}, ctx.Err()
|
return EnrolReply{}, ctx.Err()
|
||||||
case reason := <-closed:
|
case <-time.After(AskAgainAfter):
|
||||||
return EnrolReply{}, fmt.Errorf("the broker closed the connection: %v", reason)
|
|
||||||
case <-deadline.C:
|
|
||||||
return EnrolReply{}, fmt.Errorf(
|
|
||||||
"the broker accepted this node's connection and nothing answered within %s. The "+
|
|
||||||
"mesh's broker is running and its control plane is not", timeout)
|
|
||||||
case delivery, ok := <-replies:
|
|
||||||
if !ok {
|
|
||||||
return EnrolReply{}, errors.New("the broker stopped delivering")
|
|
||||||
}
|
|
||||||
// Anything else on this queue is not the answer to this question.
|
|
||||||
if delivery.CorrelationId != correlation {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var reply EnrolReply
|
|
||||||
if err := json.Unmarshal(delivery.Body, &reply); err != nil {
|
|
||||||
return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err)
|
|
||||||
}
|
|
||||||
again, err := answered(reply, time.Since(began))
|
|
||||||
if err != nil {
|
|
||||||
return reply, err
|
|
||||||
}
|
|
||||||
if !again {
|
|
||||||
return reply, nil
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return EnrolReply{}, ctx.Err()
|
|
||||||
case <-time.After(AskAgainAfter):
|
|
||||||
}
|
|
||||||
if correlation, err = ask(); err != nil {
|
|
||||||
return EnrolReply{}, err
|
|
||||||
}
|
|
||||||
if !deadline.Stop() {
|
|
||||||
select {
|
|
||||||
case <-deadline.C:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
deadline.Reset(timeout)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// withReplyTo writes this attempt's reply address into the request, as a field of its own.
|
||||||
|
//
|
||||||
|
// **Written into the bytes rather than carried beside them**, because the whole point is that the
|
||||||
|
// address survives a stream: a JetStream consumer's delivery has had the transport's reply field
|
||||||
|
// claimed for its own ack address, so a reply address that is not in the payload is one the
|
||||||
|
// controller cannot read (design 25 §2). Done by decoding and re-encoding rather than by setting the
|
||||||
|
// field before marshalling, so one request can be asked again with a fresh address each time without
|
||||||
|
// the caller knowing that is what happens.
|
||||||
|
func withReplyTo(request []byte, inbox string) ([]byte, error) {
|
||||||
|
var fields map[string]any
|
||||||
|
if err := json.Unmarshal(request, &fields); err != nil {
|
||||||
|
return nil, fmt.Errorf("this node's own enrolment request cannot be read back: %w", err)
|
||||||
|
}
|
||||||
|
fields["reply_to"] = inbox
|
||||||
|
return json.Marshal(fields)
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,84 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What a host hears, as the host's own words for it.
|
||||||
|
//
|
||||||
|
// The outbound half went behind `Bus` (bus.go) and a node's two statements stopped naming a
|
||||||
|
// transport. This is the other half — dialling, and the declarations that arrive — and it is where
|
||||||
|
// the transport reached furthest: the run loop selected on a channel of the client library's own
|
||||||
|
// delivery type, so every part of holding a node in its mesh knew which bus it was on.
|
||||||
|
//
|
||||||
|
// **The host still imports nothing of the mesh's own** (novox/hq ADR 0005). This is its own
|
||||||
|
// interface over its own libraries, and it agrees with the controller only because a conformance
|
||||||
|
// fixture holds both to one envelope.
|
||||||
|
|
||||||
|
// Link is this node's live connection to its mesh: what it hears, and what it says.
|
||||||
|
//
|
||||||
|
// One interface rather than two, because **dialling is where the transport is chosen** and choosing
|
||||||
|
// it twice is how one half of a node ends up on a different bus from the other.
|
||||||
|
type Link interface {
|
||||||
|
// Bus is what this node says: what it applied, and that it is here.
|
||||||
|
Bus
|
||||||
|
|
||||||
|
// Declarations is what the mesh tells this node to be.
|
||||||
|
Declarations() <-chan Declaration
|
||||||
|
|
||||||
|
// Lost says the link ended, and why.
|
||||||
|
//
|
||||||
|
// **Read rather than discovered.** A node that finds out by noticing silence is a node that
|
||||||
|
// believed it was in the mesh for as long as the silence lasted, which is the one state ADR
|
||||||
|
// 0004 says must never look like being connected.
|
||||||
|
Lost() <-chan error
|
||||||
|
|
||||||
|
// Close lets go of whatever was dialled.
|
||||||
|
Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Declaration is one thing the mesh told this node to be.
|
||||||
|
//
|
||||||
|
// **Handled, once — after the report is published.** A node that dies between applying and
|
||||||
|
// reporting leaves the declaration with the mesh and applies it again on return, which is safe
|
||||||
|
// because applying is reconciliation: it converges rather than repeating.
|
||||||
|
//
|
||||||
|
// There is one way of being done rather than two. A declaration set aside because a newer arrived
|
||||||
|
// with it is settled exactly as an applied one is, on both buses, and the difference between them
|
||||||
|
// is a fact the *report* carries — a second method here would be a distinction the transport does
|
||||||
|
// not make.
|
||||||
|
type Declaration interface {
|
||||||
|
// Body is the signed declaration as it arrived, bytes unchanged: a node verifies what it
|
||||||
|
// received rather than what it re-encoded.
|
||||||
|
Body() []byte
|
||||||
|
|
||||||
|
// Handled settles it. Called after the report for it has been published, either way.
|
||||||
|
Handled() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open opens this node's link to its mesh.
|
||||||
|
//
|
||||||
|
// Named Open rather than Dial because Dial is this package's raw TLS dial, which the enrolment path
|
||||||
|
// uses to see a certificate before it trusts anything.
|
||||||
|
//
|
||||||
|
// **Both transports ship and this is the one place that chooses** (novox/hq ADR 0116: nothing moves
|
||||||
|
// a node's bus before step 5). Until then every membership names the bus the mesh runs on today,
|
||||||
|
// and the rollout is this switch and the credential behind it — not a change anywhere in the loop
|
||||||
|
// that reads from what comes back.
|
||||||
|
func Open(ctx context.Context, m Membership, timeout time.Duration) (Link, error) {
|
||||||
|
switch m.Transport {
|
||||||
|
case OnNATS:
|
||||||
|
return dialNats(ctx, m, timeout)
|
||||||
|
default:
|
||||||
|
return dialCurrent(ctx, m, timeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The buses a node can be on. Empty is the one the mesh runs on today, which is every node until
|
||||||
|
// the rollout — so a membership recorded before any of this existed reads as correct rather than as
|
||||||
|
// unset.
|
||||||
|
const (
|
||||||
|
OnCurrent = ""
|
||||||
|
OnNATS = "nats"
|
||||||
|
)
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
amqp "github.com/rabbitmq/amqp091-go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The host's link on the bus the mesh runs on today.
|
||||||
|
//
|
||||||
|
// Moved out of the run loop rather than changed: the dial, the queue, the prefetch window and the
|
||||||
|
// return handler are what they were, because the mesh is running on this and a bus nothing speaks
|
||||||
|
// yet is no reason to alter the one every node is on.
|
||||||
|
|
||||||
|
// currentLink is this node's connection as a channel.
|
||||||
|
type currentLink struct {
|
||||||
|
conn *amqp.Connection
|
||||||
|
channel *amqp.Channel
|
||||||
|
arrived chan Declaration
|
||||||
|
lost chan error
|
||||||
|
}
|
||||||
|
|
||||||
|
func dialCurrent(ctx context.Context, m Membership, timeout time.Duration) (Link, error) {
|
||||||
|
config, err := PinnedConfig(m.Fingerprint)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
dsn := fmt.Sprintf("amqps://%s:%s@%s/",
|
||||||
|
url.QueryEscape(m.Node), url.QueryEscape(m.Password), m.Broker)
|
||||||
|
conn, err := amqp.DialConfig(dsn, amqp.Config{
|
||||||
|
TLSClientConfig: config,
|
||||||
|
Dial: amqp.DefaultDial(timeout),
|
||||||
|
// Kept short so a node that has silently lost its route notices, rather than holding a
|
||||||
|
// connection the broker forgot about and believing it is still in the mesh.
|
||||||
|
Heartbeat: 10 * time.Second,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrWrongCertificate) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("cannot reach the broker at %s: %w", m.Broker, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
channel, err := conn.Channel()
|
||||||
|
if err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
queue := QueueFor(m.Node)
|
||||||
|
if _, err := channel.QueueDeclare(queue, true, false, false, false, nil); err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, fmt.Errorf("cannot declare this node's queue %s: %w", queue, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Applying is one at a time — two at once would race on the same filesystem — but SEEING is
|
||||||
|
// not: with a prefetch of one the host could never know that a newer declaration was already
|
||||||
|
// waiting, and so applied every one of a backlog in turn, at the better part of a minute each,
|
||||||
|
// becoming things nobody wanted any more (novox/hq issue 031). A window of unacknowledged
|
||||||
|
// deliveries lets it drain to the newest; each declaration still survives a restart on the
|
||||||
|
// broker until it is acknowledged, which happens only after it is applied or set aside.
|
||||||
|
if err := channel.Qos(drainDepth, 0, false); err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
deliveries, err := channel.ConsumeWithContext(ctx, queue, "", false, false, false, false, nil)
|
||||||
|
if err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
l := ¤tLink{
|
||||||
|
conn: conn, channel: channel,
|
||||||
|
arrived: make(chan Declaration, drainDepth),
|
||||||
|
lost: make(chan error, 1),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Published mandatory, so the broker hands back anything it cannot route rather than dropping
|
||||||
|
// it. Without this a report goes to an exchange with no matching binding, the publisher is told
|
||||||
|
// nothing, and the mesh believes this node never answered while the node believes it did —
|
||||||
|
// which is what happened when `report` was left unbound on the other side.
|
||||||
|
returned := channel.NotifyReturn(make(chan amqp.Return, 4))
|
||||||
|
go func() {
|
||||||
|
for r := range returned {
|
||||||
|
select {
|
||||||
|
case l.lost <- fmt.Errorf("the broker could not route this node's %s: %s (%d %s)",
|
||||||
|
r.RoutingKey, r.Exchange, r.ReplyCode, r.ReplyText):
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
closed := conn.NotifyClose(make(chan *amqp.Error, 1))
|
||||||
|
go func() {
|
||||||
|
select {
|
||||||
|
case reason := <-closed:
|
||||||
|
select {
|
||||||
|
case l.lost <- fmt.Errorf("the link closed: %v", reason):
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// One goroutine turning the library's deliveries into the mesh's words, so the run loop selects
|
||||||
|
// on one kind of thing whichever bus it is on.
|
||||||
|
go func() {
|
||||||
|
defer close(l.arrived)
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case delivery, ok := <-deliveries:
|
||||||
|
if !ok {
|
||||||
|
select {
|
||||||
|
case l.lost <- errors.New("the broker stopped delivering"):
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case l.arrived <- currentDeclaration{delivery}:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return l, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *currentLink) Declarations() <-chan Declaration { return l.arrived }
|
||||||
|
func (l *currentLink) Lost() <-chan error { return l.lost }
|
||||||
|
|
||||||
|
func (l *currentLink) Close() {
|
||||||
|
if l.channel != nil {
|
||||||
|
_ = l.channel.Close()
|
||||||
|
}
|
||||||
|
if l.conn != nil {
|
||||||
|
_ = l.conn.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Report and Alive are the outbound half, over the channel this link holds.
|
||||||
|
func (l *currentLink) Report(ctx context.Context, node string, body []byte) error {
|
||||||
|
return OverCurrent{Channel: l.channel}.Report(ctx, node, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *currentLink) Alive(ctx context.Context, node string, body []byte) error {
|
||||||
|
return OverCurrent{Channel: l.channel}.Alive(ctx, node, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// currentDeclaration is one delivery from the bus the mesh has.
|
||||||
|
type currentDeclaration struct{ delivery amqp.Delivery }
|
||||||
|
|
||||||
|
func (d currentDeclaration) Body() []byte { return d.delivery.Body }
|
||||||
|
func (d currentDeclaration) Handled() error { return d.delivery.Ack(false) }
|
||||||
@@ -0,0 +1,190 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The host's link on the bus being built.
|
||||||
|
//
|
||||||
|
// Two things a host does here that it cannot do on the other bus, and one it must not try.
|
||||||
|
//
|
||||||
|
// **It declares nothing.** On the bus the mesh has, a host declares its own queue on connecting,
|
||||||
|
// because a queue that is not there means a node that hears nothing. Here the object it reads
|
||||||
|
// through is a durable consumer, and a host's account reaches no part of the JetStream API — by
|
||||||
|
// design, because the controller is the only writer of consumer definitions (design 25 §3). So the
|
||||||
|
// host **binds** to a consumer the controller made when this node enrolled, and a missing one is
|
||||||
|
// said as what it is rather than quietly created with whatever configuration this client happens to
|
||||||
|
// default to.
|
||||||
|
//
|
||||||
|
// **It gets order for free, and keeps the drain anyway.** The declaration subject is last-per-subject
|
||||||
|
// (design 29 §4), so a node that was away receives exactly the current declaration rather than a
|
||||||
|
// queue of superseded ones, and the stream's sequence orders them definitively — the wire-level
|
||||||
|
// answer to novox/hq issue 107. What the drain in run.go still answers is the live case: three
|
||||||
|
// pushes to a *connected* node are three deliveries whatever the stream later retains.
|
||||||
|
|
||||||
|
// EnrolSubject is where a joining machine asks. One subject for every node, because a machine
|
||||||
|
// enrolling has no name the mesh has agreed to yet — which is why its authority to publish here is
|
||||||
|
// the whole of what its enrolment user may do.
|
||||||
|
const EnrolSubject = "mesh.control.enrol"
|
||||||
|
|
||||||
|
// DeclareSubject is where this node's declaration lands. Its own, and no other node's: a host's
|
||||||
|
// account subscribes exactly this and the subject is the authority on which node a declaration is
|
||||||
|
// for.
|
||||||
|
func DeclareSubject(node string) string { return "mesh.node." + node + ".declare" }
|
||||||
|
|
||||||
|
// natsURL is a bus address as the client wants it. A membership records host and port, because that
|
||||||
|
// is what genesis sealed into it and what the other transport takes; the scheme is this transport's
|
||||||
|
// own business.
|
||||||
|
func natsURL(address string) string {
|
||||||
|
if strings.Contains(address, "://") {
|
||||||
|
return address
|
||||||
|
}
|
||||||
|
return "nats://" + address
|
||||||
|
}
|
||||||
|
|
||||||
|
// natsLink is this node's connection as a JetStream subscription.
|
||||||
|
type natsLink struct {
|
||||||
|
conn *nats.Conn
|
||||||
|
js nats.JetStreamContext
|
||||||
|
sub *nats.Subscription
|
||||||
|
node string
|
||||||
|
arrived chan Declaration
|
||||||
|
lost chan error
|
||||||
|
}
|
||||||
|
|
||||||
|
func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, error) {
|
||||||
|
// Pinned exactly as the other transport is, and for once the Go client makes that easy: it
|
||||||
|
// takes a *tls.Config, so the same PinnedConfig with the same VerifyPeerCertificate does the
|
||||||
|
// work. **The constraint recorded against the tool runtime does not apply here** — that client
|
||||||
|
// takes PEM strings with no verify hook, which is why the bus's certificate must carry a name
|
||||||
|
// matching the address *modules* dial it by. A host checks the fingerprint and nothing else.
|
||||||
|
config, err := PinnedConfig(m.Fingerprint)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
opts := []nats.Option{
|
||||||
|
nats.Secure(config),
|
||||||
|
// The mesh names a machine's bus user "node.<name>" (the controller's principal scheme), and
|
||||||
|
// the server refused the bare name the first time a machine dialled it: "authentication
|
||||||
|
// error - User". The same string the mesh composed into the user list, or nothing connects.
|
||||||
|
nats.UserInfo("node."+m.Node, m.Password),
|
||||||
|
nats.Name("mesh-host/" + m.Node),
|
||||||
|
nats.Timeout(timeout),
|
||||||
|
// A node that has silently lost its route notices, rather than holding a connection the
|
||||||
|
// server forgot about and believing it is still in the mesh.
|
||||||
|
nats.PingInterval(10 * time.Second),
|
||||||
|
nats.MaxPingsOutstanding(2),
|
||||||
|
// Reconnection is the caller's: Hold already decides when to try again and how long to
|
||||||
|
// wait, and a client quietly reconnecting underneath it would make that reasoning a
|
||||||
|
// duplicate of the library's.
|
||||||
|
nats.NoReconnect(),
|
||||||
|
}
|
||||||
|
|
||||||
|
conn, err := nats.Connect(natsURL(m.Broker), opts...)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrWrongCertificate) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("cannot reach the bus at %s: %w", m.Broker, err)
|
||||||
|
}
|
||||||
|
js, err := conn.JetStream()
|
||||||
|
if err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, fmt.Errorf("the bus at %s has no JetStream: %w", m.Broker, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
l := &natsLink{
|
||||||
|
conn: conn, js: js, node: m.Node,
|
||||||
|
arrived: make(chan Declaration, drainDepth),
|
||||||
|
lost: make(chan error, 1),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bound to the consumer the controller made for this node, named after the node because that is
|
||||||
|
// what the node's own ack grant allows (`$JS.ACK.NODES.<node>.>`).
|
||||||
|
feed := make(chan *nats.Msg, drainDepth)
|
||||||
|
// The subject as well as the binding: the client checks what is asked for against the
|
||||||
|
// consumer's own filter, and an empty subject is refused rather than taken to mean "whatever
|
||||||
|
// that consumer delivers".
|
||||||
|
sub, err := js.ChanSubscribe(DeclareSubject(m.Node), feed, nats.Bind("NODES", m.Node))
|
||||||
|
if err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"this node cannot read its declarations: %w. The mesh creates that when a node enrols, "+
|
||||||
|
"and a host may not create one itself — so this is the mesh's to answer, not this "+
|
||||||
|
"machine's", err)
|
||||||
|
}
|
||||||
|
l.sub = sub
|
||||||
|
|
||||||
|
conn.SetDisconnectErrHandler(func(_ *nats.Conn, err error) {
|
||||||
|
select {
|
||||||
|
case l.lost <- fmt.Errorf("the link dropped: %w", err):
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
})
|
||||||
|
conn.SetClosedHandler(func(*nats.Conn) {
|
||||||
|
select {
|
||||||
|
case l.lost <- errors.New("the link closed"):
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer close(l.arrived)
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case msg, ok := <-feed:
|
||||||
|
if !ok {
|
||||||
|
select {
|
||||||
|
case l.lost <- errors.New("the bus stopped delivering"):
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case l.arrived <- natsDeclaration{msg}:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return l, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *natsLink) Declarations() <-chan Declaration { return l.arrived }
|
||||||
|
func (l *natsLink) Lost() <-chan error { return l.lost }
|
||||||
|
|
||||||
|
func (l *natsLink) Close() {
|
||||||
|
if l.sub != nil {
|
||||||
|
_ = l.sub.Unsubscribe()
|
||||||
|
}
|
||||||
|
if l.conn != nil {
|
||||||
|
l.conn.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *natsLink) Report(ctx context.Context, node string, body []byte) error {
|
||||||
|
return OverNATS{Conn: l.conn, JS: l.js}.Report(ctx, node, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *natsLink) Alive(ctx context.Context, node string, body []byte) error {
|
||||||
|
return OverNATS{Conn: l.conn, JS: l.js}.Alive(ctx, node, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// natsDeclaration is one declaration off the NODES stream.
|
||||||
|
type natsDeclaration struct{ msg *nats.Msg }
|
||||||
|
|
||||||
|
func (d natsDeclaration) Body() []byte { return d.msg.Data }
|
||||||
|
|
||||||
|
// Handled acknowledges it. The ack goes to this node's own ack subject, which is the one thing
|
||||||
|
// besides its reports a node's account may publish.
|
||||||
|
func (d natsDeclaration) Handled() error { return d.msg.Ack() }
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The host's link against a real server, because every claim here is about one.
|
||||||
|
//
|
||||||
|
// Whether binding to a consumer the host did not create works, whether a declaration on the node's
|
||||||
|
// own subject arrives, whether acknowledging it removes it from the consumer's pending — none of
|
||||||
|
// that can be reasoned out, and the first two are the ones that would leave a node silently hearing
|
||||||
|
// nothing:
|
||||||
|
//
|
||||||
|
// docker run -d --rm --name t -p 14223:4222 nats:2.10-alpine -js
|
||||||
|
// MESH_TEST_NATS=nats://127.0.0.1:14223 go test ./internal/link/ -run TestNats
|
||||||
|
|
||||||
|
func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) {
|
||||||
|
t.Helper()
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(conn.Close)
|
||||||
|
js, err := conn.JetStream()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// **Ensured and purged, not deleted and recreated.** Delete-then-add looked like a reset and is
|
||||||
|
// not one: a test that did that inherited the previous test's messages, and the symptom was a
|
||||||
|
// declaration counted as delivered twice — which reads as a redelivery bug in the code under
|
||||||
|
// test rather than as a dirty stream. Purge is defined to empty a stream; recreating one is a
|
||||||
|
// race with the server's own teardown.
|
||||||
|
for _, want := range []*nats.StreamConfig{
|
||||||
|
{Name: "NODES", Subjects: []string{"mesh.node.*.declare"}, MaxMsgsPerSubject: 1},
|
||||||
|
{Name: "CONTROL", Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"},
|
||||||
|
Retention: nats.WorkQueuePolicy},
|
||||||
|
} {
|
||||||
|
if _, err := js.StreamInfo(want.Name); err != nil {
|
||||||
|
if _, err := js.AddStream(want); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := js.PurgeStream(want.Name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return conn, js
|
||||||
|
}
|
||||||
|
|
||||||
|
// theMeshMakes is the consumer the controller creates when a node enrols. Made here by the test
|
||||||
|
// because the host may not: its account reaches no part of the JetStream API, which is the whole
|
||||||
|
// reason this binds rather than subscribes.
|
||||||
|
//
|
||||||
|
// Removed afterwards, and each test names its own node: two tests sharing a consumer name share its
|
||||||
|
// delivery count and its pending list, and the first thing that goes wrong reads as a fault in the
|
||||||
|
// host rather than in the test beside it.
|
||||||
|
func theMeshMakes(t *testing.T, js nats.JetStreamContext, node string) {
|
||||||
|
t.Helper()
|
||||||
|
t.Cleanup(func() { _ = js.DeleteConsumer("NODES", node) })
|
||||||
|
if _, err := js.AddConsumer("NODES", &nats.ConsumerConfig{
|
||||||
|
Durable: node,
|
||||||
|
FilterSubject: DeclareSubject(node),
|
||||||
|
AckPolicy: nats.AckExplicitPolicy,
|
||||||
|
AckWait: 300 * time.Second,
|
||||||
|
DeliverSubject: "_DELIVER." + node,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func signedBy(t *testing.T, key ed25519.PrivateKey, declaration []byte) []byte {
|
||||||
|
t.Helper()
|
||||||
|
body, err := json.Marshal(Signed{
|
||||||
|
Declaration: declaration, Signature: ed25519.Sign(key, declaration),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return body
|
||||||
|
}
|
||||||
|
|
||||||
|
// A declaration on this node's own subject reaches the host, is applied, and acknowledging it
|
||||||
|
// empties the consumer — which is what tells the mesh the node has it.
|
||||||
|
func TestNatsADeclarationReachesTheHostAndIsSettled(t *testing.T) {
|
||||||
|
conn, js := aBus(t)
|
||||||
|
const node = "settling"
|
||||||
|
theMeshMakes(t, js, node)
|
||||||
|
|
||||||
|
public, private, _ := ed25519.GenerateKey(nil)
|
||||||
|
m := Membership{Node: node, Signer: public}
|
||||||
|
|
||||||
|
// Dialled directly rather than through Open: the test server has no TLS, and what is being
|
||||||
|
// checked is the subscription and the settling, not the pin — which PinnedConfig owns and its
|
||||||
|
// own tests cover.
|
||||||
|
l := &natsLink{conn: conn, js: js, node: node,
|
||||||
|
arrived: make(chan Declaration, drainDepth), lost: make(chan error, 1)}
|
||||||
|
feed := make(chan *nats.Msg, drainDepth)
|
||||||
|
sub, err := js.ChanSubscribe(DeclareSubject(node), feed, nats.Bind("NODES", node))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the host could not bind to the consumer the mesh made for it: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
go func() {
|
||||||
|
for msg := range feed {
|
||||||
|
l.arrived <- natsDeclaration{msg}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if _, err := js.Publish(DeclareSubject(node),
|
||||||
|
signedBy(t, private, []byte(`{"declared":"d1"}`))); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case d := <-l.Declarations():
|
||||||
|
report := handleBody(context.Background(), m, d.Body(),
|
||||||
|
func(context.Context, []byte, []byte) Report {
|
||||||
|
return Report{Applied: []string{"store"}}
|
||||||
|
})
|
||||||
|
if report.Refused != "" {
|
||||||
|
t.Fatalf("a declaration the mesh signed was refused: %s", report.Refused)
|
||||||
|
}
|
||||||
|
if err := d.Handled(); err != nil {
|
||||||
|
t.Fatalf("the node could not acknowledge its own declaration: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(8 * time.Second):
|
||||||
|
t.Fatal("no declaration reached the host")
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Nothing pending is the property**; a delivery count is not. Delivery is at-least-once by
|
||||||
|
// design, so pinning "delivered exactly once" would be asserting something the mesh does not
|
||||||
|
// rely on. What matters is that the acknowledgement landed, so the mesh can tell the node has
|
||||||
|
// it — and that no redelivery was needed to get there, which is what would say the node was
|
||||||
|
// too slow to answer for its own ack wait.
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
var last string
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
info, err := js.ConsumerInfo("NODES", node)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
last = err.Error()
|
||||||
|
case info.NumAckPending == 0 && info.NumRedelivered == 0:
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
last = fmt.Sprintf("pending %d, redelivered %d", info.NumAckPending, info.NumRedelivered)
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("the declaration was not settled, so the mesh cannot tell the node has it: %s", last)
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A node that was away gets exactly the current declaration and nothing older.** Three pushed
|
||||||
|
// while nothing is listening leave one on the stream, and it is the newest — the wire-level answer
|
||||||
|
// to novox/hq issue 107, and the half of the drain that stops being the host's problem.
|
||||||
|
func TestNatsANodeThatWasAwayGetsOnlyTheNewest(t *testing.T) {
|
||||||
|
_, js := aBus(t)
|
||||||
|
const node = "returning"
|
||||||
|
_, private, _ := ed25519.GenerateKey(nil)
|
||||||
|
|
||||||
|
for _, id := range []string{"d1", "d2", "d3"} {
|
||||||
|
if _, err := js.Publish(DeclareSubject(node),
|
||||||
|
signedBy(t, private, []byte(`{"declared":"`+id+`"}`))); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
info, err := js.StreamInfo("NODES")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info.State.Msgs != 1 {
|
||||||
|
t.Fatalf("%d declarations survived for one node; a node that was away would apply a backlog "+
|
||||||
|
"of things nobody wants any more", info.State.Msgs)
|
||||||
|
}
|
||||||
|
|
||||||
|
theMeshMakes(t, js, node)
|
||||||
|
feed := make(chan *nats.Msg, drainDepth)
|
||||||
|
sub, err := js.ChanSubscribe(DeclareSubject(node), feed, nats.Bind("NODES", node))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case msg := <-feed:
|
||||||
|
if declaredIn(msg.Data) != "d3" {
|
||||||
|
t.Fatalf("the node was given %q rather than the newest", declaredIn(msg.Data))
|
||||||
|
}
|
||||||
|
case <-time.After(8 * time.Second):
|
||||||
|
t.Fatal("the node that was away was given nothing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A report goes through the stream and a heartbeat does not: the one that must survive the
|
||||||
|
// controller's store restarting is kept, and the one that must not is not.
|
||||||
|
func TestNatsAReportIsKeptAndAHeartbeatIsNot(t *testing.T) {
|
||||||
|
conn, js := aBus(t)
|
||||||
|
bus := OverNATS{Conn: conn, JS: js}
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1"})
|
||||||
|
if err := bus.Report(ctx, "anchor", body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
beat, _ := json.Marshal(Alive{Node: "anchor"})
|
||||||
|
if err := bus.Alive(ctx, "anchor", beat); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := js.StreamInfo("CONTROL")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info.State.Msgs != 1 {
|
||||||
|
t.Fatalf("%d messages were kept; a report must be and a heartbeat must not", info.State.Msgs)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,10 @@
|
|||||||
package link
|
package link
|
||||||
|
|
||||||
import "time"
|
import (
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
// The wire formats shared with the control plane, which defines them separately because this
|
// The wire formats shared with the control plane, which defines them separately because this
|
||||||
// binary requires nothing present and does not import it. A test on each side asserts the field
|
// binary requires nothing present and does not import it. A test on each side asserts the field
|
||||||
@@ -100,6 +104,67 @@ type Report struct {
|
|||||||
// published container port. Only an adopted node reports it; it is what converging the node
|
// published container port. Only an adopted node reports it; it is what converging the node
|
||||||
// previews, so nothing closes without being named first.
|
// previews, so nothing closes without being named first.
|
||||||
Reachable []Reach `json:"reachable,omitempty"`
|
Reachable []Reach `json:"reachable,omitempty"`
|
||||||
|
|
||||||
|
// Tunnel is what this adopted node says about the tunnel it found and carried (novox/hq ADR
|
||||||
|
// 0105): which interface, its port, range and peer count, whether the found interface is down
|
||||||
|
// and the mesh's up in its place, and where the found configuration's original was kept.
|
||||||
|
Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
|
||||||
|
|
||||||
|
// Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq
|
||||||
|
// ADR 0105). Not an account of the machine: a report carrying one says nothing else.
|
||||||
|
Rekey *Rekey `json:"rekey,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CarriedTunnel is this node's account of the tunnel it took over. State is one of the Carried
|
||||||
|
// states below; Note is what the host did about it, when it did something.
|
||||||
|
type CarriedTunnel struct {
|
||||||
|
Interface string `json:"interface"`
|
||||||
|
Port int `json:"port"`
|
||||||
|
Range string `json:"range"`
|
||||||
|
Peers int `json:"peers"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Note string `json:"note,omitempty"`
|
||||||
|
Kept string `json:"kept,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// The states a carried tunnel can be in: the found interface still up and the mesh's not; the
|
||||||
|
// found one down and the mesh's up with its key; or the found one down and the mesh's not up — the
|
||||||
|
// one state where the peers reach nothing, said as its own word so nothing reads it as either of
|
||||||
|
// the others.
|
||||||
|
const (
|
||||||
|
CarriedNotTaken = "not-taken"
|
||||||
|
CarriedTaken = "taken"
|
||||||
|
CarriedDown = "down"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Rekey is this node saying it took a found tunnel's key as its overlay key after enrolling
|
||||||
|
// (novox/hq ADR 0105): the path for a node that enrolled before the mesh knew to take a tunnel
|
||||||
|
// over, since re-enrolling would rotate every key it holds. Signed with the identity key over
|
||||||
|
// RekeyProof, so a report forged on a stolen broker account cannot move this node's overlay key.
|
||||||
|
type Rekey struct {
|
||||||
|
// Previous is the overlay key this node held until now, as the mesh records it; the mesh
|
||||||
|
// refuses a rekey naming another, which is how a replayed one is refused.
|
||||||
|
Previous string `json:"previous"`
|
||||||
|
OverlayKey string `json:"overlay_key"`
|
||||||
|
Tunnel *Tunnel `json:"tunnel"`
|
||||||
|
Proof []byte `json:"proof"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RekeyProof is what a node signs when it rekeys — the node, the key it leaves, the key it takes
|
||||||
|
// and the tunnel it took it from — so a proof cannot be moved to another node or another tunnel.
|
||||||
|
// Byte for byte the mesh's own (mesh-controller internal/link RekeyProof).
|
||||||
|
func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte {
|
||||||
|
var t Tunnel
|
||||||
|
if tunnel != nil {
|
||||||
|
t = *tunnel
|
||||||
|
}
|
||||||
|
peers := make([]string, 0, len(t.Peers))
|
||||||
|
for _, p := range t.Peers {
|
||||||
|
peers = append(peers, p.PublicKey+"@"+p.Address)
|
||||||
|
}
|
||||||
|
return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" +
|
||||||
|
t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" +
|
||||||
|
t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ","))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Held is one file or container found on an adopted node and kept as it was.
|
// Held is one file or container found on an adopted node and kept as it was.
|
||||||
|
|||||||
@@ -3,33 +3,46 @@ package link
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
amqp "github.com/rabbitmq/amqp091-go"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// said is one declaration as a test hands it over, with no transport under it — which is what the
|
||||||
|
// seam bought: the drain's reasoning was reachable only through a real broker before.
|
||||||
|
type said struct {
|
||||||
|
body []byte
|
||||||
|
handled bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *said) Body() []byte { return s.body }
|
||||||
|
func (s *said) Handled() error { s.handled = true; return nil }
|
||||||
|
|
||||||
|
func arriving(bodies ...string) chan Declaration {
|
||||||
|
ch := make(chan Declaration, 8)
|
||||||
|
for _, b := range bodies {
|
||||||
|
ch <- &said{body: []byte(b)}
|
||||||
|
}
|
||||||
|
return ch
|
||||||
|
}
|
||||||
|
|
||||||
// A machine asked to be five things becomes the last one: what is already waiting supersedes what
|
// A machine asked to be five things becomes the last one: what is already waiting supersedes what
|
||||||
// arrived first, and everything set aside is named so it can be reported.
|
// arrived first, and everything set aside is named so it can be reported.
|
||||||
func TestWhatIsAlreadyWaitingSupersedesWhatArrivedFirst(t *testing.T) {
|
func TestWhatIsAlreadyWaitingSupersedesWhatArrivedFirst(t *testing.T) {
|
||||||
deliveries := make(chan amqp.Delivery, 8)
|
waiting := arriving("two", "three", "four")
|
||||||
for _, id := range []string{"two", "three", "four"} {
|
apply, superseded := newest(waiting, &said{body: []byte("one")}, 50*time.Millisecond)
|
||||||
deliveries <- amqp.Delivery{Body: []byte(id)}
|
if string(apply.Body()) != "four" {
|
||||||
|
t.Fatalf("applied %q, not the newest", apply.Body())
|
||||||
}
|
}
|
||||||
apply, superseded := newest(deliveries, amqp.Delivery{Body: []byte("one")}, 50*time.Millisecond)
|
if len(superseded) != 3 || string(superseded[0].Body()) != "one" ||
|
||||||
if string(apply.Body) != "four" {
|
string(superseded[2].Body()) != "three" {
|
||||||
t.Fatalf("applied %q, not the newest", apply.Body)
|
|
||||||
}
|
|
||||||
if len(superseded) != 3 || string(superseded[0].Body) != "one" || string(superseded[2].Body) != "three" {
|
|
||||||
t.Fatalf("set aside %d: %v", len(superseded), superseded)
|
t.Fatalf("set aside %d: %v", len(superseded), superseded)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// One declaration with nothing behind it is applied as it always was, after the window.
|
// One declaration with nothing behind it is applied as it always was, after the window.
|
||||||
func TestALoneDeclarationIsAppliedAfterTheWindow(t *testing.T) {
|
func TestALoneDeclarationIsAppliedAfterTheWindow(t *testing.T) {
|
||||||
deliveries := make(chan amqp.Delivery, 1)
|
|
||||||
began := time.Now()
|
began := time.Now()
|
||||||
apply, superseded := newest(deliveries, amqp.Delivery{Body: []byte("only")}, 30*time.Millisecond)
|
apply, superseded := newest(arriving(), &said{body: []byte("only")}, 30*time.Millisecond)
|
||||||
if string(apply.Body) != "only" || len(superseded) != 0 {
|
if string(apply.Body()) != "only" || len(superseded) != 0 {
|
||||||
t.Fatalf("got %q with %d set aside", apply.Body, len(superseded))
|
t.Fatalf("got %q with %d set aside", apply.Body(), len(superseded))
|
||||||
}
|
}
|
||||||
if time.Since(began) < 30*time.Millisecond {
|
if time.Since(began) < 30*time.Millisecond {
|
||||||
t.Fatal("did not wait the window for a straggler")
|
t.Fatal("did not wait the window for a straggler")
|
||||||
@@ -38,13 +51,13 @@ func TestALoneDeclarationIsAppliedAfterTheWindow(t *testing.T) {
|
|||||||
|
|
||||||
// A straggler within the window is taken; one after it is the next push.
|
// A straggler within the window is taken; one after it is the next push.
|
||||||
func TestAStragglerWithinTheWindowIsTaken(t *testing.T) {
|
func TestAStragglerWithinTheWindowIsTaken(t *testing.T) {
|
||||||
deliveries := make(chan amqp.Delivery, 2)
|
waiting := arriving()
|
||||||
go func() {
|
go func() {
|
||||||
time.Sleep(20 * time.Millisecond)
|
time.Sleep(20 * time.Millisecond)
|
||||||
deliveries <- amqp.Delivery{Body: []byte("late")}
|
waiting <- &said{body: []byte("late")}
|
||||||
}()
|
}()
|
||||||
apply, superseded := newest(deliveries, amqp.Delivery{Body: []byte("first")}, 100*time.Millisecond)
|
apply, superseded := newest(waiting, &said{body: []byte("first")}, 100*time.Millisecond)
|
||||||
if string(apply.Body) != "late" || len(superseded) != 1 {
|
if string(apply.Body()) != "late" || len(superseded) != 1 {
|
||||||
t.Fatalf("got %q with %d set aside", apply.Body, len(superseded))
|
t.Fatalf("got %q with %d set aside", apply.Body(), len(superseded))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+71
-94
@@ -6,10 +6,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/url"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
amqp "github.com/rabbitmq/amqp091-go"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrForged is what a node returns for a declaration whose signature is not the mesh's.
|
// ErrForged is what a node returns for a declaration whose signature is not the mesh's.
|
||||||
@@ -33,6 +30,10 @@ type Membership struct {
|
|||||||
Fingerprint string
|
Fingerprint string
|
||||||
Password string
|
Password string
|
||||||
Signer ed25519.PublicKey
|
Signer ed25519.PublicKey
|
||||||
|
// Transport is which bus this node speaks (hearing.go). Empty is the one the mesh runs on
|
||||||
|
// today, which is every node until the rollout — so a membership recorded before any of this
|
||||||
|
// existed reads as correct rather than as unset.
|
||||||
|
Transport string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Applier is what the host does with a declaration that has been proved to come from the mesh.
|
// Applier is what the host does with a declaration that has been proved to come from the mesh.
|
||||||
@@ -190,108 +191,62 @@ func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout
|
|||||||
if say == nil {
|
if say == nil {
|
||||||
say = func(string) {}
|
say = func(string) {}
|
||||||
}
|
}
|
||||||
config, err := PinnedConfig(m.Fingerprint)
|
link, err := Open(ctx, m, timeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer link.Close()
|
||||||
|
|
||||||
dsn := fmt.Sprintf("amqps://%s:%s@%s/",
|
// Said, because it is the event anybody watching actually wants. Without it a node logs every
|
||||||
url.QueryEscape(m.Node), url.QueryEscape(m.Password), m.Broker)
|
// failure and nothing on success, so a log full of "trying again" and then silence reads as
|
||||||
conn, err := amqp.DialConfig(dsn, amqp.Config{
|
// still broken when it means the opposite.
|
||||||
TLSClientConfig: config,
|
say("in the mesh, hearing what this node should be")
|
||||||
Dial: amqp.DefaultDial(timeout),
|
|
||||||
// Kept short so a node that has silently lost its route notices, rather than holding a
|
|
||||||
// connection the broker forgot about and believing it is still in the mesh.
|
|
||||||
Heartbeat: 10 * time.Second,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, ErrWrongCertificate) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return fmt.Errorf("cannot reach the broker at %s: %w", m.Broker, err)
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
|
|
||||||
channel, err := conn.Channel()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer channel.Close()
|
|
||||||
|
|
||||||
queue := QueueFor(m.Node)
|
|
||||||
if _, err := channel.QueueDeclare(queue, true, false, false, false, nil); err != nil {
|
|
||||||
return fmt.Errorf("cannot declare this node's queue %s: %w", queue, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Applying is one at a time — two at once would race on the same filesystem — but SEEING is
|
|
||||||
// not: with a prefetch of one the host could never know that a newer declaration was already
|
|
||||||
// waiting, and so applied every one of a backlog in turn, at the better part of a minute each,
|
|
||||||
// becoming things nobody wanted any more (novox/hq issue 031). A window of unacknowledged
|
|
||||||
// deliveries lets it drain to the newest; each declaration still survives a restart on the
|
|
||||||
// broker until it is acknowledged, which happens only after it is applied or set aside.
|
|
||||||
if err := channel.Qos(drainDepth, 0, false); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
deliveries, err := channel.ConsumeWithContext(ctx, queue, "", false, false, false, false, nil)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// Said, because it is the event anybody watching actually wants. Without it a node logs
|
|
||||||
// every failure and nothing on success, so a log full of "trying again" and then silence
|
|
||||||
// reads as still broken when it means the opposite.
|
|
||||||
say("in the mesh, consuming " + queue)
|
|
||||||
|
|
||||||
// A word every so often, so the mesh can tell a node that is quiet from one that is gone.
|
// A word every so often, so the mesh can tell a node that is quiet from one that is gone.
|
||||||
// Cheap on purpose: it carries a name and nothing else, because anything more would be a
|
// Cheap on purpose: it carries a name and nothing else, because anything more would be a
|
||||||
// report, and reports are rare where this is constant.
|
// report, and reports are rare where this is constant.
|
||||||
beat := time.NewTicker(AliveEvery)
|
beat := time.NewTicker(AliveEvery)
|
||||||
defer beat.Stop()
|
defer beat.Stop()
|
||||||
publishAlive(ctx, channel, m, say, timeout)
|
publishAlive(ctx, link, m, say, timeout)
|
||||||
|
|
||||||
closed := conn.NotifyClose(make(chan *amqp.Error, 1))
|
declarations := link.Declarations()
|
||||||
|
|
||||||
// Published mandatory, so the broker hands back anything it cannot route rather than
|
|
||||||
// dropping it. Without this a report goes to an exchange with no matching binding, the
|
|
||||||
// publisher is told nothing, and the mesh believes this node never answered while the node
|
|
||||||
// believes it did — which is what happened when `report` was left unbound on the other side.
|
|
||||||
returned := channel.NotifyReturn(make(chan amqp.Return, 4))
|
|
||||||
go func() {
|
|
||||||
for r := range returned {
|
|
||||||
say(fmt.Sprintf("the broker could not route this node's %s: %s (%d %s)",
|
|
||||||
r.RoutingKey, r.Exchange, r.ReplyCode, r.ReplyText))
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return nil
|
return nil
|
||||||
case <-beat.C:
|
case <-beat.C:
|
||||||
publishAlive(ctx, channel, m, say, timeout)
|
publishAlive(ctx, link, m, say, timeout)
|
||||||
case unasked := <-outbox:
|
case unasked := <-outbox:
|
||||||
// Said without having been asked: a reconcile found what an adopted node holds, or
|
// Said without having been asked: a reconcile found what an adopted node holds, or
|
||||||
// its firewall, changed since it last said.
|
// its firewall, changed since it last said.
|
||||||
published := publishReport(ctx, channel, m, unasked.Report, say, timeout)
|
published := publishReport(ctx, link, m, unasked.Report, say, timeout)
|
||||||
if unasked.Done != nil {
|
if unasked.Done != nil {
|
||||||
unasked.Done(published)
|
unasked.Done(published)
|
||||||
}
|
}
|
||||||
case reason := <-closed:
|
case reason := <-link.Lost():
|
||||||
return fmt.Errorf("the link closed: %v", reason)
|
return reason
|
||||||
case delivery, ok := <-deliveries:
|
case declaration, ok := <-declarations:
|
||||||
if !ok {
|
if !ok {
|
||||||
return errors.New("the broker stopped delivering")
|
// The link's own reason, when it has managed to say one: "stopped delivering" on
|
||||||
|
// its own says nothing about why, and why is the whole of what an operator wants.
|
||||||
|
select {
|
||||||
|
case reason := <-link.Lost():
|
||||||
|
return reason
|
||||||
|
default:
|
||||||
|
return errors.New("the mesh stopped sending this node declarations")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Whatever else is already waiting supersedes this one. Each set-aside declaration
|
// Whatever else is already waiting supersedes this one. Each set-aside declaration is
|
||||||
// is reported as such, then acknowledged unapplied.
|
// reported as such, then settled unapplied.
|
||||||
delivery, superseded := newest(deliveries, delivery, drainWindow)
|
declaration, superseded := newest(declarations, declaration, drainWindow)
|
||||||
for _, old := range superseded {
|
for _, old := range superseded {
|
||||||
say("set aside a declaration: a newer one arrived with it")
|
say("set aside a declaration: a newer one arrived with it")
|
||||||
publishReport(ctx, channel, m, Report{Node: m.Node, Declared: declaredIn(old.Body),
|
publishReport(ctx, link, m, Report{Node: m.Node, Declared: declaredIn(old.Body()),
|
||||||
Superseded: declaredIn(delivery.Body)}, say, timeout)
|
Superseded: declaredIn(declaration.Body())}, say, timeout)
|
||||||
_ = old.Ack(false)
|
_ = old.Handled()
|
||||||
}
|
}
|
||||||
report := handle(ctx, m, apply, delivery)
|
report := handleBody(ctx, m, declaration.Body(), apply)
|
||||||
switch {
|
switch {
|
||||||
case report.Refused != "":
|
case report.Refused != "":
|
||||||
say("refused a declaration: " + report.Refused)
|
say("refused a declaration: " + report.Refused)
|
||||||
@@ -300,12 +255,12 @@ func Run(ctx context.Context, m Membership, apply Applier, say Announce, timeout
|
|||||||
default:
|
default:
|
||||||
say(fmt.Sprintf("applied %d resource(s)", len(report.Applied)))
|
say(fmt.Sprintf("applied %d resource(s)", len(report.Applied)))
|
||||||
}
|
}
|
||||||
publishReport(ctx, channel, m, report, say, timeout)
|
publishReport(ctx, link, m, report, say, timeout)
|
||||||
// Acknowledged after the report is published. A node that dies between applying and
|
// Settled after the report is published. A node that dies between applying and
|
||||||
// reporting leaves the declaration on the broker and applies it again on return,
|
// reporting leaves the declaration with the mesh and applies it again on return,
|
||||||
// which is safe because applying is reconciliation — it converges rather than
|
// which is safe because applying is reconciliation — it converges rather than
|
||||||
// repeating.
|
// repeating.
|
||||||
_ = delivery.Ack(false)
|
_ = declaration.Handled()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -321,12 +276,24 @@ const (
|
|||||||
// newest takes what is already waiting behind `first` and returns the last of them to apply, and
|
// newest takes what is already waiting behind `first` and returns the last of them to apply, and
|
||||||
// the rest to set aside. It waits `window` for a straggler after each arrival and no longer: a
|
// the rest to set aside. It waits `window` for a straggler after each arrival and no longer: a
|
||||||
// declaration in flight from the mesh arrives within that; one that does not is the next push.
|
// declaration in flight from the mesh arrives within that; one that does not is the next push.
|
||||||
func newest(deliveries <-chan amqp.Delivery, first amqp.Delivery, window time.Duration) (amqp.Delivery, []amqp.Delivery) {
|
//
|
||||||
|
// **Its job narrows once declarations are state rather than messages, and does not disappear.**
|
||||||
|
// On the bus being built, a declaration is last-per-subject (novox/hq design 29 §4), so a node
|
||||||
|
// that was away receives exactly the current one instead of a queue of superseded ones — the
|
||||||
|
// catch-up half of what this does is then the stream's. And a stream sequence orders them
|
||||||
|
// definitively, where this window only infers order from arrival time, which is the wire-level
|
||||||
|
// answer to novox/hq issue 107.
|
||||||
|
//
|
||||||
|
// What remains is the live case: three pushes in quick succession to a *connected* node are
|
||||||
|
// three deliveries, whatever the stream later retains. So this is narrowed at the rollout, not
|
||||||
|
// deleted — and saying which half goes is worth more than a note that it "can probably be
|
||||||
|
// removed", which is how a load-bearing window gets deleted by somebody in a hurry.
|
||||||
|
func newest(arriving <-chan Declaration, first Declaration, window time.Duration) (Declaration, []Declaration) {
|
||||||
latest := first
|
latest := first
|
||||||
var superseded []amqp.Delivery
|
var superseded []Declaration
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case next, ok := <-deliveries:
|
case next, ok := <-arriving:
|
||||||
if !ok {
|
if !ok {
|
||||||
return latest, superseded
|
return latest, superseded
|
||||||
}
|
}
|
||||||
@@ -355,10 +322,6 @@ func declaredIn(body []byte) string {
|
|||||||
return d.Declared
|
return d.Declared
|
||||||
}
|
}
|
||||||
|
|
||||||
func handle(ctx context.Context, m Membership, apply Applier, delivery amqp.Delivery) Report {
|
|
||||||
return handleBody(ctx, m, delivery.Body, apply)
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleBody is the whole of deciding whether to trust a message, separated from the broker so it
|
// handleBody is the whole of deciding whether to trust a message, separated from the broker so it
|
||||||
// can be tested as the security check it is rather than as message plumbing.
|
// can be tested as the security check it is rather than as message plumbing.
|
||||||
func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) Report {
|
func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) Report {
|
||||||
@@ -377,7 +340,23 @@ func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) R
|
|||||||
}
|
}
|
||||||
|
|
||||||
// publishReport tells the mesh what this node did, and says whether the broker took it.
|
// publishReport tells the mesh what this node did, and says whether the broker took it.
|
||||||
func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report,
|
// Publish sends one report on this node's own connection and returns: the one-shot path for a
|
||||||
|
// report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same
|
||||||
|
// account, the same pinned certificate and the same exchange as the running host's reports.
|
||||||
|
func Publish(ctx context.Context, m Membership, report Report, timeout time.Duration) error {
|
||||||
|
link, err := Open(ctx, m, timeout)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer link.Close()
|
||||||
|
var said string
|
||||||
|
if !publishReport(ctx, link, m, report, func(s string) { said = s }, timeout) {
|
||||||
|
return errors.New(said)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func publishReport(ctx context.Context, bus Bus, m Membership, report Report,
|
||||||
say Announce, timeout time.Duration) bool {
|
say Announce, timeout time.Duration) bool {
|
||||||
report.Node = m.Node
|
report.Node = m.Node
|
||||||
body, err := json.Marshal(report)
|
body, err := json.Marshal(report)
|
||||||
@@ -391,8 +370,7 @@ func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, rep
|
|||||||
// Said rather than swallowed. A report that fails to publish leaves the mesh believing this
|
// Said rather than swallowed. A report that fails to publish leaves the mesh believing this
|
||||||
// node never answered, while the node believes it did — and the two would go on disagreeing
|
// node never answered, while the node believes it did — and the two would go on disagreeing
|
||||||
// with nothing anywhere saying so. That shape of fault is the one this project keeps finding.
|
// with nothing anywhere saying so. That shape of fault is the one this project keeps finding.
|
||||||
if err := channel.PublishWithContext(publish, Exchange, KeyReport, true, false,
|
if err := bus.Report(publish, m.Node, body); err != nil {
|
||||||
amqp.Publishing{ContentType: "application/json", Body: body}); err != nil {
|
|
||||||
say(fmt.Sprintf("applied, and could not tell the mesh: %v", err))
|
say(fmt.Sprintf("applied, and could not tell the mesh: %v", err))
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@@ -400,7 +378,7 @@ func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, rep
|
|||||||
}
|
}
|
||||||
|
|
||||||
// publishAlive says this node is here, and nothing else.
|
// publishAlive says this node is here, and nothing else.
|
||||||
func publishAlive(ctx context.Context, channel *amqp.Channel, m Membership, say Announce,
|
func publishAlive(ctx context.Context, bus Bus, m Membership, say Announce,
|
||||||
timeout time.Duration) {
|
timeout time.Duration) {
|
||||||
body, err := json.Marshal(Alive{Node: m.Node})
|
body, err := json.Marshal(Alive{Node: m.Node})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -411,8 +389,7 @@ func publishAlive(ctx context.Context, channel *amqp.Channel, m Membership, say
|
|||||||
// Not mandatory, unlike a report. Losing one is nothing: the next is a minute away, and the
|
// Not mandatory, unlike a report. Losing one is nothing: the next is a minute away, and the
|
||||||
// mesh is reading a gap rather than counting arrivals. Insisting on delivery would turn a
|
// mesh is reading a gap rather than counting arrivals. Insisting on delivery would turn a
|
||||||
// harmless miss into a logged failure every minute.
|
// harmless miss into a logged failure every minute.
|
||||||
if err := channel.PublishWithContext(publish, Exchange, KeyAlive, false, false,
|
if err := bus.Alive(publish, m.Node, body); err != nil {
|
||||||
amqp.Publishing{ContentType: "application/json", Body: body}); err != nil {
|
|
||||||
say("could not tell the mesh this node is here: " + err.Error())
|
say("could not tell the mesh this node is here: " + err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -80,6 +80,25 @@ func SaveDeclared(path string, d Declared) error {
|
|||||||
return os.Rename(tmp.Name(), path)
|
return os.Rename(tmp.Name(), path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReadDeclared reads what was kept without proving it is the mesh's.
|
||||||
|
//
|
||||||
|
// For naming and comparing only — which declaration this node was last told, and what mode it
|
||||||
|
// said — never for applying. A declaration to apply goes through LoadDeclared, which verifies.
|
||||||
|
func ReadDeclared(path string) (Declared, error) {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return Declared{}, ErrNothingDeclared
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return Declared{}, fmt.Errorf("this node was told something and cannot read it back: %w", err)
|
||||||
|
}
|
||||||
|
var d Declared
|
||||||
|
if err := json.Unmarshal(raw, &d); err != nil {
|
||||||
|
return Declared{}, fmt.Errorf("what this node was told is unreadable at %s: %w", path, err)
|
||||||
|
}
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
// LoadDeclared reads it back and proves it is still the mesh's.
|
// LoadDeclared reads it back and proves it is still the mesh's.
|
||||||
//
|
//
|
||||||
// Verified against the signing key this node holds, which came from its token. A declaration on
|
// Verified against the signing key this node holds, which came from its token. A declaration on
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// One apply at a time on a machine, whoever asks.
|
||||||
|
//
|
||||||
|
// Three things apply here and each reads the state, acts, and writes the state back: the link
|
||||||
|
// and the reconcile loop inside `mesh-host run`, the host's own `reconcile` and `apply` run by
|
||||||
|
// hand, and the installer at genesis. Inside one process a mutex serialises them; across
|
||||||
|
// processes nothing did, and two applies interleaved leave the last saver writing a state read
|
||||||
|
// before the other acted — a hold, a firewall record, a resource just applied, lost (novox/hq
|
||||||
|
// issue 104 review). A lock on a file beside the state is what every one of them can take, however
|
||||||
|
// it was started: a `reconcile` run against a service, or against a `mesh-host run` somebody
|
||||||
|
// started by hand, waits the same way. Refusing while a named service is active would have known
|
||||||
|
// the service's name and missed the hand-started one.
|
||||||
|
//
|
||||||
|
// An advisory lock, held for the life of the open file and released by the kernel when the
|
||||||
|
// process ends, so a host that dies mid-apply leaves no lock behind for the next one to clear.
|
||||||
|
|
||||||
|
// LockName is the file the lock is taken on, beside the state.
|
||||||
|
const LockName = "state.lock"
|
||||||
|
|
||||||
|
// Lock takes the machine's apply lock and returns what releases it. When another process holds
|
||||||
|
// it, wait is told once — so a person running a command knows what they are waiting for — and
|
||||||
|
// Lock blocks until it is free.
|
||||||
|
func Lock(statePath string, wait func()) (func(), error) {
|
||||||
|
dir := filepath.Dir(statePath)
|
||||||
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
f, err := os.OpenFile(filepath.Join(dir, LockName), os.O_CREATE|os.O_RDWR, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot take this node's apply lock: %w", err)
|
||||||
|
}
|
||||||
|
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
||||||
|
if !errors.Is(err, syscall.EWOULDBLOCK) {
|
||||||
|
f.Close()
|
||||||
|
return nil, fmt.Errorf("cannot take this node's apply lock: %w", err)
|
||||||
|
}
|
||||||
|
if wait != nil {
|
||||||
|
wait()
|
||||||
|
}
|
||||||
|
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return nil, fmt.Errorf("waiting for this node's apply lock: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return func() {
|
||||||
|
_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
||||||
|
f.Close()
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends the node's record across processes: a `reconcile` run by hand beside the link service,
|
||||||
|
// or the installer beside either, waits for the other's apply rather than saving over it.
|
||||||
|
func TestASecondApplyWaitsForTheFirst(t *testing.T) {
|
||||||
|
state := filepath.Join(t.TempDir(), "state.json")
|
||||||
|
release, err := Lock(state, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
waited := make(chan struct{}, 1)
|
||||||
|
got := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
unlock, err := Lock(state, func() { waited <- struct{}{} })
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
close(got)
|
||||||
|
unlock()
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-waited:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the second apply was not told it is waiting")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-got:
|
||||||
|
t.Fatal("the second apply took the lock while the first held it")
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
select {
|
||||||
|
case <-got:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the second apply never got the lock once the first let go")
|
||||||
|
}
|
||||||
|
}
|
||||||
+178
-1
@@ -70,6 +70,25 @@ type Applied struct {
|
|||||||
// anywhere saying why.
|
// anywhere saying why.
|
||||||
Wrote string `json:"wrote,omitempty"`
|
Wrote string `json:"wrote,omitempty"`
|
||||||
|
|
||||||
|
// Kept is where the original of a file this host wrote over was kept (novox/hq ADR 0100):
|
||||||
|
// by the keep on its first write, or by the hold that was released when its module was taken.
|
||||||
|
// Recorded rather than only reported, because a file once written whole and now written into
|
||||||
|
// (novox/hq issue 128) is given back its original with the mesh's region in it — and a path
|
||||||
|
// said once in a log line is not a path the host can find again.
|
||||||
|
Kept string `json:"kept,omitempty"`
|
||||||
|
|
||||||
|
// Stateless is, for a service, that its unit's lifecycle was never the mesh's (novox/hq ADR
|
||||||
|
// 0117) — kept here because removal happens once the declaration that said so is gone, and a
|
||||||
|
// service removed as if it had a state is stopped: the machine's network manager, for one.
|
||||||
|
Stateless bool `json:"stateless,omitempty"`
|
||||||
|
|
||||||
|
// Found is, for a service, the state its unit was in when this host first applied it — before
|
||||||
|
// the mesh started, stopped, enabled or disabled anything. Removal gives that back and nothing
|
||||||
|
// more (novox/hq ADR 0118): a unit that was running before the mesh arrived keeps running
|
||||||
|
// when its declaration goes; one the mesh started is stopped again. Absent on a record written
|
||||||
|
// before the host kept it, and then the unit is left exactly as it is.
|
||||||
|
Found *FoundUnit `json:"found,omitempty"`
|
||||||
|
|
||||||
// Into is set for a file written into rather than over (novox/hq ADR 0102): the format, what
|
// Into is set for a file written into rather than over (novox/hq ADR 0102): the format, what
|
||||||
// each of the mesh's keys held before it set them, which of them were absent, and whether the
|
// each of the mesh's keys held before it set them, which of them were absent, and whether the
|
||||||
// file itself was — so undeclaring it gives the machine back exactly what it had.
|
// file itself was — so undeclaring it gives the machine back exactly what it had.
|
||||||
@@ -84,7 +103,7 @@ type Applied struct {
|
|||||||
Reads map[string]string `json:"reads,omitempty"`
|
Reads map[string]string `json:"reads,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Into is what a file written into held before the mesh's keys.
|
// Into is what a file written into held before the mesh's keys, or before the mesh's block.
|
||||||
type Into struct {
|
type Into struct {
|
||||||
Format string `json:"format"`
|
Format string `json:"format"`
|
||||||
Before map[string]json.RawMessage `json:"before,omitempty"`
|
Before map[string]json.RawMessage `json:"before,omitempty"`
|
||||||
@@ -94,6 +113,22 @@ type Into struct {
|
|||||||
// to the machine's list — never a member that was already there. Undeclared, only these go,
|
// to the machine's list — never a member that was already there. Undeclared, only these go,
|
||||||
// and drift is judged on these alone (novox/hq ADR 0102).
|
// and drift is judged on these alone (novox/hq ADR 0102).
|
||||||
Added map[string][]json.RawMessage `json:"added,omitempty"`
|
Added map[string][]json.RawMessage `json:"added,omitempty"`
|
||||||
|
|
||||||
|
// Region is, for a file written into a block (novox/hq issue 128), what the lines between the
|
||||||
|
// mesh's markers held before the mesh wrote them — nil when there was no region, which is
|
||||||
|
// different from a region that was there and empty. Undeclared, a recorded region is put back
|
||||||
|
// and an unrecorded one is taken out, markers and all. It is the block's "what each key held
|
||||||
|
// before": the one thing the host needs to give the file back.
|
||||||
|
Region *string `json:"region,omitempty"`
|
||||||
|
// Separated says the host put a blank line between the region and the machine's lines when it
|
||||||
|
// added the region — before it at the end, after it at the start, as At says — so taking the
|
||||||
|
// region out takes that line with it and nothing of the operator's.
|
||||||
|
Separated bool `json:"separated,omitempty"`
|
||||||
|
// At is where the host added the region: "start", or empty for the end.
|
||||||
|
At string `json:"at,omitempty"`
|
||||||
|
// Ended says the machine's last line had no line end and the host gave it one to add the
|
||||||
|
// region after it, so taking the region out takes that line end too.
|
||||||
|
Ended bool `json:"ended,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// State is the whole of what a node knows about what it has done.
|
// State is the whole of what a node knows about what it has done.
|
||||||
@@ -112,6 +147,80 @@ type State struct {
|
|||||||
// Firewall is the firewall found on this machine when it was first adopted, and whether the
|
// Firewall is the firewall found on this machine when it was first adopted, and whether the
|
||||||
// mesh has since retired it (novox/hq ADR 0100). Nil on a node that was never adopted.
|
// mesh has since retired it (novox/hq ADR 0100). Nil on a node that was never adopted.
|
||||||
Firewall *FoundFirewall `json:"firewall,omitempty"`
|
Firewall *FoundFirewall `json:"firewall,omitempty"`
|
||||||
|
|
||||||
|
// Mode is the node's mode as this host last recorded it: adopted or converged (novox/hq ADR
|
||||||
|
// 0100). Empty on a machine nothing has said a mode to yet. Recorded from every declaration
|
||||||
|
// the mesh sends and at genesis from what the operator said, so a declaration that says the
|
||||||
|
// other mode can be refused before it is applied (novox/hq issue 104).
|
||||||
|
Mode string `json:"mode,omitempty"`
|
||||||
|
|
||||||
|
// FoundFirst is, by resource id, what a service's unit was found as by an apply of it that did
|
||||||
|
// not finish — kept apart from Resources, because a record follows the fact and this apply's
|
||||||
|
// fact never came (novox/hq ADR 0118).
|
||||||
|
//
|
||||||
|
// **The capture is the one reading that cannot be taken again.** A first apply that enabled a
|
||||||
|
// unit and then failed to start it leaves no record; without this, the next apply would find
|
||||||
|
// the unit enabled, take that for what the machine had, and undeclaring would leave enabled a
|
||||||
|
// unit the mesh enabled. So what is found is written the moment it is read, whatever the apply
|
||||||
|
// of the resource then does, and a later apply reads it here before it reads the machine.
|
||||||
|
// Dropped once a record carrying it is written, and when its resource is no longer declared.
|
||||||
|
FoundFirst map[string]PendingFound `json:"found_first,omitempty"`
|
||||||
|
|
||||||
|
// Genesis is the bundle this host consumed raising the foundation, if it has. Once recorded,
|
||||||
|
// the bundle carried in the binary is not applied again: what genesis applied was rewritten
|
||||||
|
// for this machine, and the mesh has said more since (novox/hq issue 104).
|
||||||
|
Genesis *Genesis `json:"genesis,omitempty"`
|
||||||
|
|
||||||
|
// Retired is each found tunnel configuration the mesh removed from where its unit reads it,
|
||||||
|
// once the private network's take of that tunnel was proven (novox/hq ADR 0119).
|
||||||
|
//
|
||||||
|
// **Not a hold, and never released with one.** The hold on the found configuration ends at the
|
||||||
|
// retirement — what it held for has been replaced, and the node stops reporting it — so without
|
||||||
|
// this the next apply would find no hold and no file and read the take as one whose
|
||||||
|
// configuration vanished before it could be kept. It is kept whether or not the private
|
||||||
|
// network stays declared: undeclaring brings nothing back (ADR 0118), and a private network
|
||||||
|
// assigned again finds the tunnel's configuration retired rather than missing.
|
||||||
|
Retired []Retired `json:"retired,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retired is a found configuration the mesh removed once what replaced it was proven (novox/hq
|
||||||
|
// ADR 0119): where it was, under which hold it had been kept, and where its original still is.
|
||||||
|
type Retired struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Path string `json:"path"`
|
||||||
|
// Kept is the original as found (novox/hq ADR 0100) — the record of what the predecessor was,
|
||||||
|
// and a person's way back if one is ever wanted. The mesh never copies it back. It is the FIRST
|
||||||
|
// original, and stays so however often the file comes back: a retirement repeated never moves
|
||||||
|
// it. Digest is what it holds.
|
||||||
|
Kept string `json:"kept"`
|
||||||
|
Digest string `json:"digest,omitempty"`
|
||||||
|
// Extra is where what was at the path when it was last retired is kept, when that differed from
|
||||||
|
// the first original — rewritten since it was found, or put back with other content — and
|
||||||
|
// ExtraDigest what it holds. One copy per distinct content: a file that comes back as it was
|
||||||
|
// last retired keeps nothing more.
|
||||||
|
Extra string `json:"extra,omitempty"`
|
||||||
|
ExtraDigest string `json:"extra_digest,omitempty"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
// Again is how many times the configuration came back after it was retired, and was retired
|
||||||
|
// again (novox/hq ADR 0119).
|
||||||
|
Again int `json:"again,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Modes a node can be in (novox/hq ADR 0100).
|
||||||
|
const (
|
||||||
|
ModeAdopted = "adopted"
|
||||||
|
ModeConverged = "converged"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Genesis is the bundle a host consumed raising this machine's foundation.
|
||||||
|
type Genesis struct {
|
||||||
|
// Digest is sha256 of the exact bytes applied.
|
||||||
|
Digest string `json:"digest"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
// Rewritten is true when the bytes applied were the carried bundle rewritten for this
|
||||||
|
// machine — its foundation ports, root credentials, and adoption — so the bundle the binary
|
||||||
|
// carries is not what was applied.
|
||||||
|
Rewritten bool `json:"rewritten,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// FoundFirewall is what the host found filtering this machine, and what it did about it.
|
// FoundFirewall is what the host found filtering this machine, and what it did about it.
|
||||||
@@ -237,6 +346,27 @@ func (s *State) Release(id string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RetiredAt returns the retirement of the found configuration at a path, if the mesh retired one.
|
||||||
|
func (s State) RetiredAt(path string) (Retired, bool) {
|
||||||
|
for _, r := range s.Retired {
|
||||||
|
if r.Path == path {
|
||||||
|
return r, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Retired{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordRetired adds or replaces the retirement of the configuration at one path.
|
||||||
|
func (s *State) RecordRetired(r Retired) {
|
||||||
|
for i, existing := range s.Retired {
|
||||||
|
if existing.Path == r.Path {
|
||||||
|
s.Retired[i] = r
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.Retired = append(s.Retired, r)
|
||||||
|
}
|
||||||
|
|
||||||
// Find returns what was applied under an identity.
|
// Find returns what was applied under an identity.
|
||||||
func (s State) Find(id string) (Applied, bool) {
|
func (s State) Find(id string) (Applied, bool) {
|
||||||
for _, r := range s.Resources {
|
for _, r := range s.Resources {
|
||||||
@@ -391,3 +521,50 @@ func originOf(r Applied) string {
|
|||||||
}
|
}
|
||||||
return r.Origin
|
return r.Origin
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FoundUnit is a service's unit as the host first found it.
|
||||||
|
type FoundUnit struct {
|
||||||
|
// Unit is which unit this was read from. What was found about one unit says nothing about
|
||||||
|
// another, so a service whose declaration moves to a different unit is read again for that one
|
||||||
|
// (novox/hq ADR 0118). Empty on what was kept before this was: the record's Target then says.
|
||||||
|
Unit string `json:"unit,omitempty"`
|
||||||
|
// State is "running" or "stopped".
|
||||||
|
State string `json:"state"`
|
||||||
|
// Boot is "enabled" or "disabled" — or empty when the declaration never set it, and the host
|
||||||
|
// never touched it.
|
||||||
|
Boot string `json:"boot,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PendingFound is a unit as found by an apply of its service that has not yet been recorded, and
|
||||||
|
// who asked for that apply — so only a declaration from the same origin can say it is gone.
|
||||||
|
type PendingFound struct {
|
||||||
|
FoundUnit
|
||||||
|
Origin string `json:"origin,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// KeepFound writes down what an unfinished apply found a service's unit as.
|
||||||
|
func (s *State) KeepFound(id, origin string, f FoundUnit) {
|
||||||
|
if s.FoundFirst == nil {
|
||||||
|
s.FoundFirst = map[string]PendingFound{}
|
||||||
|
}
|
||||||
|
s.FoundFirst[id] = PendingFound{FoundUnit: f, Origin: origin}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DropFound forgets what was found for one resource: its record now carries it, or it is gone.
|
||||||
|
func (s *State) DropFound(id string) {
|
||||||
|
delete(s.FoundFirst, id)
|
||||||
|
if len(s.FoundFirst) == 0 {
|
||||||
|
s.FoundFirst = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DropFoundUndeclared forgets what was found for every resource of this origin the declaration no
|
||||||
|
// longer names. Only this origin's, for the reason Orphans gives: a mesh declaration's silence says
|
||||||
|
// nothing about what the bundle applies, nor the other way round.
|
||||||
|
func (s *State) DropFoundUndeclared(declared map[string]bool, origin string) {
|
||||||
|
for id, p := range s.FoundFirst {
|
||||||
|
if !declared[id] && originOf(Applied{Origin: p.Origin}) == origin {
|
||||||
|
s.DropFound(id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,328 @@
|
|||||||
|
// Package tunnel reads the tunnel a predecessor left on a machine, so the mesh's private network
|
||||||
|
// can take it over in place (novox/hq ADR 0105).
|
||||||
|
//
|
||||||
|
// On an adopted node that is the hub, the mesh's interface is raised with the found interface's
|
||||||
|
// private key, on its port, with its address and range, and every peer it had. The found interface
|
||||||
|
// is stopped, never flushed; its configuration stays on disk until the take is proven — a peer
|
||||||
|
// has handshaken with the mesh's interface — and is then retired (novox/hq ADR 0119). What this
|
||||||
|
// package does is the
|
||||||
|
// reading: which interface is there, what its file says, and what of that travels to the mesh —
|
||||||
|
// everything but the private key, which becomes the node's own overlay key and is stored the way
|
||||||
|
// that key is stored.
|
||||||
|
package tunnel
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ecdh"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Runner executes a command. The same shape as everywhere else in this host.
|
||||||
|
type Runner func(ctx context.Context, name string, args ...string) (string, error)
|
||||||
|
|
||||||
|
// MeshInterface is the private network's own interface, which is never the found one.
|
||||||
|
const MeshInterface = "mesh0"
|
||||||
|
|
||||||
|
// ConfigDir is where wg-quick keeps an interface's configuration.
|
||||||
|
const ConfigDir = "/etc/wireguard"
|
||||||
|
|
||||||
|
// Found is a tunnel as found on the machine: everything the mesh is told about it, and the
|
||||||
|
// private key, which it is not.
|
||||||
|
type Found struct {
|
||||||
|
// Interface, Unit and Config are what the mesh's interface takes over.
|
||||||
|
Interface string `json:"interface"`
|
||||||
|
Unit string `json:"unit"`
|
||||||
|
Config string `json:"config"`
|
||||||
|
// Port is the port the interface listens on; Address its own address with prefix length;
|
||||||
|
// Range the network that prefix names.
|
||||||
|
Port int `json:"port"`
|
||||||
|
Address string `json:"address"`
|
||||||
|
Range string `json:"range"`
|
||||||
|
// MTU is the interface's, when the found config set one. Kept because a tuned tunnel (a path
|
||||||
|
// that needs 1380, say) breaks silently if the mesh's interface comes up at the 1420 default:
|
||||||
|
// no ping fails, but TLS handshakes stall and transfers hang (novox/hq: a taken tunnel carries
|
||||||
|
// its MTU). Zero when the config named none, and the mesh sets no MTU line then.
|
||||||
|
MTU int `json:"mtu,omitempty"`
|
||||||
|
// PublicKey is what every peer knows this tunnel by — derived here from the private key, so
|
||||||
|
// it is the key the file actually holds and not a comment beside it.
|
||||||
|
PublicKey string `json:"public_key"`
|
||||||
|
Peers []Peer `json:"peers,omitempty"`
|
||||||
|
|
||||||
|
// privateKey never travels and never prints: not in JSON, not in %v. It is read once, to
|
||||||
|
// become the node's overlay key, and the file it came from is kept as found.
|
||||||
|
privateKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Peer is one peer of the found tunnel.
|
||||||
|
type Peer struct {
|
||||||
|
PublicKey string `json:"public_key"`
|
||||||
|
// Address is the one address the tunnel routes to the peer, as the file's AllowedIPs said it
|
||||||
|
// (with or without a /32).
|
||||||
|
Address string `json:"address"`
|
||||||
|
// Endpoint is where the found tunnel dialled the peer, if it did. Not carried to the mesh —
|
||||||
|
// a carried peer dials in, as it always did — but kept so a person reading the report sees
|
||||||
|
// what the file said.
|
||||||
|
Endpoint string `json:"endpoint,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PrivateKey is the found interface's private key, base64 as WireGuard writes it. The one
|
||||||
|
// accessor; a caller that has it is taking it as the node's key.
|
||||||
|
func (f Found) PrivateKey() string { return f.privateKey }
|
||||||
|
|
||||||
|
// String is what a found tunnel prints as: never the key.
|
||||||
|
func (f Found) String() string {
|
||||||
|
return fmt.Sprintf("%s on port %d, %s in %s, %d peer(s)", f.Interface, f.Port, f.Address,
|
||||||
|
f.Range, len(f.Peers))
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalJSON writes everything but the private key, whatever a caller passes to an encoder.
|
||||||
|
func (f Found) MarshalJSON() ([]byte, error) {
|
||||||
|
type wire Found
|
||||||
|
return json.Marshal(wire(f))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrNone is a machine with no tunnel to take over.
|
||||||
|
var ErrNone = errors.New("no tunnel is up on this machine besides the mesh's own")
|
||||||
|
|
||||||
|
// ErrSeveral is a machine with more than one, when nobody said which.
|
||||||
|
var ErrSeveral = errors.New("more than one tunnel is up on this machine")
|
||||||
|
|
||||||
|
// ReadFile is how a configuration is read; a variable so a test can hand in a file.
|
||||||
|
var ReadFile = os.ReadFile
|
||||||
|
|
||||||
|
// Find reads the tunnel to take over: the one named, or the one interface up besides the mesh's
|
||||||
|
// own. Nothing up is ErrNone — an ordinary answer, the machine has no tunnel to adopt — and two
|
||||||
|
// or more with none named is ErrSeveral, naming them, because choosing would be deciding.
|
||||||
|
//
|
||||||
|
// Read from the interface's configuration file rather than from the running interface: the file
|
||||||
|
// is what wg-quick raised and what carries the address, which the kernel does not report per
|
||||||
|
// interface the way the key and peers are. The running interface is consulted only to know the
|
||||||
|
// tunnel is up — a file for an interface nothing runs is not a tunnel the peers are reaching.
|
||||||
|
func Find(ctx context.Context, run Runner, named string) (Found, error) {
|
||||||
|
out, err := run(ctx, "wg", "show", "interfaces")
|
||||||
|
if err != nil {
|
||||||
|
return Found{}, fmt.Errorf("cannot ask which tunnels are up on this machine: %w", err)
|
||||||
|
}
|
||||||
|
var up []string
|
||||||
|
for _, iface := range strings.Fields(out) {
|
||||||
|
if iface != MeshInterface {
|
||||||
|
up = append(up, iface)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(up)
|
||||||
|
iface := named
|
||||||
|
switch {
|
||||||
|
case named != "":
|
||||||
|
found := false
|
||||||
|
for _, u := range up {
|
||||||
|
if u == named {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return Found{}, fmt.Errorf("%s was named as the tunnel to take over and is not up; up: %s",
|
||||||
|
named, orNone(up))
|
||||||
|
}
|
||||||
|
case len(up) == 0:
|
||||||
|
return Found{}, ErrNone
|
||||||
|
case len(up) > 1:
|
||||||
|
return Found{}, fmt.Errorf("%w: %s. Name the one the predecessor's machines reach with --tunnel",
|
||||||
|
ErrSeveral, strings.Join(up, ", "))
|
||||||
|
default:
|
||||||
|
iface = up[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
path := ConfigDir + "/" + iface + ".conf"
|
||||||
|
raw, err := ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return Found{}, fmt.Errorf("%s is up and its configuration cannot be read: %w", iface, err)
|
||||||
|
}
|
||||||
|
found, err := Parse(raw)
|
||||||
|
if err != nil {
|
||||||
|
return Found{}, fmt.Errorf("%s: %w", path, err)
|
||||||
|
}
|
||||||
|
found.Interface, found.Unit, found.Config = iface, "wg-quick@"+iface, path
|
||||||
|
return found, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handshaken is how many peers of an interface have completed a handshake with it: the proof that
|
||||||
|
// the interface carries the tunnel, rather than merely being up (novox/hq ADR 0119).
|
||||||
|
//
|
||||||
|
// Asked of the running interface, since a handshake is a fact about the kernel's tunnel that no
|
||||||
|
// file records. A question that cannot be asked — no `wg` on the machine, no such interface, a
|
||||||
|
// permission refused — is an error and never a zero: "no peer has handshaken" retires nothing
|
||||||
|
// either, but it is a different thing to tell a person.
|
||||||
|
func Handshaken(ctx context.Context, run Runner, iface string) (int, error) {
|
||||||
|
out, err := run(ctx, "wg", "show", iface, "latest-handshakes")
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("cannot ask %s which peers have handshaken: %w", iface, err)
|
||||||
|
}
|
||||||
|
return ParseHandshakes(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseHandshakes reads `wg show <interface> latest-handshakes`: one line per peer, its public key
|
||||||
|
// and the Unix time of its latest handshake, tab-separated — zero for a peer that never has. What
|
||||||
|
// is counted is the peers with a time. A line that is not a key and a time is refused rather than
|
||||||
|
// skipped: output this does not understand is not evidence of anything.
|
||||||
|
func ParseHandshakes(out string) (int, error) {
|
||||||
|
n := 0
|
||||||
|
for i, line := range strings.Split(out, "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fields := strings.Fields(line)
|
||||||
|
if len(fields) != 2 {
|
||||||
|
return 0, fmt.Errorf("line %d of the handshakes is not a peer and a time: %q", i+1, line)
|
||||||
|
}
|
||||||
|
at, err := strconv.ParseInt(fields[1], 10, 64)
|
||||||
|
if err != nil || at < 0 {
|
||||||
|
return 0, fmt.Errorf("line %d of the handshakes does not end in a time: %q", i+1, line)
|
||||||
|
}
|
||||||
|
if at > 0 {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNone(names []string) string {
|
||||||
|
if len(names) == 0 {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
return strings.Join(names, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse reads a wg-quick configuration: the interface's key, port and address, and each peer's
|
||||||
|
// key and allowed address. Refused when it lacks what the mesh needs — a key, an address with a
|
||||||
|
// prefix — because a tunnel taken over without them is one the peers cannot reach.
|
||||||
|
func Parse(raw []byte) (Found, error) {
|
||||||
|
var f Found
|
||||||
|
section := ""
|
||||||
|
var peer *Peer
|
||||||
|
closePeer := func() error {
|
||||||
|
if peer == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if peer.PublicKey == "" {
|
||||||
|
return errors.New("a [Peer] section has no PublicKey")
|
||||||
|
}
|
||||||
|
if peer.Address == "" {
|
||||||
|
return fmt.Errorf("the peer %s has no AllowedIPs, so the tunnel routes nothing to it",
|
||||||
|
short(peer.PublicKey))
|
||||||
|
}
|
||||||
|
f.Peers = append(f.Peers, *peer)
|
||||||
|
peer = nil
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for n, line := range strings.Split(string(raw), "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if i := strings.IndexAny(line, "#;"); i >= 0 {
|
||||||
|
line = strings.TrimSpace(line[:i])
|
||||||
|
}
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(line, "[") {
|
||||||
|
if err := closePeer(); err != nil {
|
||||||
|
return Found{}, err
|
||||||
|
}
|
||||||
|
section = strings.ToLower(strings.Trim(line, "[]"))
|
||||||
|
if section == "peer" {
|
||||||
|
peer = &Peer{}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key, value, ok := strings.Cut(line, "=")
|
||||||
|
if !ok {
|
||||||
|
return Found{}, fmt.Errorf("line %d is not `key = value`", n+1)
|
||||||
|
}
|
||||||
|
key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value)
|
||||||
|
switch section {
|
||||||
|
case "interface":
|
||||||
|
switch key {
|
||||||
|
case "privatekey":
|
||||||
|
f.privateKey = value
|
||||||
|
case "listenport":
|
||||||
|
port, err := strconv.Atoi(value)
|
||||||
|
if err != nil || port < 1 || port > 65535 {
|
||||||
|
return Found{}, fmt.Errorf("ListenPort %q is not a port", value)
|
||||||
|
}
|
||||||
|
f.Port = port
|
||||||
|
case "mtu":
|
||||||
|
mtu, err := strconv.Atoi(value)
|
||||||
|
if err != nil || mtu < 576 || mtu > 65535 {
|
||||||
|
return Found{}, fmt.Errorf("MTU %q is not a plausible MTU", value)
|
||||||
|
}
|
||||||
|
f.MTU = mtu
|
||||||
|
case "address":
|
||||||
|
// The first address is the interface's; a second family would be a second
|
||||||
|
// tunnel's worth of addressing, which this does not carry.
|
||||||
|
first := strings.TrimSpace(strings.Split(value, ",")[0])
|
||||||
|
ip, network, err := net.ParseCIDR(first)
|
||||||
|
if err != nil {
|
||||||
|
return Found{}, fmt.Errorf("Address %q is not an address with a prefix length, "+
|
||||||
|
"and the range the mesh takes over is read from the prefix", first)
|
||||||
|
}
|
||||||
|
f.Address = first
|
||||||
|
f.Range = network.String()
|
||||||
|
_ = ip
|
||||||
|
}
|
||||||
|
case "peer":
|
||||||
|
switch key {
|
||||||
|
case "publickey":
|
||||||
|
peer.PublicKey = value
|
||||||
|
case "allowedips":
|
||||||
|
peer.Address = strings.TrimSpace(strings.Split(value, ",")[0])
|
||||||
|
case "endpoint":
|
||||||
|
peer.Endpoint = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := closePeer(); err != nil {
|
||||||
|
return Found{}, err
|
||||||
|
}
|
||||||
|
if f.privateKey == "" {
|
||||||
|
return Found{}, errors.New("no PrivateKey in [Interface]; the mesh takes a tunnel over with its key or not at all")
|
||||||
|
}
|
||||||
|
if f.Address == "" {
|
||||||
|
return Found{}, errors.New("no Address in [Interface], so neither the hub's address nor the range can be read")
|
||||||
|
}
|
||||||
|
if f.Port == 0 {
|
||||||
|
return Found{}, errors.New("no ListenPort in [Interface]: a tunnel with no port is one nothing dials, so there is nothing to take over")
|
||||||
|
}
|
||||||
|
public, err := PublicKeyOf(f.privateKey)
|
||||||
|
if err != nil {
|
||||||
|
return Found{}, err
|
||||||
|
}
|
||||||
|
f.PublicKey = public
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PublicKeyOf derives the public half of a WireGuard private key, both base64.
|
||||||
|
func PublicKeyOf(privateBase64 string) (string, error) {
|
||||||
|
raw, err := base64.StdEncoding.DecodeString(privateBase64)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the private key is not base64: %w", err)
|
||||||
|
}
|
||||||
|
private, err := ecdh.X25519().NewPrivateKey(raw)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the private key is not a Curve25519 key: %w", err)
|
||||||
|
}
|
||||||
|
return base64.StdEncoding.EncodeToString(private.PublicKey().Bytes()), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func short(key string) string {
|
||||||
|
if len(key) > 8 {
|
||||||
|
return key[:8] + "…"
|
||||||
|
}
|
||||||
|
return key
|
||||||
|
}
|
||||||
@@ -0,0 +1,242 @@
|
|||||||
|
package tunnel
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ecdh"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0105: the host reads the predecessor's tunnel — key, port, address and range, every
|
||||||
|
// peer — and the private key becomes the node's, never printed and never sent.
|
||||||
|
|
||||||
|
// aKey is a real WireGuard keypair, made here so a key that stopped being a key is caught.
|
||||||
|
func aKey(t *testing.T) (private, public string) {
|
||||||
|
t.Helper()
|
||||||
|
k, err := ecdh.X25519().GenerateKey(rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return base64.StdEncoding.EncodeToString(k.Bytes()),
|
||||||
|
base64.StdEncoding.EncodeToString(k.PublicKey().Bytes())
|
||||||
|
}
|
||||||
|
|
||||||
|
func aConfig(private string, peers ...string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
fmt.Fprintf(&b, "# the predecessor's hub\n[Interface]\nPrivateKey = %s\nListenPort = 51900\n"+
|
||||||
|
"Address = 192.0.2.1/24\n", private)
|
||||||
|
for i, key := range peers {
|
||||||
|
fmt.Fprintf(&b, "\n[Peer]\nPublicKey = %s\nAllowedIPs = 192.0.2.%d/32\n", key, i+2)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheConfigurationIsReadWhole(t *testing.T) {
|
||||||
|
private, public := aKey(t)
|
||||||
|
_, peerA := aKey(t)
|
||||||
|
_, peerB := aKey(t)
|
||||||
|
found, err := Parse([]byte(aConfig(private, peerA, peerB) + "PersistentKeepalive = 25 ; a comment\n"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if found.Port != 51900 || found.Address != "192.0.2.1/24" || found.Range != "192.0.2.0/24" {
|
||||||
|
t.Errorf("port, address or range misread: %+v", found)
|
||||||
|
}
|
||||||
|
if found.PublicKey != public {
|
||||||
|
t.Errorf("the public key is not the one derived from the file's private key")
|
||||||
|
}
|
||||||
|
if found.PrivateKey() != private {
|
||||||
|
t.Error("the private key was not read")
|
||||||
|
}
|
||||||
|
if len(found.Peers) != 2 || found.Peers[0].PublicKey != peerA || found.Peers[0].Address != "192.0.2.2/32" ||
|
||||||
|
found.Peers[1].PublicKey != peerB || found.Peers[1].Address != "192.0.2.3/32" {
|
||||||
|
t.Errorf("the peers were misread: %+v", found.Peers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestThePrivateKeyNeverPrintsAndNeverTravels(t *testing.T) {
|
||||||
|
private, _ := aKey(t)
|
||||||
|
found, err := Parse([]byte(aConfig(private)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(found)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for what, said := range map[string]string{
|
||||||
|
"JSON": string(raw),
|
||||||
|
"String": found.String(),
|
||||||
|
"%v": fmt.Sprintf("%v", found),
|
||||||
|
"%+v": fmt.Sprintf("%+v", found),
|
||||||
|
"%#v via %v": fmt.Sprintf("%v", []Found{found}),
|
||||||
|
} {
|
||||||
|
if strings.Contains(said, private) {
|
||||||
|
t.Errorf("the private key appears in %s: %s", what, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(raw), found.PublicKey) {
|
||||||
|
t.Error("the public key does not travel, so the mesh could not know the tunnel's key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATunnelWithoutWhatTheMeshNeedsIsRefused(t *testing.T) {
|
||||||
|
private, _ := aKey(t)
|
||||||
|
_, peer := aKey(t)
|
||||||
|
for name, conf := range map[string]string{
|
||||||
|
"no key": "[Interface]\nListenPort = 51900\nAddress = 192.0.2.1/24\n",
|
||||||
|
"no address": fmt.Sprintf("[Interface]\nPrivateKey = %s\nListenPort = 51900\n", private),
|
||||||
|
"bare address": fmt.Sprintf("[Interface]\nPrivateKey = %s\nListenPort = 51900\nAddress = 192.0.2.1\n", private),
|
||||||
|
"no port": fmt.Sprintf("[Interface]\nPrivateKey = %s\nAddress = 192.0.2.1/24\n", private),
|
||||||
|
"peer no route": aConfig(private) + "\n[Peer]\nPublicKey = " + peer + "\n",
|
||||||
|
"peer no key": aConfig(private) + "\n[Peer]\nAllowedIPs = 192.0.2.9/32\n",
|
||||||
|
"not a key": "[Interface]\nPrivateKey = not-base64!\nListenPort = 1\nAddress = 192.0.2.1/24\n",
|
||||||
|
} {
|
||||||
|
if _, err := Parse([]byte(conf)); err == nil {
|
||||||
|
t.Errorf("%s was accepted", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// aMachine answers `wg show interfaces` and reads configurations from a map.
|
||||||
|
type aMachine struct {
|
||||||
|
up string
|
||||||
|
files map[string]string
|
||||||
|
asked []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *aMachine) run(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
m.asked = append(m.asked, name+" "+strings.Join(args, " "))
|
||||||
|
if name == "wg" && len(args) == 2 && args[0] == "show" && args[1] == "interfaces" {
|
||||||
|
return m.up, nil
|
||||||
|
}
|
||||||
|
return "", errors.New("unexpected: " + name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *aMachine) read(path string) ([]byte, error) {
|
||||||
|
if raw, ok := m.files[path]; ok {
|
||||||
|
return []byte(raw), nil
|
||||||
|
}
|
||||||
|
return nil, errors.New("no such file: " + path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheOneTunnelUpBesidesTheMeshsIsFound(t *testing.T) {
|
||||||
|
private, public := aKey(t)
|
||||||
|
m := &aMachine{up: "mesh0 wg0\n", files: map[string]string{ConfigDir + "/wg0.conf": aConfig(private)}}
|
||||||
|
ReadFile = m.read
|
||||||
|
t.Cleanup(func() { ReadFile = os.ReadFile })
|
||||||
|
|
||||||
|
found, err := Find(context.Background(), m.run, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if found.Interface != "wg0" || found.Unit != "wg-quick@wg0" || found.Config != ConfigDir+"/wg0.conf" ||
|
||||||
|
found.PublicKey != public {
|
||||||
|
t.Errorf("the wrong tunnel, or misnamed: %+v", found)
|
||||||
|
}
|
||||||
|
for _, asked := range m.asked {
|
||||||
|
if strings.HasPrefix(asked, "wg set") || strings.Contains(asked, "private-key") {
|
||||||
|
t.Errorf("finding a tunnel ran %q; reading is reading", asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoneUpIsAnOrdinaryAnswerAndSeveralIsAQuestion(t *testing.T) {
|
||||||
|
private, _ := aKey(t)
|
||||||
|
m := &aMachine{up: "mesh0\n", files: map[string]string{
|
||||||
|
ConfigDir + "/wg0.conf": aConfig(private), ConfigDir + "/wg1.conf": aConfig(private)}}
|
||||||
|
ReadFile = m.read
|
||||||
|
t.Cleanup(func() { ReadFile = os.ReadFile })
|
||||||
|
|
||||||
|
if _, err := Find(context.Background(), m.run, ""); !errors.Is(err, ErrNone) {
|
||||||
|
t.Errorf("a machine with only the mesh's interface up was not an ordinary none: %v", err)
|
||||||
|
}
|
||||||
|
m.up = "wg1 mesh0 wg0\n"
|
||||||
|
_, err := Find(context.Background(), m.run, "")
|
||||||
|
if !errors.Is(err, ErrSeveral) || !strings.Contains(err.Error(), "wg0, wg1") || strings.Contains(err.Error(), "mesh0") {
|
||||||
|
t.Errorf("two tunnels up were not refused naming both and only them: %v", err)
|
||||||
|
}
|
||||||
|
found, err := Find(context.Background(), m.run, "wg1")
|
||||||
|
if err != nil || found.Interface != "wg1" {
|
||||||
|
t.Errorf("naming one of two did not find it: %+v %v", found, err)
|
||||||
|
}
|
||||||
|
if _, err := Find(context.Background(), m.run, "wg9"); err == nil || !strings.Contains(err.Error(), "wg9") {
|
||||||
|
t.Errorf("naming a tunnel that is not up was not refused: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFoundTunnelReadsItsMTU(t *testing.T) {
|
||||||
|
// A tuned path sets MTU in [Interface]; the mesh must carry it or the tunnel regresses to the
|
||||||
|
// default silently (novox/hq: a taken tunnel carries its MTU).
|
||||||
|
private, public := aKey(t)
|
||||||
|
withMTU := "# tuned\n[Interface]\nPrivateKey = " + private +
|
||||||
|
"\nListenPort = 51820\nAddress = 10.10.0.3/24\nMTU = 1380\n" +
|
||||||
|
"[Peer]\nPublicKey = " + public + "\nAllowedIPs = 10.10.0.1/32\n"
|
||||||
|
f, err := Parse([]byte(withMTU))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f.MTU != 1380 {
|
||||||
|
t.Fatalf("MTU 1380 was not read; got %d", f.MTU)
|
||||||
|
}
|
||||||
|
// And a config with none leaves MTU zero, so the mesh writes no MTU line.
|
||||||
|
f2, err := Parse([]byte(aConfig(private, public)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f2.MTU != 0 {
|
||||||
|
t.Fatalf("a config with no MTU must leave it zero; got %d", f2.MTU)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0119: a take is proven by a handshake on the mesh's interface, read from `wg show
|
||||||
|
// <interface> latest-handshakes` — as wg prints it, a key and a Unix time per peer, zero for never.
|
||||||
|
func TestAHandshakeIsAPeerWithATime(t *testing.T) {
|
||||||
|
cases := map[string]struct {
|
||||||
|
out string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
"two peers, one handshaken": {"PEER-A=\t1790000000\nPEER-B=\t0\n", 1},
|
||||||
|
"every peer handshaken": {"PEER-A=\t1790000000\nPEER-B=\t1790000042\n", 2},
|
||||||
|
"no peer ever": {"PEER-A=\t0\nPEER-B=\t0\n", 0},
|
||||||
|
"an interface with no peer": {"", 0},
|
||||||
|
"spaces, a trailing line": {"PEER-A= 1790000000\n\n", 1},
|
||||||
|
}
|
||||||
|
for name, c := range cases {
|
||||||
|
got, err := ParseHandshakes(c.out)
|
||||||
|
if err != nil || got != c.want {
|
||||||
|
t.Errorf("%s: %d peer(s) handshaken (%v), want %d", name, got, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Output that is not a key and a time is not evidence of anything, and not a zero either.
|
||||||
|
for _, nonsense := range []string{"PEER-A=\n", "PEER-A=\tyesterday\n", "PEER-A=\t-1\n", "a b c\n"} {
|
||||||
|
if _, err := ParseHandshakes(nonsense); err == nil {
|
||||||
|
t.Errorf("%q was read as handshakes", nonsense)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandshakesThatCannotBeAskedAreAnErrorNotAZero(t *testing.T) {
|
||||||
|
var asked string
|
||||||
|
ok := func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
asked = name + " " + strings.Join(args, " ")
|
||||||
|
return "PEER-A=\t1790000000\n", nil
|
||||||
|
}
|
||||||
|
if n, err := Handshaken(context.Background(), ok, "mesh0"); err != nil || n != 1 ||
|
||||||
|
asked != "wg show mesh0 latest-handshakes" {
|
||||||
|
t.Fatalf("asked %q and read %d (%v)", asked, n, err)
|
||||||
|
}
|
||||||
|
missing := func(context.Context, string, ...string) (string, error) {
|
||||||
|
return "", errors.New(`exec: "wg": executable file not found in $PATH`)
|
||||||
|
}
|
||||||
|
if _, err := Handshaken(context.Background(), missing, "mesh0"); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "mesh0") {
|
||||||
|
t.Fatalf("a machine with no wg was read as one with no handshake: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user