Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7181675c4a | ||
|
|
19e14901c0 | ||
|
|
b9fc3afc14 | ||
|
|
ca7c4a5915 | ||
|
|
b30d9c5b5a | ||
|
|
5b73e04192 |
@@ -0,0 +1,153 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Joining through the tunnel (novox/hq ADR 0169).
|
||||||
|
//
|
||||||
|
// **The bus is never open to the internet, so a joining machine reaches it over the tunnel.** It
|
||||||
|
// makes its tunnel key first and prints the public half; the token is issued for that key, and the
|
||||||
|
// hub is told the key before the token is shown; the token carries the one peer this machine needs.
|
||||||
|
// So the tunnel can come up before the mesh has said anything else — the circle ADR 0004 broke by
|
||||||
|
// carrying the bus's address in the token is broken here by carrying the hub's.
|
||||||
|
|
||||||
|
// tunnelConfigPath and tunnelUnit are where the mesh's own declaration puts the private network, so
|
||||||
|
// the first tunnel is the same interface and unit the mesh takes over, not a second one beside it.
|
||||||
|
var (
|
||||||
|
tunnelConfigPath = "/etc/wireguard/mesh0.conf"
|
||||||
|
tunnelUnit = "wg-quick@mesh0"
|
||||||
|
// lookPath finds WireGuard's tools; a variable so a test needs none installed.
|
||||||
|
lookPath = exec.LookPath
|
||||||
|
)
|
||||||
|
|
||||||
|
// keyCommand makes this machine's tunnel key, or reads the one it already made, and prints the
|
||||||
|
// public half: what the token is issued for. Making it twice would be a token issued for a key the
|
||||||
|
// machine no longer has, so an existing key is kept.
|
||||||
|
func keyCommand(opts options) error {
|
||||||
|
path := identity.OverlayKeyPath(opts.state)
|
||||||
|
if key, err := identity.LoadOverlayKey(path); err == nil {
|
||||||
|
fmt.Println(key.Public)
|
||||||
|
return nil
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(identity.Path(opts.state)); err == nil {
|
||||||
|
return fmt.Errorf("this machine has joined already (%s), and its tunnel key is its own; "+
|
||||||
|
"there is no key to make", identity.Path(opts.state))
|
||||||
|
}
|
||||||
|
key, err := identity.GenerateOverlayKey()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(key.Private+"\n"), 0o600); err != nil {
|
||||||
|
return fmt.Errorf("cannot write this machine's tunnel key: %w", err)
|
||||||
|
}
|
||||||
|
fmt.Println(key.Public)
|
||||||
|
fmt.Fprintln(os.Stderr, "\nthis machine's tunnel key, made here; the private half stays in "+path+".\n"+
|
||||||
|
"Issue the token for it — `token issue --new <name> --overlay-key <the line above>` — and enrol with that token.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// tunnelKeyFor is the key a token through the tunnel was issued for, read from where `key` left it.
|
||||||
|
// Refused when there is none, or it is another: the hub knows only the key the token names.
|
||||||
|
func tunnelKeyFor(t *identity.TokenTunnel, state string) (identity.OverlayKey, error) {
|
||||||
|
path := identity.OverlayKeyPath(state)
|
||||||
|
key, err := identity.LoadOverlayKey(path)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return identity.OverlayKey{}, fmt.Errorf("this token was issued for a tunnel key, and this " +
|
||||||
|
"machine has none: run `nox-mesh-host key` here first and issue the token for the key it prints")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return identity.OverlayKey{}, err
|
||||||
|
}
|
||||||
|
if key.Public != t.Key {
|
||||||
|
return identity.OverlayKey{}, fmt.Errorf("this token was issued for the tunnel key %s, and this "+
|
||||||
|
"machine's is %s — it is another machine's token, or the key was made again; issue a new "+
|
||||||
|
"token for %s", t.Key, key.Public, key.Public)
|
||||||
|
}
|
||||||
|
return key, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// tunnelConfig is the first tunnel: this machine's address, and the hub as its one peer, reaching the
|
||||||
|
// whole private network through it. The private key is set from its file, as the mesh's own
|
||||||
|
// declaration does it, so the file holds no secret.
|
||||||
|
func tunnelConfig(t *identity.TokenTunnel, keyPath string) string {
|
||||||
|
return fmt.Sprintf(`# Written by nox-mesh-host enrol: the one peer a joining machine needs (novox/hq ADR 0169).
|
||||||
|
# The mesh's own declaration replaces this once the machine has joined.
|
||||||
|
[Interface]
|
||||||
|
Address = %s
|
||||||
|
PostUp = wg set %%i private-key %s
|
||||||
|
|
||||||
|
[Peer]
|
||||||
|
PublicKey = %s
|
||||||
|
Endpoint = %s
|
||||||
|
AllowedIPs = %s
|
||||||
|
PersistentKeepalive = 25
|
||||||
|
`, t.Address, keyPath, t.HubKey, t.HubEndpoint, t.Range)
|
||||||
|
}
|
||||||
|
|
||||||
|
// bringTheTunnelUp writes the first tunnel and starts it, so the bus the token names can be reached.
|
||||||
|
func bringTheTunnelUp(ctx context.Context, t *identity.TokenTunnel, keyPath string, run apply.Runner) error {
|
||||||
|
if _, err := lookPath("wg-quick"); err != nil {
|
||||||
|
return errors.New("joining through the tunnel needs WireGuard's tools on this machine " +
|
||||||
|
"(wireguard-tools), and wg-quick is not here")
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(tunnelConfigPath), 0o700); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(tunnelConfigPath, []byte(tunnelConfig(t, keyPath)), 0o600); err != nil {
|
||||||
|
return fmt.Errorf("cannot write the first tunnel: %w", err)
|
||||||
|
}
|
||||||
|
if out, err := run(ctx, "systemctl", "restart", tunnelUnit); err != nil {
|
||||||
|
return fmt.Errorf("the first tunnel would not start (%s): %v %s", tunnelUnit, err, strings.TrimSpace(out))
|
||||||
|
}
|
||||||
|
fmt.Printf("the tunnel to the hub is up: %s, through %s\n", t.Address, t.HubEndpoint)
|
||||||
|
return waitForTheHub(ctx, run, handshakeWithin)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handshakeWithin is how long the hub has to answer the first tunnel. Issuing the token sent the hub
|
||||||
|
// this machine as a peer; the hub applies that on its own time, and a bus dialled before it has is a
|
||||||
|
// timeout that names the bus rather than the tunnel.
|
||||||
|
var handshakeWithin = 90 * time.Second
|
||||||
|
|
||||||
|
// waitForTheHub waits until the tunnel has shaken hands with the hub, so the bus is dialled over a
|
||||||
|
// tunnel that answers — and says so in the tunnel's own words when it does not.
|
||||||
|
func waitForTheHub(ctx context.Context, run apply.Runner, within time.Duration) error {
|
||||||
|
deadline := time.Now().Add(within)
|
||||||
|
for {
|
||||||
|
out, err := run(ctx, "wg", "show", "mesh0", "latest-handshakes")
|
||||||
|
if err == nil {
|
||||||
|
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||||||
|
fields := strings.Fields(line)
|
||||||
|
if len(fields) == 2 && fields[1] != "0" {
|
||||||
|
fmt.Println("the hub answered the tunnel")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
return fmt.Errorf("the hub has not answered the tunnel in %s: the token may be another machine's, "+
|
||||||
|
"the hub may not have been sent this machine as a peer, or its tunnel's port is not reachable "+
|
||||||
|
"from here", within)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
)
|
||||||
|
|
||||||
|
// `key` makes the tunnel key once and prints its public half; asked again it prints the same one,
|
||||||
|
// because a token may already have been issued for it (novox/hq ADR 0169).
|
||||||
|
func TestKeyMakesTheTunnelKeyOnceAndKeepsIt(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
opts := options{state: filepath.Join(dir, "state.json")}
|
||||||
|
first := captureStdout(t, func() {
|
||||||
|
if err := keyCommand(opts); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
second := captureStdout(t, func() {
|
||||||
|
if err := keyCommand(opts); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if strings.TrimSpace(first) == "" || strings.TrimSpace(first) != strings.TrimSpace(second) {
|
||||||
|
t.Fatalf("the key changed between two asks: %q then %q", first, second)
|
||||||
|
}
|
||||||
|
info, err := os.Stat(identity.OverlayKeyPath(opts.state))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info.Mode().Perm() != 0o600 {
|
||||||
|
t.Errorf("the private half is readable beyond root: %v", info.Mode().Perm())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A token through the tunnel takes the key it was issued for, and says so when this machine has none
|
||||||
|
// or another.
|
||||||
|
func TestATokenThroughTheTunnelTakesItsOwnKey(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
state := filepath.Join(dir, "state.json")
|
||||||
|
tt := &identity.TokenTunnel{Key: "x", Address: "10.42.0.9/32", Range: "10.42.0.0/16", HubKey: "h", HubEndpoint: "198.51.100.1:51820"}
|
||||||
|
if _, err := tunnelKeyFor(tt, state); err == nil || !strings.Contains(err.Error(), "nox-mesh-host key") {
|
||||||
|
t.Fatalf("a machine with no key was not told to make one: %v", err)
|
||||||
|
}
|
||||||
|
captureStdout(t, func() { _ = keyCommand(options{state: state}) })
|
||||||
|
if _, err := tunnelKeyFor(tt, state); err == nil || !strings.Contains(err.Error(), "issued for the tunnel key x") {
|
||||||
|
t.Fatalf("another machine's token was taken: %v", err)
|
||||||
|
}
|
||||||
|
mine, _ := identity.LoadOverlayKey(identity.OverlayKeyPath(state))
|
||||||
|
tt.Key = mine.Public
|
||||||
|
if got, err := tunnelKeyFor(tt, state); err != nil || got.Public != mine.Public {
|
||||||
|
t.Fatalf("this machine's own token was refused: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first tunnel is the mesh's interface and unit, with the hub as its one peer and no secret in
|
||||||
|
// the file — the same shape the mesh's declaration replaces it with.
|
||||||
|
func TestTheFirstTunnelIsTheMeshsInterfaceWithTheHubAsItsPeer(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
tunnelConfigPath = filepath.Join(dir, "wireguard", "mesh0.conf")
|
||||||
|
lookPath = func(string) (string, error) { return "/usr/bin/wg-quick", nil }
|
||||||
|
t.Cleanup(func() { tunnelConfigPath = "/etc/wireguard/mesh0.conf" })
|
||||||
|
var ran []string
|
||||||
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
if name == "wg" {
|
||||||
|
return "HUBKEY\t1759400000\n", nil
|
||||||
|
}
|
||||||
|
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
tt := &identity.TokenTunnel{Key: "k", Address: "10.42.0.9/32", Range: "10.42.0.0/16", HubKey: "HUBKEY", HubEndpoint: "198.51.100.1:51820"}
|
||||||
|
captureStdout(t, func() {
|
||||||
|
if err := bringTheTunnelUp(context.Background(), tt, "/var/lib/mesh-host/overlay.key", run); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
raw, err := os.ReadFile(tunnelConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
conf := string(raw)
|
||||||
|
for _, want := range []string{"Address = 10.42.0.9/32", "PostUp = wg set %i private-key /var/lib/mesh-host/overlay.key",
|
||||||
|
"PublicKey = HUBKEY", "Endpoint = 198.51.100.1:51820", "AllowedIPs = 10.42.0.0/16", "PersistentKeepalive = 25"} {
|
||||||
|
if !strings.Contains(conf, want) {
|
||||||
|
t.Errorf("the first tunnel lacks %q:\n%s", want, conf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(conf, "PrivateKey") {
|
||||||
|
t.Error("the first tunnel's file holds the private key")
|
||||||
|
}
|
||||||
|
if len(ran) != 1 || ran[0] != "systemctl restart wg-quick@mesh0" {
|
||||||
|
t.Errorf("the tunnel was started as %v", ran)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// captureStdout is what fn printed to standard output.
|
||||||
|
func captureStdout(t *testing.T, fn func()) string {
|
||||||
|
t.Helper()
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
was := os.Stdout
|
||||||
|
os.Stdout = w
|
||||||
|
fn()
|
||||||
|
os.Stdout = was
|
||||||
|
w.Close()
|
||||||
|
out, _ := io.ReadAll(r)
|
||||||
|
return string(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bus is dialled only once the hub has answered the tunnel, and a hub that never does is said
|
||||||
|
// as the tunnel's fault rather than the bus's.
|
||||||
|
func TestTheBusWaitsForTheHubToAnswer(t *testing.T) {
|
||||||
|
asked := 0
|
||||||
|
answersOnThird := func(_ context.Context, name string, args ...string) (string, error) {
|
||||||
|
asked++
|
||||||
|
if asked < 3 {
|
||||||
|
return "HUBKEY\t0\n", nil
|
||||||
|
}
|
||||||
|
return "HUBKEY\t1759400000\n", nil
|
||||||
|
}
|
||||||
|
captureStdout(t, func() {
|
||||||
|
if err := waitForTheHub(context.Background(), answersOnThird, time.Minute); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
never := func(context.Context, string, ...string) (string, error) { return "HUBKEY\t0\n", nil }
|
||||||
|
err := waitForTheHub(context.Background(), never, 0)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "has not answered the tunnel") {
|
||||||
|
t.Fatalf("a hub that never answered was not said: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+18
-1
@@ -96,6 +96,9 @@ const usage = `mesh-host — the node host
|
|||||||
reconcile make this machine match what the mesh last told it — or, before any
|
reconcile make this machine match what the mesh last told it — or, before any
|
||||||
mesh has, the bundle this host carries
|
mesh has, the bundle this host carries
|
||||||
bundle show what this host carries
|
bundle show what this host carries
|
||||||
|
key make this machine's tunnel key, or read the one it made, and print the public
|
||||||
|
half: what its join token is issued for (novox/hq ADR 0169)
|
||||||
|
enrol --token T join the mesh — through the tunnel when the token was issued for a key
|
||||||
overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this
|
overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this
|
||||||
node's overlay key and the mesh is told, signed; --tunnel <iface> when several are up
|
node's overlay key and the mesh is told, signed; --tunnel <iface> when several are up
|
||||||
owned what this host has applied and still owns
|
owned what this host has applied and still owns
|
||||||
@@ -212,6 +215,9 @@ func parseArgs(args []string) (string, options, error) {
|
|||||||
func run(ctx context.Context, command string, opts options) error {
|
func run(ctx context.Context, command string, opts options) error {
|
||||||
jsonOut, timeout := opts.json, opts.timeout
|
jsonOut, timeout := opts.json, opts.timeout
|
||||||
switch command {
|
switch command {
|
||||||
|
case "key":
|
||||||
|
return keyCommand(opts)
|
||||||
|
|
||||||
case "profile":
|
case "profile":
|
||||||
p := profile.Detect(ctx, profile.Default(nil), timeout)
|
p := profile.Detect(ctx, profile.Default(nil), timeout)
|
||||||
if jsonOut {
|
if jsonOut {
|
||||||
@@ -788,7 +794,18 @@ func enrol(ctx context.Context, opts options) error {
|
|||||||
fmt.Printf("this node's overlay key is the found tunnel's (%s): %s\n", tun, mine.Overlay.Public)
|
fmt.Printf("this node's overlay key is the found tunnel's (%s): %s\n", tun, mine.Overlay.Public)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if found == nil {
|
switch {
|
||||||
|
case found == nil && token.Tunnel != nil:
|
||||||
|
// Through the tunnel (novox/hq ADR 0169): the key `key` made, which the token names, and the
|
||||||
|
// tunnel brought up from the token before the bus is dialled — the bus is reached over it.
|
||||||
|
mine.Overlay, err = tunnelKeyFor(token.Tunnel, opts.state)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := bringTheTunnelUp(ctx, token.Tunnel, identity.OverlayKeyPath(opts.state), apply.ExecRunner); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case found == nil:
|
||||||
mine.Overlay, err = identity.GenerateOverlayKey()
|
mine.Overlay, err = identity.GenerateOverlayKey()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -1607,6 +1607,10 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
|
|||||||
for _, c := range r.Capabilities {
|
for _, c := range r.Capabilities {
|
||||||
b.WriteString("cap " + c + "\n")
|
b.WriteString("cap " + c + "\n")
|
||||||
}
|
}
|
||||||
|
// And where it logs (ADR 0179): the runtime cannot move a running container's output.
|
||||||
|
if r.Logging != "" {
|
||||||
|
b.WriteString("log " + r.Logging + "\n")
|
||||||
|
}
|
||||||
// The cadence is part of what was declared, so a changed schedule is a changed spec — the marker
|
// The cadence is part of what was declared, so a changed schedule is a changed spec — the marker
|
||||||
// moves and the install is reported "updated" and re-established. Added only when present, so no
|
// moves and the install is reported "updated" and re-established. Added only when present, so no
|
||||||
// ordinary container's or run-once step's digest moves for a field it does not set.
|
// ordinary container's or run-once step's digest moves for a field it does not set.
|
||||||
@@ -1804,6 +1808,11 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
|
|||||||
for _, c := range r.Capabilities {
|
for _, c := range r.Capabilities {
|
||||||
args = append(args, "--cap-add", c)
|
args = append(args, "--cap-add", c)
|
||||||
}
|
}
|
||||||
|
if r.Logging != "" {
|
||||||
|
// The journal keeps the container's name on every line (CONTAINER_NAME), which is what a
|
||||||
|
// jail matches on (novox/hq ADR 0179); `docker logs` keeps working against the journal.
|
||||||
|
args = append(args, "--log-driver", r.Logging)
|
||||||
|
}
|
||||||
for _, d := range r.Dns {
|
for _, d := range r.Dns {
|
||||||
args = append(args, "--dns", d)
|
args = append(args, "--dns", d)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/declaration"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A container declared to log to the journal is run with the journal as its log driver, and the
|
||||||
|
// place it logs is part of its spec, so moving it recreates the container (novox/hq ADR 0179).
|
||||||
|
func TestAContainerLoggingToTheJournalIsRunThatWayAndRecreatedWhenMoved(t *testing.T) {
|
||||||
|
pinned := "postgres@sha256:" + strings.Repeat("a", 64)
|
||||||
|
var ran []string
|
||||||
|
run := func(_ context.Context, cmd string, args ...string) (string, error) {
|
||||||
|
if cmd == "docker" && len(args) > 0 && args[0] == "run" {
|
||||||
|
ran = args
|
||||||
|
return "deadbeef\n", nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
||||||
|
{"id":"front","type":"container","name":"front","image":"`+pinned+`","logging":"journald"}
|
||||||
|
]}`)
|
||||||
|
_, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
||||||
|
sent := false
|
||||||
|
for i, a := range ran {
|
||||||
|
if a == "--log-driver" && i+1 < len(ran) && ran[i+1] == "journald" {
|
||||||
|
sent = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !sent {
|
||||||
|
t.Fatalf("the container's output was not sent to the journal: %v", ran)
|
||||||
|
}
|
||||||
|
with := d.Resources[0].(*declaration.Container)
|
||||||
|
without := *with
|
||||||
|
without.Logging = ""
|
||||||
|
if containerSpec(with, inputs{}) == containerSpec(&without, inputs{}) {
|
||||||
|
t.Fatal("where a container logs is not part of its spec, so moving it would not recreate it")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -119,6 +119,11 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
|
|||||||
// its ports and range on and not another that came up since.
|
// its ports and range on and not another that came up since.
|
||||||
args = append(args, "--tunnel", o.Tunnel)
|
args = append(args, "--tunnel", o.Tunnel)
|
||||||
}
|
}
|
||||||
|
// The enrolment account the token's secret is the password of exists in the mesh's records
|
||||||
|
// and nowhere on the bus yet (novox/hq 04-ISSUES/146): placed before the machine presents it.
|
||||||
|
if err := placeTheBusUsers(ctx, control, say); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
joined, err := control.run(joining, o.Host, args...)
|
joined, err := control.run(joining, o.Host, args...)
|
||||||
cancel()
|
cancel()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -137,6 +142,10 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
|
|||||||
}
|
}
|
||||||
out.Joined = true
|
out.Joined = true
|
||||||
say(" enrolled as " + o.Node)
|
say(" enrolled as " + o.Node)
|
||||||
|
// And the node's own account, minted as it enrolled, before its agent connects as it.
|
||||||
|
if err := placeTheBusUsers(ctx, control, say); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. The agent.
|
// 4. The agent.
|
||||||
@@ -148,6 +157,40 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// busAccounts and busContainer are where the installer's bundle raises the bus: the file its
|
||||||
|
// configuration includes, and the container that reads it. The installer raised them, so it is the
|
||||||
|
// one that knows them (examples/foundation-first-node-nats.lock).
|
||||||
|
const (
|
||||||
|
busAccounts = "/var/lib/mesh-bus-conf/accounts.conf"
|
||||||
|
busContainer = "mesh-broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// placeTheBusUsers writes the mesh's composed user list beside the bus the installer raised, and makes
|
||||||
|
// the bus re-read it.
|
||||||
|
//
|
||||||
|
// **Genesis's own step** (novox/hq 04-ISSUES/146). Every account on the bus reaches it in the
|
||||||
|
// declaration of the machine that runs it — which needs that machine to be an enrolled node, and at
|
||||||
|
// genesis it is not. The control plane composes the list and says it; whoever raised the bus places
|
||||||
|
// it. That is this installer: it carried the bus in its bundle, so it knows where the bus reads it,
|
||||||
|
// and the control plane never has to.
|
||||||
|
func placeTheBusUsers(ctx context.Context, control controlPlane, say func(string)) error {
|
||||||
|
users, err := control.tell(ctx, "broker", "accounts")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the control plane would not say the bus's users, so no machine could "+
|
||||||
|
"join it: %w", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(users, "accounts") {
|
||||||
|
return fmt.Errorf("the control plane's account of the bus's users is not one:\n%s", indent(users))
|
||||||
|
}
|
||||||
|
script := "umask 077 && cat > " + busAccounts + ".next <<'MESHBUSUSERS'\n" + users + "\nMESHBUSUSERS\n" +
|
||||||
|
"mv " + busAccounts + ".next " + busAccounts + " && docker kill -s HUP " + busContainer + " >/dev/null"
|
||||||
|
if out, err := control.run(ctx, "sh", "-c", script); err != nil {
|
||||||
|
return fmt.Errorf("the bus's users could not be placed at %s: %w\n%s", busAccounts, err, indent(out))
|
||||||
|
}
|
||||||
|
say(" bus users placed")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// runTheHost makes sure something on this machine is listening to the mesh, and proves it.
|
// runTheHost makes sure something on this machine is listening to the mesh, and proves it.
|
||||||
//
|
//
|
||||||
// **The installer does not install the service, and says so.** A unit file is a packaging decision
|
// **The installer does not install the service, and says so.** A unit file is a packaging decision
|
||||||
|
|||||||
@@ -261,3 +261,56 @@ func TestAListingIsMatchedByNameAndNotBySubstring(t *testing.T) {
|
|||||||
t.Error("registry-mirror was not found")
|
t.Error("registry-mirror was not found")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **Genesis places the bus's users, before the machine enrols and again after** (novox/hq
|
||||||
|
// 04-ISSUES/146). The enrolment account exists only in the mesh's records until somebody writes it
|
||||||
|
// beside the bus; so does the node's own, minted as it enrols. Without the first, the machine is
|
||||||
|
// refused by the bus it just raised; without the second, its agent is.
|
||||||
|
func TestAFirstNodeIsLetOntoTheBusItRaised(t *testing.T) {
|
||||||
|
enrolled := false
|
||||||
|
runtime := &asked{answer: func(name string, args []string) (string, error) {
|
||||||
|
joined := strings.Join(args, " ")
|
||||||
|
switch {
|
||||||
|
case strings.Contains(joined, "node list"):
|
||||||
|
if enrolled {
|
||||||
|
return "anchor here 01J0\n", nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
case strings.Contains(joined, "node add"):
|
||||||
|
return "added anchor\n", nil
|
||||||
|
case strings.Contains(joined, "token issue"):
|
||||||
|
return "a token for anchor, good once:\n\n " + strings.Repeat("t", 240) + "\n\n", nil
|
||||||
|
case strings.Contains(joined, "broker accounts"):
|
||||||
|
return "accounts {\n MESH { users = [] }\n}\n", nil
|
||||||
|
case name == "/usr/local/bin/mesh-host":
|
||||||
|
enrolled = true
|
||||||
|
return "enrolled as anchor\n", nil
|
||||||
|
case name == "pgrep", name == "sh":
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("unexpected: %s %v", name, args)
|
||||||
|
}}
|
||||||
|
|
||||||
|
if _, err := Enrol(context.Background(), Options{
|
||||||
|
Node: "anchor", State: filepath.Join(t.TempDir(), "state.json"), Timeout: time.Second,
|
||||||
|
Host: "/usr/local/bin/mesh-host", HostInBackground: true,
|
||||||
|
}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
|
||||||
|
func(string) {}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
placed, enrol := []int{}, -1
|
||||||
|
for i, c := range runtime.commands {
|
||||||
|
if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker") &&
|
||||||
|
strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") {
|
||||||
|
placed = append(placed, i)
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(c, "/usr/local/bin/mesh-host enrol") {
|
||||||
|
enrol = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if enrol < 0 || len(placed) != 2 || placed[0] > enrol || placed[1] < enrol {
|
||||||
|
t.Fatalf("the bus's users were not placed before the machine enrolled and again after "+
|
||||||
|
"(placed at %v, enrolled at %d):\n%s", placed, enrol, strings.Join(runtime.commands, "\n"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -952,6 +952,14 @@ type Container struct {
|
|||||||
// container; a privileged container stays undeclarable.
|
// container; a privileged container stays undeclarable.
|
||||||
Capabilities []string `json:"capabilities,omitempty"`
|
Capabilities []string `json:"capabilities,omitempty"`
|
||||||
|
|
||||||
|
// Logging names where the runtime sends this container's output: "journald" sends it to the
|
||||||
|
// machine's journal, under the container's name, where what reads the machine's logs — its
|
||||||
|
// intrusion prevention first of all (novox/hq ADR 0179) — can read it the way it reads the
|
||||||
|
// machine's own services. Empty keeps the runtime's default, which is a file of the runtime's
|
||||||
|
// own that nothing but the runtime reads. Part of the spec: a container that logs elsewhere
|
||||||
|
// is a different container, and the runtime cannot change a running one's driver.
|
||||||
|
Logging string `json:"logging,omitempty"`
|
||||||
|
|
||||||
// Networks are networks this container also joins once created, by name — a found network a
|
// Networks are networks this container also joins once created, by name — a found network a
|
||||||
// per-machine setting keeps for a taken container (novox/hq ADR 0163, rule 4), so a
|
// per-machine setting keeps for a taken container (novox/hq ADR 0163, rule 4), so a
|
||||||
// neighbour that resolves it there keeps resolving it until the neighbour is taken too.
|
// neighbour that resolves it there keeps resolving it until the neighbour is taken too.
|
||||||
@@ -1057,6 +1065,10 @@ func (c *Container) validate(where string, _ bool) []string {
|
|||||||
"capability's name (CAP_NET_ADMIN or NET_ADMIN)")
|
"capability's name (CAP_NET_ADMIN or NET_ADMIN)")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if c.Logging != "" && c.Logging != "journald" {
|
||||||
|
problems = append(problems, where+": logging is "+strconv.Quote(c.Logging)+", and the only place a "+
|
||||||
|
"container's output can be sent besides the runtime's own file is \"journald\"")
|
||||||
|
}
|
||||||
for _, n := range c.Networks {
|
for _, n := range c.Networks {
|
||||||
problems = append(problems, (&Network{Name: n}).validate(where+": networks", false)...)
|
problems = append(problems, (&Network{Name: n}).validate(where+": networks", false)...)
|
||||||
if n == c.Network {
|
if n == c.Network {
|
||||||
|
|||||||
@@ -524,3 +524,24 @@ func TestACapabilityIsNamedOrRefused(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A container may send its output to the machine's journal, and nowhere else but the runtime's own
|
||||||
|
// file (novox/hq ADR 0179): what reads the machine's logs then reads the container's too.
|
||||||
|
func TestAContainerMayLogToTheJournalAndNowhereElse(t *testing.T) {
|
||||||
|
image := "postgres@sha256:" + strings.Repeat("a", 64)
|
||||||
|
d, err := Parse([]byte(`{"declaration":1,"resources":[
|
||||||
|
{"id":"front","type":"container","name":"front","image":"` + image + `","logging":"journald"}
|
||||||
|
]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := d.Resources[0].(*Container).Logging; got != "journald" {
|
||||||
|
t.Fatalf("logging read as %q", got)
|
||||||
|
}
|
||||||
|
for _, bad := range []string{`"syslog"`, `"none"`, `"json-file"`} {
|
||||||
|
if _, err := Parse([]byte(`{"declaration":1,"resources":[
|
||||||
|
{"id":"front","type":"container","name":"front","image":"` + image + `","logging":` + bad + `}]}`)); err == nil {
|
||||||
|
t.Errorf("%s was accepted as a place to log", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -409,3 +409,26 @@ func TestATokenSaysWhatTheMeshCallsThisMachine(t *testing.T) {
|
|||||||
t.Fatalf("the name did not survive the token: %q", token.Node)
|
t.Fatalf("the name did not survive the token: %q", token.Node)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A token through the tunnel carries the one peer, in the field names the control plane writes
|
||||||
|
// (novox/hq ADR 0169), and an incomplete tunnel is refused naming what is missing.
|
||||||
|
func TestATokenThroughTheTunnelParsesAndAPartOneIsRefused(t *testing.T) {
|
||||||
|
whole := map[string]any{"v": 1, "node": "n", "broker": "10.42.0.1:4222", "fingerprint": "sha256:x",
|
||||||
|
"signer": make([]byte, 32), "secret": "s",
|
||||||
|
"tunnel": map[string]any{"key": "k", "address": "10.42.0.9/32", "range": "10.42.0.0/16",
|
||||||
|
"hub_key": "h", "hub_endpoint": "198.51.100.1:51820"}}
|
||||||
|
raw, _ := json.Marshal(whole)
|
||||||
|
got, err := ParseToken(base64.RawURLEncoding.EncodeToString(raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Tunnel == nil || got.Tunnel.HubEndpoint != "198.51.100.1:51820" || got.Tunnel.Range != "10.42.0.0/16" {
|
||||||
|
t.Fatalf("the tunnel was not read: %+v", got.Tunnel)
|
||||||
|
}
|
||||||
|
whole["tunnel"] = map[string]any{"key": "k"}
|
||||||
|
raw, _ = json.Marshal(whole)
|
||||||
|
if _, err := ParseToken(base64.RawURLEncoding.EncodeToString(raw)); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "the hub's tunnel key") {
|
||||||
|
t.Fatalf("a token with half a tunnel was taken: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The node's key on the private network, which is a different key from the one that says who it
|
// The node's key on the private network, which is a different key from the one that says who it
|
||||||
@@ -64,6 +66,19 @@ func OverlayKeyFrom(privateBase64 string) (OverlayKey, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LoadOverlayKey reads the key `key` made and left in its file (novox/hq ADR 0169).
|
||||||
|
func LoadOverlayKey(path string) (OverlayKey, error) {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return OverlayKey{}, err
|
||||||
|
}
|
||||||
|
key, err := OverlayKeyFrom(strings.TrimSpace(string(raw)))
|
||||||
|
if err != nil {
|
||||||
|
return OverlayKey{}, fmt.Errorf("%s does not hold a tunnel key: %w", path, err)
|
||||||
|
}
|
||||||
|
return key, nil
|
||||||
|
}
|
||||||
|
|
||||||
// OverlayKeyPath is where the private half lives: a file of its own, referenced by the interface
|
// OverlayKeyPath is where the private half lives: a file of its own, referenced by the interface
|
||||||
// configuration rather than embedded in it.
|
// configuration rather than embedded in it.
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -35,6 +35,21 @@ type Token struct {
|
|||||||
// firewall found here before enrolling, because an adopted node keeps that firewall in force.
|
// firewall found here before enrolling, because an adopted node keeps that firewall in force.
|
||||||
// Absent for a converged node.
|
// Absent for a converged node.
|
||||||
Adopted bool `json:"adopted,omitempty"`
|
Adopted bool `json:"adopted,omitempty"`
|
||||||
|
|
||||||
|
// Tunnel is this machine's first tunnel, when the token was issued for the key it made with
|
||||||
|
// `key` (novox/hq ADR 0169): its own address and the hub to reach. It brings the tunnel up from
|
||||||
|
// this alone and reaches the bus over it, so the bus never has to face the internet.
|
||||||
|
Tunnel *TokenTunnel `json:"tunnel,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TokenTunnel is the joining machine's side of its first tunnel. Field names are the wire format
|
||||||
|
// the control plane writes.
|
||||||
|
type TokenTunnel struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Address string `json:"address"`
|
||||||
|
Range string `json:"range"`
|
||||||
|
HubKey string `json:"hub_key"`
|
||||||
|
HubEndpoint string `json:"hub_endpoint"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseToken reads a token a person pasted.
|
// ParseToken reads a token a person pasted.
|
||||||
@@ -70,6 +85,17 @@ func ParseToken(encoded string) (Token, error) {
|
|||||||
if strings.TrimSpace(t.Secret) == "" {
|
if strings.TrimSpace(t.Secret) == "" {
|
||||||
missing = append(missing, "the one-time secret")
|
missing = append(missing, "the one-time secret")
|
||||||
}
|
}
|
||||||
|
if tt := t.Tunnel; tt != nil {
|
||||||
|
for _, part := range []struct{ value, says string }{
|
||||||
|
{tt.Key, "the tunnel key it was issued for"}, {tt.Address, "this machine's address"},
|
||||||
|
{tt.Range, "the private network's range"}, {tt.HubKey, "the hub's tunnel key"},
|
||||||
|
{tt.HubEndpoint, "where the hub's tunnel is dialled"},
|
||||||
|
} {
|
||||||
|
if strings.TrimSpace(part.value) == "" {
|
||||||
|
missing = append(missing, part.says)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if len(missing) > 0 {
|
if len(missing) > 0 {
|
||||||
// Refused whole rather than used partially. A token missing the fingerprint would have
|
// Refused whole rather than used partially. A token missing the fingerprint would have
|
||||||
// this node connect to whatever answers at that address, and one missing the signing key
|
// this node connect to whatever answers at that address, and one missing the signing key
|
||||||
|
|||||||
Reference in New Issue
Block a user