jschoubben
b32a27c115
mesh/delivery merged: source.main moved to novox/mesh-host@5f16cf507a0f
mesh/check pass b32a27c115a2 on 0dfbe1a179f2 | unit 292 run, 1060 kept, 5 skipped | integration 295 run, 1062 kept, 4 skipped | over bound: unit (slow…
The facts look inside an apply runs while the module is still applying and judges nothing; the running write then left xorg running and unknown until the five-minute statement, holding every x11-display consumer behind it. The running write now nudges the health loop, which looks and says at once. A declaration's report gets the same motion and composition check, and a removed module leaves nothing marked unjudged (review of fix 15).
2133 lines
90 KiB
Go
2133 lines
90 KiB
Go
// Command mesh-host is tier 0 of the Novox Mesh: the one thing installed by hand, and the
|
|
// only thing that changes a machine.
|
|
//
|
|
// Stage 1 (novox/hq 03-DESIGN/01-to-be/05-the-node-host.md) is profile and inventory only —
|
|
// the host reads what this machine can do and what it is, and reports it. It applies nothing,
|
|
// connects to nothing, and listens on nothing.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ed25519"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"text/tabwriter"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
|
"github.com/novox/mesh-host/internal/accounts"
|
|
"github.com/novox/mesh-host/internal/agent"
|
|
"github.com/novox/mesh-host/internal/apply"
|
|
"github.com/novox/mesh-host/internal/bundle"
|
|
"github.com/novox/mesh-host/internal/firewall"
|
|
"github.com/novox/mesh-host/internal/identity"
|
|
"github.com/novox/mesh-host/internal/inventory"
|
|
"github.com/novox/mesh-host/internal/link"
|
|
"github.com/novox/mesh-host/internal/liveness"
|
|
"github.com/novox/mesh-host/internal/network"
|
|
"github.com/novox/mesh-host/internal/outward"
|
|
"github.com/novox/mesh-host/internal/profile"
|
|
"github.com/novox/mesh-host/internal/reachable"
|
|
"github.com/novox/mesh-host/internal/store"
|
|
"github.com/novox/mesh-host/internal/system"
|
|
"github.com/novox/mesh-host/internal/tunnel"
|
|
"github.com/novox/mesh-host/internal/units"
|
|
"github.com/novox/mesh-host/internal/upgrade"
|
|
)
|
|
|
|
// version is stamped at build time. Unset in a development build, and said so rather than
|
|
// defaulted to something that looks like a release.
|
|
// builtFor names the operating system this host was built for, set at link time
|
|
// (novox/hq ADR 0005). A host built without one refuses to do anything that touches the
|
|
// machine, rather than guessing and calling a package manager that is not there.
|
|
var builtFor = ""
|
|
|
|
var version = "development build"
|
|
|
|
// runningVersion is this host's version: the directory it was delivered into, or the link-time stamp
|
|
// for one placed by hand.
|
|
//
|
|
// **From where it sits, not from its linker** (novox/hq ADR 0142): "It is unpacked into a directory
|
|
// named for its version, so it can read its own version from its path. The stamp goes, and with it the
|
|
// need for a build to know what it will be called."
|
|
//
|
|
// The mesh's toolchain does not stamp a version, on purpose — a build does not know what it will be
|
|
// called — so a delivered host read as "development build" and the mesh could not tell which host any
|
|
// machine ran (novox/hq 04-ISSUES/161, and 087 for why that matters). The path knows: a delivered host
|
|
// lives at `<libexec>/versions/<version>/<binary>`.
|
|
//
|
|
// A host placed by hand keeps its stamp, which is the honest answer for one the mesh did not deliver.
|
|
func runningVersion() string {
|
|
self, err := os.Executable()
|
|
if err != nil {
|
|
return version
|
|
}
|
|
return versionAt(self, version)
|
|
}
|
|
|
|
// versionAt is runningVersion's decision, with the executable's path and the link-time stamp given —
|
|
// so a test can ask it about a path without being that binary.
|
|
func versionAt(self, stamped string) string {
|
|
// .../versions/<version>/<binary> — the parent is the version, and its parent is the versions
|
|
// directory. Checked rather than assumed, so a binary somewhere else does not read a directory
|
|
// name as a version.
|
|
dir := filepath.Dir(self)
|
|
if filepath.Base(filepath.Dir(dir)) != upgrade.VersionsDirName {
|
|
return stamped
|
|
}
|
|
if name := filepath.Base(dir); name != "" && name != "." && name != string(filepath.Separator) {
|
|
return name
|
|
}
|
|
return stamped
|
|
}
|
|
|
|
const usage = `nox-mesh-host — the node-engine
|
|
|
|
A daemon: its launcher runs it, and it applies what the mesh declares for this machine. The
|
|
operator's command is nox (mesh-cli), on any node, from their own account (novox/hq ADR 0272);
|
|
the subcommands here are for installing a machine, joining it, and rescuing one no mesh can
|
|
reach, run as root at the machine.
|
|
|
|
profile what this machine can be asked to do
|
|
inventory what this machine is, and what it holds
|
|
bundle show what this host carries
|
|
key make this machine's tunnel key, or read the one it made, and print the public
|
|
half: what its join token is issued for (novox/hq ADR 0169)
|
|
enrol --token T join the mesh — through the tunnel when the token was issued for a key
|
|
overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this
|
|
node's overlay key and the mesh is told, signed; --tunnel <iface> when several are up
|
|
owned what this host has applied and still owns
|
|
hand-over <dir> hand a directory used as found to the mesh: the next apply gives it its declared
|
|
owner and mode. The operator's line is nox node hand-over <node> <dir> on the
|
|
control-node; this is the same act for a machine with no mesh to reach
|
|
judge-directories which directories this node is declared the walk that follows no link another
|
|
account could make would refuse, read-only (novox/hq issue 358); run as root
|
|
version
|
|
|
|
--json machine-readable output
|
|
--timeout how long any single probe may take (default 10s)
|
|
--state where this node keeps what it knows (default /var/lib/mesh-host/state.json)
|
|
--dry-run say what applying would change, and change nothing
|
|
|
|
Apply says what it would change before changing anything, and refuses a declaration for the other mode than this
|
|
node is in. It is for a machine the mesh has not spoken to. Once enrolled, the node follows its wants (novox/hq ADR
|
|
0539): it applies a module from the composition its want names, and no whole declaration.
|
|
`
|
|
|
|
func main() {
|
|
// A probe runs a command on a real machine. Ctrl-C must stop the host, not be swallowed by
|
|
// whatever it is waiting for.
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
|
|
command, opts, err := parseArgs(os.Args[1:])
|
|
if err == nil {
|
|
err = run(ctx, command, opts)
|
|
}
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "mesh-host: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
type options struct {
|
|
json bool
|
|
timeout time.Duration
|
|
state string
|
|
token string
|
|
nodeName string
|
|
tunnel string
|
|
dryRun bool
|
|
file string
|
|
// out is where what a command says goes. Stdout, and a buffer under test.
|
|
out io.Writer
|
|
}
|
|
|
|
// parseArgs takes the subcommand first, then its flags.
|
|
//
|
|
// The standard library stops parsing at the first non-flag argument, so `mesh-host inventory
|
|
// --json` left `--json` sitting in the positional arguments and printed text — a flag the user
|
|
// passed, silently ignored, with a successful exit. That is the fault this whole project keeps
|
|
// naming, so the parser takes the subcommand off the front and parses what follows.
|
|
func parseArgs(args []string) (string, options, error) {
|
|
opts := options{timeout: 10 * time.Second, state: store.DefaultPath, out: os.Stdout}
|
|
|
|
command := ""
|
|
if len(args) > 0 {
|
|
command = args[0]
|
|
args = args[1:]
|
|
}
|
|
|
|
set := flag.NewFlagSet("mesh-host", flag.ContinueOnError)
|
|
set.SetOutput(os.Stderr)
|
|
set.Usage = func() { fmt.Fprint(os.Stderr, usage) }
|
|
set.BoolVar(&opts.json, "json", false, "machine-readable output")
|
|
set.DurationVar(&opts.timeout, "timeout", opts.timeout, "how long any single probe may take")
|
|
set.StringVar(&opts.state, "state", opts.state, "where this node keeps what it knows")
|
|
set.BoolVar(&opts.dryRun, "dry-run", false, "read and check the declaration, change nothing")
|
|
set.StringVar(&opts.token, "token", "", "enrol: the one-time token, carried here by a person")
|
|
set.StringVar(&opts.nodeName, "name", "", "enrol: override the name the token carries")
|
|
set.StringVar(&opts.tunnel, "tunnel", "", "enrol, adopted: the found tunnel's interface whose key "+
|
|
"this node takes as its own; found by itself when one is up")
|
|
|
|
// Parsed in a loop, because the standard library stops at the FIRST non-flag argument.
|
|
// `mesh-host inventory --json` hit that once, and taking the subcommand off the front
|
|
// fixed only half of it: `mesh-host apply decl.json --dry-run` left --dry-run unread in
|
|
// exactly the same way. A flag may sit before, after or between positionals, and one that
|
|
// is silently dropped is the fault this whole project keeps naming.
|
|
var positionals []string
|
|
rest := args
|
|
for {
|
|
if err := set.Parse(rest); err != nil {
|
|
return "", opts, err
|
|
}
|
|
rest = set.Args()
|
|
if len(rest) == 0 {
|
|
break
|
|
}
|
|
positionals = append(positionals, rest[0])
|
|
rest = rest[1:]
|
|
}
|
|
|
|
if command == "hand-over" {
|
|
if len(positionals) != 1 {
|
|
return "", opts, errors.New("hand-over <directory>: the path of one directory used as found")
|
|
}
|
|
opts.file = positionals[0]
|
|
return command, opts, nil
|
|
}
|
|
if command == "overlay" {
|
|
if len(positionals) != 1 {
|
|
return "", opts, errors.New("overlay take [--tunnel <iface>]")
|
|
}
|
|
opts.file = positionals[0]
|
|
return command, opts, nil
|
|
}
|
|
// Anything left over was neither the command nor a flag. Refused rather than ignored: a
|
|
// mistyped argument that changes nothing and reports success is worse than an error.
|
|
if len(positionals) > 0 {
|
|
return "", opts, fmt.Errorf("unexpected argument %q — try `nox-mesh-host help`", positionals[0])
|
|
}
|
|
return command, opts, nil
|
|
}
|
|
|
|
func run(ctx context.Context, command string, opts options) error {
|
|
jsonOut, timeout := opts.json, opts.timeout
|
|
// Every apply this process makes counts what a person handed over, and every look says what is used as
|
|
// found (novox/hq issue 339).
|
|
handedOverFrom(opts.state)
|
|
foundState.Store(opts.state)
|
|
// And what it knows of the libraries it keeps is beside its state (novox/hq ADR 0328 §6).
|
|
apply.Libraries = store.NewLibraryFile(opts.state)
|
|
// And the guard asks the running container runtimes where they keep their data (novox/hq issue 339).
|
|
apply.AskRuntimes = apply.ExecRunner
|
|
switch command {
|
|
case "hand-over":
|
|
return handOverCommand(opts)
|
|
case "judge-directories":
|
|
return judgeDirectoriesCommand(opts)
|
|
case "key":
|
|
return keyCommand(opts)
|
|
|
|
case "profile":
|
|
p := profile.Detect(ctx, profile.Default(nil), timeout)
|
|
if jsonOut {
|
|
return writeJSON(p)
|
|
}
|
|
writeProfile(p)
|
|
return nil
|
|
|
|
case "inventory":
|
|
inv := inventory.Collect(ctx, nil, profile.Default(nil), timeout)
|
|
if jsonOut {
|
|
return writeJSON(inv)
|
|
}
|
|
writeInventory(inv)
|
|
return nil
|
|
|
|
case "bundle":
|
|
if bundle.IsEmpty(builtFor) {
|
|
fmt.Println("this host carries no bundle")
|
|
return nil
|
|
}
|
|
_, err := bundle.Load(builtFor)
|
|
if err != nil {
|
|
// Asked before it matters, rather than discovered on a first node.
|
|
return fmt.Errorf("this host carries a bundle it cannot itself read: %w", err)
|
|
}
|
|
os.Stdout.Write(bundle.Raw(builtFor))
|
|
return nil
|
|
|
|
case "owned":
|
|
known, err := store.Load(opts.state)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if jsonOut {
|
|
return writeJSON(known)
|
|
}
|
|
if len(known.Resources) == 0 {
|
|
fmt.Println("this host has applied nothing on this machine")
|
|
return nil
|
|
}
|
|
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
|
for _, r := range known.Resources {
|
|
fmt.Fprintf(w, " %s\t%s\t%s\n", r.Type, r.ID, r.Target)
|
|
}
|
|
return w.Flush()
|
|
|
|
case "enrol", "enroll":
|
|
return enrol(ctx, opts)
|
|
case "overlay":
|
|
return overlayCommand(ctx, opts)
|
|
|
|
case "run":
|
|
return runLink(ctx, opts)
|
|
|
|
case "version":
|
|
fmt.Println(runningVersion())
|
|
return nil
|
|
|
|
case "", "help", "-h", "--help":
|
|
fmt.Fprint(os.Stderr, usage)
|
|
return nil
|
|
|
|
default:
|
|
return fmt.Errorf("unknown command %q — try `nox-mesh-host help`", command)
|
|
}
|
|
}
|
|
|
|
func writeJSON(v any) error { return writeJSONTo(os.Stdout, v) }
|
|
|
|
func writeJSONTo(w io.Writer, v any) error {
|
|
enc := json.NewEncoder(w)
|
|
enc.SetIndent("", " ")
|
|
return enc.Encode(v)
|
|
}
|
|
|
|
// writeProfile prints every verdict WITH its reason.
|
|
//
|
|
// The reason is not decoration: a capability reported absent with no reason is something
|
|
// nobody can act on, and this is the surface where a person meets that.
|
|
func writeProfile(p profile.Profile) {
|
|
fmt.Printf("%s/%s\n\n", p.Kernel, p.Architecture)
|
|
|
|
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
|
for _, v := range p.Capabilities {
|
|
mark := "no "
|
|
if v.Present {
|
|
mark = "yes"
|
|
}
|
|
fmt.Fprintf(w, " %s\t%s\t%s\n", mark, v.Name, v.Detail)
|
|
}
|
|
w.Flush()
|
|
|
|
if missing := p.Missing(); len(missing) > 0 {
|
|
fmt.Printf("\ncannot be asked to: %v\n", missing)
|
|
}
|
|
}
|
|
|
|
func writeInventory(inv inventory.Inventory) {
|
|
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
|
fmt.Fprintf(w, "machine\t%s\n", inv.Machine)
|
|
if inv.Distribution != "" {
|
|
fmt.Fprintf(w, "distribution\t%s\n", inv.Distribution)
|
|
}
|
|
if inv.Kernel != "" {
|
|
fmt.Fprintf(w, "kernel\t%s\n", inv.Kernel)
|
|
}
|
|
fmt.Fprintf(w, "architecture\t%s/%s\n", inv.OS, inv.Architecture)
|
|
fmt.Fprintf(w, "cpus\t%d\n", inv.CPUs)
|
|
if inv.MemoryKB > 0 {
|
|
fmt.Fprintf(w, "memory\t%d MB\n", inv.MemoryKB/1024)
|
|
}
|
|
fmt.Fprintf(w, "observed\t%s\n", inv.ObservedAt.Format(time.RFC3339))
|
|
w.Flush()
|
|
|
|
fmt.Println()
|
|
writeProfile(inv.Profile)
|
|
|
|
// Printed last and never hidden. An inventory that quietly omits what it could not read
|
|
// is the same fault as a report assembled from intent (novox/hq ADR 0018).
|
|
if len(inv.Unreadable) > 0 {
|
|
fmt.Println("\ncould not read:")
|
|
for _, u := range inv.Unreadable {
|
|
fmt.Printf(" %s\n", u)
|
|
}
|
|
}
|
|
}
|
|
|
|
// provenance is where a declaration a command applies came from. It decides the origin its
|
|
// resources are recorded under, what it is held against, and what is recorded once it applied.
|
|
type provenance int
|
|
|
|
const (
|
|
// fromFile is `apply FILE`: handed to the host by someone already running it as root.
|
|
fromFile provenance = iota
|
|
// fromBundle is genesis on a machine the mesh has told nothing yet: the bundle carried in
|
|
// the binary, the first node's path before its mesh is up (novox/hq ADR 0004).
|
|
fromBundle
|
|
)
|
|
|
|
func (p provenance) origin() string {
|
|
// A file handed to the host is applied as the bundle is: what it puts here is invisible to
|
|
// the removal pass of a declaration that later arrives from the mesh (novox/hq issue 010).
|
|
return store.OriginCarried
|
|
}
|
|
|
|
func (p provenance) name(opts options) string {
|
|
switch p {
|
|
case fromBundle:
|
|
return "the carried bundle"
|
|
}
|
|
return opts.file
|
|
}
|
|
|
|
// short is a digest as a person reads one aloud.
|
|
func short(digest string) string {
|
|
if len(digest) > 12 {
|
|
return digest[:12]
|
|
}
|
|
return digest
|
|
}
|
|
|
|
// refuseStale refuses a declaration that is not the one this node should be held to (novox/hq
|
|
// issue 104), naming both.
|
|
//
|
|
// **A declaration carries no order.** The controller signs a version — the vocabulary — the
|
|
// node it is for, the mode, and the resources: no sequence, no issued-at. What exists is
|
|
// identity: the mesh names what it sends by the digest of the bytes, the node keeps the last one
|
|
// it was sent, and genesis records the digest of what it consumed. So the bundle is held to
|
|
// genesis's digest, and a file is not applied at all once the mesh has spoken: a file is applied
|
|
// as the bundle is, its resources recorded as this machine's own — so the mesh could never remove
|
|
// them again — and everything the mesh declared read as no longer declared and removed. Even the
|
|
// very declaration the mesh last sent, applied from a file, would plan to remove the foundation.
|
|
// `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so.
|
|
func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance) error {
|
|
switch from {
|
|
case fromBundle:
|
|
if known.Genesis == nil || known.Genesis.Digest == digest {
|
|
return nil
|
|
}
|
|
// By digest. Genesis applies the bundle rewritten for this machine — its foundation
|
|
// ports, root credentials, mode — and writes that lock out; a host built from the
|
|
// carried template does not match it, and one built from the written lock does.
|
|
return fmt.Errorf("the bundle this host carries (%s) is not the one genesis applied here on %s "+
|
|
"(%s), which was the bundle rewritten for this machine. It was consumed then, and nothing "+
|
|
"the mesh has said is kept on this node yet, so there is nothing to reconcile against: "+
|
|
"enrol this node, or push to it from the controller",
|
|
short(digest), known.Genesis.At.Format(time.RFC3339), short(known.Genesis.Digest))
|
|
}
|
|
// A file.
|
|
switch {
|
|
case errors.Is(keptErr, store.ErrNothingDeclared):
|
|
// The mesh has said nothing here: a machine a file is for.
|
|
return nil
|
|
case keptErr != nil:
|
|
return fmt.Errorf("%w\n\nNothing is applied over what this node cannot read back", keptErr)
|
|
}
|
|
last := apply.DigestOf(kept.Declaration)
|
|
what := fmt.Sprintf("this file (%s) is not it", short(digest))
|
|
switch {
|
|
case digest == last:
|
|
what = "this file is that declaration, and from a file it would still be applied as a bundle is"
|
|
case known.Genesis != nil && digest == known.Genesis.Digest:
|
|
what = fmt.Sprintf("this file is the bundle genesis consumed on %s (%s), older than it",
|
|
known.Genesis.At.Format(time.RFC3339), short(digest))
|
|
}
|
|
return fmt.Errorf("`apply FILE` is for a machine the mesh has not spoken to. The mesh has told this "+
|
|
"node declaration %s, kept as %s, and %s. A file is applied as the bundle is — its resources "+
|
|
"recorded as this machine's own, which the mesh could then never remove, and what the mesh "+
|
|
"declared read as no longer declared — so no file is applied here: the node follows its wants "+
|
|
"(novox/hq ADR 0539)",
|
|
short(last), store.DeclaredName, what)
|
|
}
|
|
|
|
// applied is what an apply says in machine-readable form: what it planned, then what it did.
|
|
type applied struct {
|
|
Plan []apply.Step `json:"plan"`
|
|
// Report is absent on a dry run, which is the plan and nothing more.
|
|
Report *apply.Report `json:"report,omitempty"`
|
|
}
|
|
|
|
// runApply makes the machine match a declaration — after refusing one this node must not apply,
|
|
// and after saying what it would change.
|
|
//
|
|
// Refused first, and before anything is read from the machine: a declaration for the other
|
|
// mode than this node is in, or one older than what the mesh last said (novox/hq issue 104).
|
|
// Then the plan, printed whole before a single resource is touched; `--dry-run` is that and
|
|
// nothing more.
|
|
//
|
|
// The state is loaded before anything is touched and saved after, including when the apply
|
|
// fails part-way: what was applied before the failure is on the machine, and a host that did
|
|
// not record it would believe it owns less than it does and leave that behind forever.
|
|
func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw []byte, from provenance) error {
|
|
out := opts.out
|
|
if out == nil {
|
|
out = os.Stdout
|
|
}
|
|
// One apply at a time on this machine, whichever process asks: the link service applies too,
|
|
// and two saves of the state interleaved lose what one of them recorded.
|
|
unlock, err := store.Lock(opts.state, func() {
|
|
fmt.Fprintln(out, "another apply holds this node's state — mesh-host run, or the installer; waiting for it to finish")
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer unlock()
|
|
known, err := store.Load(opts.state)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
source, origin, digest := from.name(opts), from.origin(), apply.DigestOf(raw)
|
|
|
|
// The mode this node is in. What the mesh last said is signed and was verified on load; the
|
|
// state's note of it is this host's own, and where they disagree the note is what is wrong
|
|
// — a genesis re-run over a node the mesh converged since, a state saved when the kept
|
|
// declaration could not be — and is repaired, not obeyed. A bundle or a file is held to the
|
|
// note, or to the kept declaration when the note predates it.
|
|
kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state))
|
|
if known.Mode == "" && keptErr == nil {
|
|
if last, err := declaration.Parse(kept.Declaration); err == nil {
|
|
known.Mode = apply.ModeOf(last)
|
|
}
|
|
}
|
|
if err := apply.CheckMode(known, d); err != nil {
|
|
return err
|
|
}
|
|
if err := refuseStale(known, kept, keptErr, digest, from); err != nil {
|
|
return err
|
|
}
|
|
|
|
// A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has
|
|
// no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty.
|
|
if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil {
|
|
return fmt.Errorf("a file of this declaration could not be had by its digest: %w", err)
|
|
}
|
|
|
|
// Said before anything is done.
|
|
steps := apply.Plan(d, known, origin)
|
|
if opts.json && opts.dryRun {
|
|
return writeJSONTo(out, applied{Plan: steps})
|
|
}
|
|
mode := known.Mode
|
|
if mode == "" {
|
|
mode = "in no mode yet — nothing has said one"
|
|
}
|
|
fmt.Fprintf(out, "%s (%s): %d resource(s), version %d\n", source, short(digest), len(d.Resources), d.Version)
|
|
fmt.Fprintf(out, "this node is %s; the declaration says %s\n", mode, apply.ModeOf(d))
|
|
fmt.Fprintf(out, "\nwould change, in this order:\n")
|
|
for _, step := range steps {
|
|
fmt.Fprintln(out, " "+step.String())
|
|
}
|
|
if opts.dryRun {
|
|
fmt.Fprintf(out, "\n--dry-run: nothing applied\n")
|
|
return nil
|
|
}
|
|
|
|
sys, err := system.For(builtFor)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Refuse a declaration naming a shape this host cannot apply, before anything is applied.
|
|
// An android host has no `package` applier, and finding that out half way through is the
|
|
// half-configured machine this host exists to prevent (novox/hq ADR 0005).
|
|
if err := system.Check(sys, d); err != nil {
|
|
return err
|
|
}
|
|
// And prove this is the machine the host was built for. Installing the arch host on Alpine
|
|
// must say so once, at the start, rather than failing later inside pacman. A package manager that
|
|
// does not answer fails only what needs it (novox/hq issue 233, probe.go).
|
|
probe, err := probePackageManager(ctx, sys)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if probe != nil {
|
|
fmt.Fprintf(out, "\n%s\n", probe.said())
|
|
}
|
|
|
|
fmt.Fprintf(out, "\napplying:\n")
|
|
// Minding the windows the daemon's scheduler has open: this runs as its own process, and a
|
|
// `reconcile` by hand at 03:31 is exactly the apply that must not restart a server a collector
|
|
// is holding still (novox/hq issue 224).
|
|
// Once more when the package manager did not answer and this apply wrote something: what it wrote may be
|
|
// what makes it answer, and then the packages go in now rather than in five minutes (issue 233, probe.go).
|
|
tell := func(line string) {
|
|
if !opts.json {
|
|
fmt.Fprintln(out, line)
|
|
}
|
|
}
|
|
report, updated, applyErr, probe, _ := applyMindingProbe(ctx, sys, probe, known, func(known store.State) (apply.Report, store.State, error) {
|
|
return apply.ApplyMindingWindows(ctx, sys, d, known, origin, apply.ExecRunner, tell, sealOpener(opts.state),
|
|
apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)))
|
|
}, tell)
|
|
|
|
// What this apply settles about the node, whichever way it went. The mode is what the
|
|
// declaration said and the check above agreed with; the bundle, once applied, is consumed.
|
|
updated.Mode = apply.ModeOf(d)
|
|
if from == fromBundle {
|
|
updated.Genesis = &store.Genesis{Digest: digest, At: time.Now().UTC()}
|
|
}
|
|
|
|
// Saved whichever way it went. Recording only on success would lose the footprint of a
|
|
// failed apply, and that footprint is on the machine either way.
|
|
if saveErr := store.Save(opts.state, updated); saveErr != nil {
|
|
if applyErr != nil {
|
|
return fmt.Errorf("%w\n\nand the node's state could not be saved: %v", applyErr, saveErr)
|
|
}
|
|
return saveErr
|
|
}
|
|
if applyErr != nil {
|
|
return applyErr
|
|
}
|
|
if probe != nil {
|
|
return probe.partial()
|
|
}
|
|
|
|
// Only now, and only after a clean apply: this version got as far as a completed
|
|
// reconcile, which is the whole of what "known good" claims (novox/hq ADR 0005). Not
|
|
// health — a disconnected node is ordinary, and a resource that fails is the machine's
|
|
// problem rather than the binary's.
|
|
//
|
|
// A failure to record is reported and does not fail the apply. The apply worked; what is
|
|
// lost is a rollback's ability to come back here, which is worse to hide than to say.
|
|
if v := runningVersion(); v != "" {
|
|
if err := upgrade.RecordKnownGood(upgrade.KnownGoodPath(opts.state), v); err != nil {
|
|
fmt.Fprintf(os.Stderr,
|
|
"mesh-host: applied, but could not record %s as known-good: %v\n"+
|
|
" a rollback would have nothing to return to.\n", version, err)
|
|
}
|
|
}
|
|
// And retire what is older than this version's predecessor, on the same evidence known-good is
|
|
// written on (novox/hq ADR 0141). The predecessor stays, because it is exactly what a rollback
|
|
// starts; everything before it has no reader. Never this version, whatever the answer.
|
|
//
|
|
// A failure is said and does not fail the apply, for the reason above: what is lost is disk, and
|
|
// hiding it would make a machine quietly fill up.
|
|
// Asked with the version this host RUNS, read from where it sits: the link-time stamp names no
|
|
// delivered version, and retiring around a name that is not there would remove the one running.
|
|
if v := runningVersion(); v != "" {
|
|
if retired, err := upgrade.Retire(upgrade.VersionsDir(""), v); err != nil {
|
|
fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not retire an older host: %v\n", err)
|
|
} else if len(retired) > 0 {
|
|
fmt.Fprintf(os.Stderr, "mesh-host: retired the host version(s) %s\n",
|
|
strings.Join(retired, ", "))
|
|
}
|
|
}
|
|
// And tell the launcher this start worked. Without it the counter only climbs, and a node
|
|
// that has been up for months rolls itself back on its third ordinary restart.
|
|
if err := upgrade.ClearAttempts(upgrade.AttemptsPath(opts.state)); err != nil {
|
|
fmt.Fprintf(os.Stderr,
|
|
"mesh-host: applied, but could not clear the start counter: %v\n"+
|
|
" this node may roll itself back after a few more restarts.\n", err)
|
|
}
|
|
|
|
if opts.json {
|
|
return writeJSONTo(out, applied{Plan: steps, Report: &report})
|
|
}
|
|
if !report.Changed() {
|
|
fmt.Fprintf(out, "%s: already matches — %d resource(s) checked\n", source, len(report.Outcomes))
|
|
return nil
|
|
}
|
|
fmt.Fprintf(out, "%s: applied — %d resource(s)\n", source, len(report.Outcomes))
|
|
return nil
|
|
}
|
|
|
|
// enrol joins this machine to a mesh.
|
|
//
|
|
// novox/hq 09-the-node-lifecycle: the token carries four things, the node dials the broker over
|
|
// the underlay, checks the certificate against the pin *before sending anything*, and presents
|
|
// the one-time secret together with a public key it generated itself.
|
|
//
|
|
// The mesh issues no identity. This machine arrives holding one; what it receives is being known.
|
|
func enrol(ctx context.Context, opts options) error {
|
|
tokenText, name := &opts.token, &opts.nodeName
|
|
if strings.TrimSpace(*tokenText) == "" {
|
|
return errors.New("enrol --token <token>: the token is carried to this machine by a " +
|
|
"person, and is the only thing it needs")
|
|
}
|
|
|
|
// Refused whole if incomplete. A token without the fingerprint would have this machine
|
|
// connect to whatever answers; without the signing key it could not tell a declaration from
|
|
// a forgery, and it applies whatever the link delivers.
|
|
token, err := identity.ParseToken(*tokenText)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// The name comes from the token, because the node cannot work it out: the broker account it
|
|
// authenticates as is named after it, and that account exists before this machine has been
|
|
// told anything. --name remains for a token issued before the name travelled in one, and
|
|
// saying so beats a connection refused with an empty username — which is what this was.
|
|
if strings.TrimSpace(*name) == "" {
|
|
*name = token.Node
|
|
}
|
|
if strings.TrimSpace(*name) == "" {
|
|
return errors.New(
|
|
"this token does not say what the mesh calls this machine, and no --name was given. " +
|
|
"A token issued by a current control plane carries the name")
|
|
}
|
|
|
|
// Before anything else: an already-enrolled machine must not quietly acquire a second
|
|
// identity. The mesh believes the first one, and re-enrolling is a deliberate act that
|
|
// starts with a person issuing a new token for that node record.
|
|
identityPath := identity.Path(opts.state)
|
|
switch existing, err := identity.Load(identityPath); {
|
|
case err == nil:
|
|
return fmt.Errorf(
|
|
"this machine is already node %q. Re-enrolling replaces the identity the mesh "+
|
|
"believes, so it is done deliberately: remove %s first",
|
|
existing.Node, identityPath)
|
|
case errors.Is(err, identity.ErrNoIdentity):
|
|
default:
|
|
return err
|
|
}
|
|
|
|
// An adopted node keeps the firewall it was found with (novox/hq ADR 0100), so a host that
|
|
// cannot speak that firewall must say so now — before the mesh records a node it could never
|
|
// open anything on.
|
|
if token.Adopted {
|
|
kind, name, err := firewall.Detect(ctx, apply.ExecRunner)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if kind == firewall.Unsupported {
|
|
return fmt.Errorf(
|
|
"this token joins this machine adopted, keeping the firewall found on it, and it is "+
|
|
"filtered by %s, which no host speaks yet. Nothing was enrolled", name)
|
|
}
|
|
fmt.Printf("joining adopted: what is on this machine is kept, and its firewall (%s) stays in force\n",
|
|
string(kind))
|
|
}
|
|
|
|
fmt.Printf("token for broker %s\n", token.Broker)
|
|
fmt.Printf(" pinned certificate %s\n", token.Fingerprint)
|
|
fmt.Printf(" signing key %s\n",
|
|
base64.StdEncoding.EncodeToString(token.Signer)[:16]+"...")
|
|
|
|
// **The pin is checked by the connection that presents this token, not by a dial of our own**
|
|
// (novox/hq 04-ISSUES/146). This opened a raw TLS connection to the bus first, which worked
|
|
// against the broker the mesh used to run and cannot work against the one it runs now: NATS
|
|
// speaks its own protocol before it upgrades to TLS, so an immediate handshake is answered
|
|
// with a plaintext line and the enrolment failed with "first record does not look like a TLS
|
|
// handshake" — on every node that has tried to join since the bus changed, which is why this
|
|
// went unnoticed: none had.
|
|
//
|
|
// What ADR 0004 requires still holds, and holds better: the client that presents the token
|
|
// carries the same pinned configuration, reads the server's greeting, upgrades, and the
|
|
// verification runs inside that handshake — so the one-time secret is sent only after the
|
|
// certificate has been checked, and nothing of this node's reaches an impostor.
|
|
|
|
mine, err := identity.Generate(*name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("generated this node's identity: %s\n", mine.PublicBase64())
|
|
|
|
// Its key on the private network. Generated here and now, for the same reason: the private
|
|
// half must never have been anywhere else. The mesh receives only the public half and uses it
|
|
// to compute a graph it cannot impersonate.
|
|
//
|
|
// **Except on an adopted node with a tunnel** (novox/hq ADR 0105): the found interface's key
|
|
// becomes this node's, so the peers that know the tunnel by that key keep reaching it once
|
|
// the mesh's interface takes the tunnel over. The one case where the mesh takes a credential
|
|
// it did not mint — read from the found configuration, written where a generated one is
|
|
// written, never printed, never sent.
|
|
var found *link.Tunnel
|
|
if token.Adopted {
|
|
tun, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel)
|
|
switch {
|
|
case errors.Is(err, tunnel.ErrNone):
|
|
fmt.Println("no tunnel is up on this machine; the private network's key is generated")
|
|
case err != nil:
|
|
return err
|
|
default:
|
|
mine.Overlay, err = identity.OverlayKeyFrom(tun.PrivateKey())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
found = carried(tun)
|
|
fmt.Printf("this node's overlay key is the found tunnel's (%s): %s\n", tun, mine.Overlay.Public)
|
|
}
|
|
}
|
|
switch {
|
|
case found == nil && token.Tunnel != nil:
|
|
// Through the tunnel (novox/hq ADR 0169): the key `key` made, which the token names, and the
|
|
// tunnel brought up from the token before the bus is dialled — the bus is reached over it.
|
|
mine.Overlay, err = tunnelKeyFor(token.Tunnel, opts.state)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := bringTheTunnelUp(ctx, token.Tunnel, identity.OverlayKeyPath(opts.state), apply.ExecRunner); err != nil {
|
|
return err
|
|
}
|
|
case found == nil:
|
|
mine.Overlay, err = identity.GenerateOverlayKey()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("generated this node's overlay key: %s\n", mine.Overlay.Public)
|
|
}
|
|
|
|
// And the key secrets are sealed to. Here, with the others, because the mesh cannot seal
|
|
// anything to a key it has not been told about — a key made later would leave a node that
|
|
// looks enrolled and can receive no credential.
|
|
sealing, err := identity.GenerateSealingKey()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("generated this node's sealing key: %s\n", sealing.Public)
|
|
|
|
// And the key it serves TLS with on its name inside the mesh. Generated here for the same
|
|
// reason as the others: the private half must never have been anywhere else, and the mesh
|
|
// only ever certifies the public one.
|
|
serving, err := identity.GenerateServingKey()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("generated this node's serving key: %s\n", serving.Public)
|
|
|
|
// What this machine can be asked to do, gathered before joining rather than after. The
|
|
// control plane cannot decide what a node should run without it, so it travels with the
|
|
// request instead of being asked for in a second round trip.
|
|
detected := profile.Detect(ctx, profile.Default(nil), opts.timeout)
|
|
reported := profileAsReported(detected)
|
|
|
|
// Signed with the identity just generated, so the mesh can tell this machine from anyone else
|
|
// who knows its public key (novox/hq issue 083).
|
|
proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
|
|
sealing.Public, serving.Public))
|
|
// The token says where to go and which certificate that address must present. It says nothing
|
|
// about which bus is there, and does not need to: there is one, and this host knows which
|
|
// (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone).
|
|
//
|
|
// **It used to leave this empty** and mean "whatever the mesh runs today", which was true
|
|
// while two buses existed and became a refusal the moment one did: an empty transport is not
|
|
// the bus's name, so every enrolment ended at "this token is for the \"\" bus"
|
|
// (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus
|
|
// changed.
|
|
reply, err := link.Enrol(ctx,
|
|
link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS},
|
|
*name, token.Secret,
|
|
mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found,
|
|
opts.timeout)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// The mesh's name for this node wins over what the machine called itself: the token was
|
|
// issued for a node record, and that record is what the identity binds to.
|
|
mine.Node = reply.Node
|
|
mine.Membership = identity.Membership{
|
|
Broker: firstNonEmpty(reply.Broker, token.Broker),
|
|
Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint),
|
|
Signer: firstNonEmpty2(reply.Signer, token.Signer),
|
|
Password: reply.Password,
|
|
// **Which bus this membership is for, said rather than left empty** (novox/hq
|
|
// 04-ISSUES/146). The link refuses a membership that names another bus, and an empty name
|
|
// is not this one's — so a node enrolled without it came up and reconnected for ever
|
|
// against its own record: "this membership is for \"\", and the mesh's bus is nats". The
|
|
// reply does not carry it because there is one bus and the host knows which (ADR 0131);
|
|
// what was missing was writing that down where the link reads it.
|
|
Transport: link.OnNATS,
|
|
}
|
|
if mine.Membership.Password == "" {
|
|
// The mesh did not replace the token's secret, so it is still this node's broker
|
|
// password. Said rather than silently kept: a one-time secret living on as a credential
|
|
// is worth knowing about.
|
|
mine.Membership.Password = token.Secret
|
|
fmt.Println("\nnote: the mesh issued no separate broker password, so the token's secret " +
|
|
"remains this node's credential")
|
|
}
|
|
|
|
// Saved only now, and only once the mesh has said it knows this node. A node holding an
|
|
// identity the mesh has never recorded would believe it had joined and be believed by
|
|
// nobody — worse than not having joined, because nothing would look wrong.
|
|
if err := identity.Save(identityPath, mine); err != nil {
|
|
return fmt.Errorf(
|
|
"the mesh accepted this node as %q and its identity could not be saved: %w\n"+
|
|
"That token is spent, so getting back needs a new one", reply.Node, err)
|
|
}
|
|
|
|
if !mine.Membership.Joined() {
|
|
return fmt.Errorf(
|
|
"the mesh accepted this node as %q but did not say how to reach it again, so this "+
|
|
"identity could not be used after a restart. Nothing was saved", reply.Node)
|
|
}
|
|
|
|
// Written before the identity, so a node that dies between the two has a key file with no
|
|
// identity — which enrols again cleanly — rather than an identity naming a key that is not
|
|
// there, which looks joined and cannot come up.
|
|
if err := os.WriteFile(identity.OverlayKeyPath(opts.state),
|
|
[]byte(mine.Overlay.Private+"\n"), 0o600); err != nil {
|
|
return fmt.Errorf("cannot write this node's overlay key: %w", err)
|
|
}
|
|
if err := os.WriteFile(identity.SealingKeyPath(opts.state),
|
|
[]byte(sealing.Private+"\n"), 0o600); err != nil {
|
|
return fmt.Errorf("cannot write this node's sealing key: %w", err)
|
|
}
|
|
if err := identity.WriteServingKey(identity.ServingKeyPath(opts.state), serving); err != nil {
|
|
return fmt.Errorf("cannot write this node's serving key: %w", err)
|
|
}
|
|
|
|
fmt.Printf("\nenrolled as %s\n", reply.Node)
|
|
fmt.Printf(" identity %s\n", identityPath)
|
|
fmt.Printf(" queue %s\n", reply.Queue)
|
|
return nil
|
|
}
|
|
|
|
// overlayCommand is `mesh-host overlay take`: this node takes the tunnel found on its machine over
|
|
// after it enrolled (novox/hq ADR 0105). For a node that enrolled before the mesh knew to take a
|
|
// tunnel over — re-enrolling would rotate its identity, sealing and serving keys, and with them
|
|
// every credential the mesh sealed to it.
|
|
func overlayCommand(ctx context.Context, opts options) error {
|
|
if opts.file != "take" {
|
|
return errors.New("overlay take [--tunnel <iface>] — the one thing `overlay` does here")
|
|
}
|
|
identityPath := identity.Path(opts.state)
|
|
mine, err := identity.Load(identityPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
found, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel)
|
|
if err != nil {
|
|
return fmt.Errorf("%w. Nothing was changed", err)
|
|
}
|
|
taken, rekey, err := rekeyOnto(mine, found)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("taking over %s: this node's overlay key becomes the tunnel's, %s\n", found, taken.Overlay.Public)
|
|
fmt.Printf(" identity, sealing and serving keys are untouched\n")
|
|
|
|
// Told first, then written: a mesh told and a machine not yet written is put right by running
|
|
// this again (the mesh refuses the stale second rekey and changes nothing; the files are
|
|
// rewritten the same). A machine written and a mesh not told would raise the mesh's interface
|
|
// on a key the mesh does not know at the next restart.
|
|
if err := link.Publish(ctx, link.Membership{
|
|
Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint,
|
|
Password: mine.Membership.Password, Signer: mine.Membership.Signer,
|
|
Transport: mine.Membership.Transport,
|
|
}, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil {
|
|
return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err)
|
|
}
|
|
fmt.Printf(" told the mesh signed rekey sent; `node show %s` on the controller says whether it took\n", mine.Node)
|
|
|
|
if err := os.WriteFile(identity.OverlayKeyPath(opts.state),
|
|
[]byte(taken.Overlay.Private+"\n"), 0o600); err != nil {
|
|
return fmt.Errorf("the mesh was told and this node's overlay key could not be written: %w — run this again", err)
|
|
}
|
|
if err := identity.Save(identityPath, taken); err != nil {
|
|
return fmt.Errorf("the mesh was told and this node's identity could not be saved: %w — run this again", err)
|
|
}
|
|
fmt.Printf(" written %s and the identity; the mesh's interface reads the key when the next push restarts it\n",
|
|
identity.OverlayKeyPath(opts.state))
|
|
fmt.Printf(" next on the controller: `overlay place %s --hub --endpoint <host>:%d …`, `plan %s --json`, then push\n",
|
|
mine.Node, found.Port, mine.Node)
|
|
return nil
|
|
}
|
|
|
|
// rekeyOnto is the identity with the found tunnel's key as its overlay key, and the signed rekey
|
|
// that tells the mesh. Pure, so it can be held to: node, sealing and serving keys are the same
|
|
// bytes in and out; only the overlay key moves. Run again after a take, the previous key it names
|
|
// is the one before the take, so the mesh can tell a repeat from a replay.
|
|
func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, link.Rekey, error) {
|
|
overlay, err := identity.OverlayKeyFrom(found.PrivateKey())
|
|
if err != nil {
|
|
return identity.Identity{}, link.Rekey{}, err
|
|
}
|
|
previous := mine.Overlay.Public
|
|
if previous == overlay.Public && mine.OverlayBefore != "" {
|
|
previous = mine.OverlayBefore
|
|
}
|
|
taken := mine
|
|
taken.Overlay = overlay
|
|
if previous != overlay.Public {
|
|
taken.OverlayBefore = previous
|
|
}
|
|
presented := carried(found)
|
|
rekey := link.Rekey{Previous: previous, OverlayKey: overlay.Public, Tunnel: presented}
|
|
rekey.Proof = mine.Sign(link.RekeyProof(mine.Node, previous, overlay.Public, presented))
|
|
return taken, rekey, nil
|
|
}
|
|
|
|
// carried is a found tunnel as it is presented to the mesh: everything but its private key.
|
|
func carried(t tunnel.Found) *link.Tunnel {
|
|
out := &link.Tunnel{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port,
|
|
Address: t.Address, Range: t.Range, MTU: t.MTU, PublicKey: t.PublicKey}
|
|
for _, p := range t.Peers {
|
|
out.Peers = append(out.Peers, link.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func firstNonEmpty(values ...string) string {
|
|
for _, v := range values {
|
|
if strings.TrimSpace(v) != "" {
|
|
return v
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func firstNonEmpty2(values ...[]byte) []byte {
|
|
for _, v := range values {
|
|
if len(v) > 0 {
|
|
return v
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// runLink holds this node's link to the mesh open, applying what arrives.
|
|
//
|
|
// One outbound connection and nothing listening. While it is up this node is enrolled; while it
|
|
// is down it is disconnected, which is an ordinary situation rather than a failure — the machine
|
|
// keeps running whatever it was last told, from its own store.
|
|
func runLink(ctx context.Context, opts options) error {
|
|
// A machine that has not enrolled waits here rather than failing. It is *hosted*: the host is
|
|
// running, it has no identity, and there is nobody to link to — an ordinary state, and the
|
|
// one every machine passes through (novox/hq 09-the-node-lifecycle).
|
|
//
|
|
// Exiting instead would be worse than untidy. The launcher counts a failed start, and three
|
|
// of them roll the binary back — so a freshly installed host, waiting to be enrolled exactly
|
|
// as intended, would undo its own installation.
|
|
mine, err := waitForEnrolment(ctx, opts)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if mine.Node == "" {
|
|
return nil // asked to stop while waiting
|
|
}
|
|
|
|
// **A membership delivered while this host was not running is adopted before the first dial.**
|
|
// The ordinary path is a declaration, read after it applies; the rescue path is an operator
|
|
// writing the file by hand on a machine no bus can reach — rotated while it held the old
|
|
// password, say — and restarting the host (design 28, task 5.2). Same file, same check.
|
|
say := func(line string) { fmt.Println(line) }
|
|
adoptDeliveredMembership(identity.Path(opts.state), &mine, say)
|
|
// So every statement about a directory used as found names the operator's line for this node.
|
|
foundNode.Store(mine.Node)
|
|
|
|
fmt.Printf("node %s, linking to %s\n", mine.Node, mine.Membership.Broker)
|
|
|
|
// One scheduler for the life of the process, re-established from each applied declaration
|
|
// (novox/hq ADR 0053). It fires scheduled steps on their cadence, surviving across applies and
|
|
// across the reconcile loop; a host restart rebuilds it from the declaration the node kept, the
|
|
// first time either path applies. Its own loop is the thing on the clock — no system timer.
|
|
sched := apply.NewScheduler(apply.SystemClock(), apply.ExecRunner, say)
|
|
// A window it opens is written beside the node's state, where every apply on this machine —
|
|
// this process's, a `reconcile` run by hand, the installer's — reads it (novox/hq issue 224).
|
|
sched.RecordWindowsIn(apply.WindowsIn(filepath.Dir(opts.state)))
|
|
go sched.Run(ctx)
|
|
// The take act's walks that stopped at their bound go on beside the applies (novox/hq ADR 0328 §7).
|
|
startKeeping(ctx, func(line string) { say(line) })
|
|
|
|
// The one liveness judge on this machine (novox/hq ADR 0240), made before anything applies: every
|
|
// report this host makes carries what it says.
|
|
if j, err := liveness.Open(filepath.Join(filepath.Dir(opts.state), liveness.FileName),
|
|
&liveness.Exec{Run: apply.ExecRunner, System: builtFor}); j != nil {
|
|
if err != nil {
|
|
say(err.Error())
|
|
}
|
|
windows := apply.WindowsIn(filepath.Dir(opts.state))
|
|
j.HeldNow = func(now time.Time) map[string]bool {
|
|
held := map[string]bool{}
|
|
for _, w := range windows.Open(now) {
|
|
for _, name := range w.Holds {
|
|
held[name] = true
|
|
}
|
|
}
|
|
return held
|
|
}
|
|
judging = j
|
|
// And which units its service managers say failed, and whose each is (novox/hq issue 315).
|
|
unitJudge = units.New(units.Exec{Run: apply.ExecRunner, System: builtFor})
|
|
// And whether an account a module put in a group has it where it runs (novox/hq ADR 0252).
|
|
// Its search for setuid programs runs to its end on a thread at idle priority, keeping how far its walk
|
|
// got so a restart carries it on, and the last complete one beside this state, so a restart judges from
|
|
// it (novox/hq issue 361).
|
|
setuidSearch = &accounts.SetuidCache{Path: filepath.Join(filepath.Dir(opts.state), accounts.SetuidSearchFile),
|
|
ProgressPath: filepath.Join(filepath.Dir(opts.state), accounts.SetuidProgressFile),
|
|
Lower: accounts.LowerThisThread, Say: say}
|
|
accountJudge = accounts.New(accounts.Exec{Run: accounts.CLocale, SearchRun: accounts.Idle, Cache: setuidSearch})
|
|
}
|
|
|
|
// **Standing aside for a successor happens between reconciles and nowhere else** (novox/hq ADR
|
|
// 0141). A host that stood aside mid-apply is the half-configured machine this host exists to
|
|
// prevent, so the question is asked after an apply has finished and the answer is a clean exit —
|
|
// which the launcher already reads as "run whatever is on disk now".
|
|
aside, standAside := context.WithCancel(ctx)
|
|
defer standAside()
|
|
stoodAside := false
|
|
membership := link.Membership{
|
|
Node: mine.Node,
|
|
Broker: mine.Membership.Broker,
|
|
Fingerprint: mine.Membership.Fingerprint,
|
|
Password: mine.Membership.Password,
|
|
Transport: mine.Membership.Transport,
|
|
Signer: mine.Membership.Signer,
|
|
}
|
|
|
|
// What follows an apply of a module (the node agent's, novox/hq ADR 0539): a membership it delivered for another bus
|
|
// is adopted, and a newer node-engine it placed makes this one stand aside.
|
|
afterApply := func() {
|
|
// **A declaration may carry this machine's membership for another bus.** It arrives as a
|
|
// sealed file like any secret, and is read after the rest has applied so the bus it names is
|
|
// standing before this machine leaves the one it is on (novox/hq design 28, task 5.2).
|
|
adoptDeliveredMembership(identity.Path(opts.state), &mine, say)
|
|
|
|
// Asked with the version this host is RUNNING, read from where it sits — not the link-time
|
|
// stamp, which every delivered host carries as "development build". Asked with the stamp,
|
|
// a delivered host never matched the newest delivered version, so it stood aside on every
|
|
// push for ever, and standing aside then cancelled the report, so the mesh never heard from it
|
|
// again (novox/hq 04-ISSUES/163).
|
|
switch next, waiting, err := upgrade.Successor(upgrade.VersionsDir(""), runningVersion(), rolledBackHosts(opts.state)...); {
|
|
case err != nil:
|
|
// Said, not fatal. A host that cannot read the delivered versions is still running this
|
|
// machine correctly; what it has lost is the ability to be replaced.
|
|
say(fmt.Sprintf("cannot tell whether a newer host is delivered: %v", err))
|
|
case waiting:
|
|
say(fmt.Sprintf("host %s is delivered; standing aside so the launcher runs it", next.Version))
|
|
stoodAside = true
|
|
standAside()
|
|
}
|
|
}
|
|
|
|
// The node follows its wants; it is never sent a declaration (novox/hq ADR 0539 §9). What the session does is hold
|
|
// the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the
|
|
// controller asks.
|
|
proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
|
|
queue := &link.Session{
|
|
Membership: membership,
|
|
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
|
|
HaveWritten: proof.haveWritten,
|
|
// And the operator's hand-over of a directory used as found, carried by the controller from the
|
|
// terminal (novox/hq issue 356): judged and recorded here, applied by the next apply.
|
|
HandOver: handOverFrom(opts.state),
|
|
// A fresh search for setuid programs, asked at the controller's terminal (novox/hq issue 361).
|
|
SetuidSearch: func(by string) (string, error) {
|
|
if setuidSearch == nil {
|
|
return "", errors.New("this engine judges no account for root, so it keeps no search for setuid programs")
|
|
}
|
|
return setuidSearch.SearchNow(by), nil
|
|
},
|
|
// A read-only judgement of the directories this node is declared, asked by the controller (novox/hq
|
|
// issue 358): which the walk that follows no link another account could make would refuse.
|
|
JudgeDirectories: judgeDirectoriesFrom(opts.state),
|
|
// The take act on a library, on the operator's warrant (novox/hq ADR 0328 §7).
|
|
AccessAct: accessActFrom(opts.state, opts.timeout),
|
|
Say: say,
|
|
Timeout: opts.timeout,
|
|
}
|
|
current := &agentNow{}
|
|
setRunsHere(runsFromAgent(current))
|
|
factsOfNow := func(ctx context.Context, ag *agent.Agent) {
|
|
sayFacts(ctx, opts, mine.Membership.Signer, queue, say, ag)
|
|
}
|
|
queue.OnLink = func(ctx context.Context) {
|
|
factsOfNow(ctx, current.get())
|
|
afterApply()
|
|
}
|
|
// And the move of a value sealed to this machine into the vault, asked by the controller (design 57's V3,
|
|
// issue 676): the vault is asked over this queue's link, so the queue is named once it exists.
|
|
queue.MoveSecret = moveSecretFrom(opts.state, queue)
|
|
// And this machine's sealing key stated to the vault in its own name, at start and whenever it changes (ADR 0372,
|
|
// issue 786), whether or not any value was ever moved in.
|
|
go keepNodeKeyStated(ctx, func() (identity.SealingKey, error) {
|
|
return identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
|
}, func(verb string, args any) (json.RawMessage, error) {
|
|
return queue.AskVault(ctx, verb, args)
|
|
}, say, nodeKeyRetryFrom, nodeKeyRetryUpTo)
|
|
// And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq
|
|
// ADR 0240) — between applies, which is when a container crash-loops.
|
|
if judging != nil {
|
|
// The looks it makes itself — http, tcp, a module's own tool — each at its declared interval,
|
|
// spaced to the budget (ADR 0240 Phase B); a tool is asked of this machine's node tools over the
|
|
// link open at the time.
|
|
judging.Probes = &liveness.Probes{AskTool: queue.AskTool}
|
|
// And the machine's own networking (novox/hq ADR 0241): the resolver file the uplink holder
|
|
// declared, the names through it, the tunnel, the bus and the route — said in the same statement.
|
|
netJudge.set(network.New(network.Machine{Run: apply.ExecRunner, Linked: queue.Linked},
|
|
hostOf(mine.Membership.Broker)))
|
|
go judging.Probe(aside)
|
|
go judgeWhatRuns(aside, judging, queue, say, func(modules ...string) {
|
|
if ag := current.get(); ag != nil {
|
|
ag.Retry(modules...)
|
|
}
|
|
})
|
|
}
|
|
// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
|
|
// busy containers are asked for again soon (ADR 0360 §11).
|
|
go nodeAgent(aside, opts, mine.Membership.Signer, queue, sched, say, afterApply, current, factsOfNow)
|
|
go busyPasses(aside, func() {
|
|
if ag := current.get(); ag != nil {
|
|
ag.Retry(busyModules()...)
|
|
}
|
|
}, BusyAskEvery)
|
|
// And the core builds this host placed are judged, whichever host placed them (to-be 45 §8).
|
|
go watchWhatThisHostPlaced(aside, mine.Node, queue, say)
|
|
// And mesh-cli is answered on this machine: the account from the kernel, the line asked of the controller
|
|
// (novox/hq ADR 0272).
|
|
go serveMeshCLI(aside, queue, say)
|
|
// And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR
|
|
// 0349 §4).
|
|
go serveFilling(aside, opts, mine.Membership.Signer, say)
|
|
|
|
held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
|
|
// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
|
|
// before it has run once. The context this returns on was cancelled deliberately, so its error
|
|
// is not a fault to report.
|
|
if stoodAside {
|
|
return nil
|
|
}
|
|
return held
|
|
}
|
|
|
|
// rousedBySignal is the machine telling this process that its link is probably stale.
|
|
//
|
|
// **A signal, because nothing may listen on a node** (novox/hq ADR 0004). A socket for this would
|
|
// be a control surface on every machine, reachable by anything that can reach the machine, in
|
|
// exchange for saving twenty seconds — and the whole security argument rests on there not being
|
|
// one. A signal is delivered by the service manager to a process it already supervises.
|
|
// (mesh-cli's socket is no such surface: it has no network address and nothing sent to it acts on
|
|
// the machine — it is asked of the controller, novox/hq ADR 0272 §3.)
|
|
//
|
|
// SIGHUP, because that is the signal a long-running program conventionally reads as *look again*,
|
|
// and nothing here is being reloaded from a file that a different signal would suit better.
|
|
//
|
|
// Dropped rather than queued when one arrives while another is unread: two wakes in the same
|
|
// instant are one wake, and a machine that suspends and resumes repeatedly must not build a
|
|
// backlog of reconnections to work through.
|
|
func rousedBySignal(ctx context.Context) link.Roused {
|
|
woken := make(chan os.Signal, 1)
|
|
signal.Notify(woken, syscall.SIGHUP)
|
|
|
|
out := make(chan struct{}, 1)
|
|
go func() {
|
|
defer signal.Stop(woken)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-woken:
|
|
select {
|
|
case out <- struct{}{}:
|
|
default:
|
|
// One is already waiting to be read. Two wakes in the same instant are one.
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
return out
|
|
}
|
|
|
|
// judging is the serving host's liveness judge (novox/hq ADR 0240); nil in a one-shot command, which
|
|
// reports no health, and in a test.
|
|
var judging *liveness.Judge
|
|
|
|
// unitJudge reads which units the machine's service managers say failed, and whose each is (novox/hq
|
|
// issue 315); nil where judging is.
|
|
var unitJudge *units.Judge
|
|
|
|
// accountJudge reads whether an account a module put in a group has it in its running session (novox/hq
|
|
// ADR 0252); nil where judging is.
|
|
var accountJudge *accounts.Judge
|
|
|
|
// setuidSearch is accountJudge's search for setuid programs, asked to search again after an apply that moved
|
|
// what root controls (novox/hq issue 361); nil where accountJudge is.
|
|
var setuidSearch *accounts.SetuidCache
|
|
|
|
// netJudge is the serving host's judge of its machine's networking (novox/hq ADR 0241); empty in a
|
|
// one-shot command and in a test, which say nothing of the network.
|
|
var netJudge networkJudge
|
|
|
|
type networkJudge struct {
|
|
mu sync.Mutex
|
|
m *network.Judge
|
|
// The resolver file the last apply declared, kept for a judge made after it.
|
|
content, owner string
|
|
declared bool
|
|
}
|
|
|
|
func (n *networkJudge) get() *network.Judge {
|
|
n.mu.Lock()
|
|
defer n.mu.Unlock()
|
|
return n.m
|
|
}
|
|
|
|
func (n *networkJudge) set(m *network.Judge) {
|
|
n.mu.Lock()
|
|
defer n.mu.Unlock()
|
|
n.m = m
|
|
m.Declare(n.content, n.owner, n.declared)
|
|
}
|
|
|
|
// declare is the resolver file an apply just applied, for the judge now and any made later.
|
|
func (n *networkJudge) declare(content, owner string, ok bool) {
|
|
n.mu.Lock()
|
|
defer n.mu.Unlock()
|
|
n.content, n.owner, n.declared = content, owner, ok
|
|
if n.m != nil {
|
|
n.m.Declare(content, owner, ok)
|
|
}
|
|
}
|
|
|
|
// hostOf is the bus's name without its port: the mesh name the machine needs most. Empty when the bus
|
|
// is reached by address, and then no mesh name is asked.
|
|
func hostOf(broker string) string {
|
|
host := broker
|
|
if h, _, err := net.SplitHostPort(broker); err == nil {
|
|
host = h
|
|
}
|
|
if net.ParseIP(host) != nil {
|
|
return ""
|
|
}
|
|
return host
|
|
}
|
|
|
|
// declaredResolvConf is the resolver file a declaration has a module write whole — the uplink holder's
|
|
// (ADR 0223) — and that module.
|
|
func declaredResolvConf(d *declaration.Declaration) (string, string, bool) {
|
|
if d == nil {
|
|
return "", "", false
|
|
}
|
|
for _, r := range d.Resources {
|
|
f, ok := r.(*declaration.File)
|
|
if !ok || f.Path != network.ResolvConf || f.Seeded() || f.Into != "" {
|
|
continue
|
|
}
|
|
module, ok := d.ModuleOfID(f.ID)
|
|
if !ok {
|
|
return "", "", false
|
|
}
|
|
return f.Content, module, true
|
|
}
|
|
return "", "", false
|
|
}
|
|
|
|
// How often a statement of health is said between reports: again every minute while anything is not
|
|
// healthy (to-be 48 §4), so a lost event is not a lost fault; and every five minutes anyway, so a
|
|
// controller that restarted knows a healthy machine's state without waiting for its next apply.
|
|
const (
|
|
sayUnhealthyAgain = time.Minute
|
|
sayAnyway = 5 * time.Minute
|
|
)
|
|
|
|
// judgeWhatRuns looks at every long-running resource every liveness.LookEvery and says the statement on
|
|
// the bus when a state changed, again every minute while one is not healthy, and every five minutes
|
|
// anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
|
|
// rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
|
|
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply func(modules ...string)) {
|
|
var missingBefore map[string]bool
|
|
ticker := time.NewTicker(liveness.LookEvery)
|
|
defer ticker.Stop()
|
|
var lastSaid time.Time
|
|
owed := true
|
|
spaced := 1.0
|
|
// A module whose change ended unjudged is looked at and said at once (review of fix 15): a module with nothing
|
|
// long-running is then ok within a look of its apply, not at the five-minute statement.
|
|
var nudged <-chan struct{}
|
|
if queue != nil {
|
|
nudged = queue.HealthOwed()
|
|
}
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
case <-nudged:
|
|
owed = true
|
|
}
|
|
st, changed := j.Look(ctx)
|
|
owed = owed || changed
|
|
var unitSt *units.Statement
|
|
if u := unitJudge; u != nil {
|
|
us, unitsChanged := u.Look(ctx)
|
|
unitSt = &us
|
|
owed = owed || unitsChanged
|
|
if unitsChanged {
|
|
for _, f := range us.Failed {
|
|
say(failedUnitWords(f))
|
|
}
|
|
}
|
|
}
|
|
var accountSt *accounts.Statement
|
|
if a := accountJudge; a != nil {
|
|
as, accountsChanged := a.Look(ctx)
|
|
accountSt = &as
|
|
owed = owed || accountsChanged
|
|
if accountsChanged {
|
|
for _, v := range as.Accounts {
|
|
if v.State != accounts.Healthy {
|
|
say(fmt.Sprintf("%s (account %s) is %s: %s", v.ID, v.Name, v.State, v.Reason))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Every library, judged on every look (novox/hq ADR 0328 §8, §9); and a take act stopped at its bound
|
|
// goes on, beside.
|
|
accessSt, accessesChanged := accesses.look(time.Now())
|
|
keeping.poke(ctx)
|
|
owed = owed || accessesChanged
|
|
if accessesChanged {
|
|
for _, v := range accessSt {
|
|
if v.State != link.StateHealthy {
|
|
say(fmt.Sprintf("%s (access %s) is %s: %s", v.Resource, v.Target, v.State, v.Reason))
|
|
}
|
|
}
|
|
}
|
|
var netSt *network.Statement
|
|
if n := netJudge.get(); n != nil {
|
|
ns, netChanged := n.Look(ctx)
|
|
netSt = &ns
|
|
owed = owed || netChanged
|
|
if netChanged {
|
|
for _, p := range ns.Parts {
|
|
if p.State == network.Unhealthy {
|
|
say(fmt.Sprintf("this machine's network is unhealthy: %s: %s (%s)", p.Part, p.Reason, p.Said))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Never more looks than the budget (ADR 0240): said when the engine has to space its own out.
|
|
if sp := j.Spacing(); sp != spaced {
|
|
if sp > 1 {
|
|
say(fmt.Sprintf("the declared health checks here would cost more than %d looks a minute; the engine's "+
|
|
"own looks are spaced %.1f times their declared interval", liveness.Budget, sp))
|
|
}
|
|
spaced = sp
|
|
}
|
|
// Every directory used as found, read from the node's state on every look (novox/hq issue 339).
|
|
found := foundNow()
|
|
since := time.Since(lastSaid)
|
|
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
|
|
(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 &&
|
|
accessesHealthy(accessSt)
|
|
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
|
|
continue
|
|
}
|
|
if changed {
|
|
for _, r := range st.Resources {
|
|
if r.State == liveness.Unhealthy {
|
|
say(unhealthyWords(r))
|
|
}
|
|
}
|
|
// A resource the machine does not have at all is made again: its module's composition applied anew (issue
|
|
// 1945), once each time the statement changes, never on a clock.
|
|
var m []string
|
|
m, missingBefore = newlyMissing(missingBefore, st)
|
|
if len(m) > 0 && reapply != nil {
|
|
say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", ")))
|
|
reapply(m...)
|
|
}
|
|
}
|
|
if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
|
|
unitSt), accountSt), found), accessSt)) {
|
|
lastSaid, owed = time.Now(), false
|
|
}
|
|
}
|
|
}
|
|
|
|
// healthAsReported is a statement as the report and the event carry it.
|
|
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
|
|
// LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
|
|
// user's declared `root` (novox/hq ADR 0266), and reads a declaration's identities, groups and libraries
|
|
// (novox/hq ADR 0328), a container's busy (ADR 0335) and a file's owner-at-runtime and the filled files (ADR
|
|
// 0349 §5) and the provisioner client's account (ADR 0349 §4), which is what tells the controller it may be sent
|
|
// any of those fields; and it writes a filled file only on the client's word (HandedContract, design 57 P7),
|
|
// which is what tells it it may hand a moved module's files to the provisioner here; and it opens the vault's box
|
|
// for a core credential (VaultBoxContract, ADR 0349 §16), which is what tells it it may send that box here; and it states its sealing key to the vault (NodeKeyContract, ADR
|
|
// 0372, issue 786).
|
|
h := &link.Health{Contract: link.NodeKeyContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
|
|
for _, r := range st.Resources {
|
|
h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind,
|
|
Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak,
|
|
Restarts: r.Restarts, Check: r.CheckOf(), Needs: r.NeedsOf(), Account: userOf(r.Scope, r.User),
|
|
Waits: r.Waits})
|
|
}
|
|
if ns != nil && ns.State != "" {
|
|
h.Network = &link.NetworkHealth{State: ns.State, Since: ns.Since.UTC(), Parts: []link.NetworkPart{}}
|
|
for _, p := range ns.Parts {
|
|
h.Network.Parts = append(h.Network.Parts, link.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
|
|
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since.UTC(), Streak: p.Streak})
|
|
}
|
|
}
|
|
return h
|
|
}
|
|
|
|
// withUnits adds to a statement the machine's failed units (novox/hq issue 315): each a module places,
|
|
// among the resources as that module's unhealthy unit; the rest, the machine's own, beside them. Nil — an
|
|
// engine not reading them — says nothing of them, which the controller reads as not known.
|
|
func withUnits(h *link.Health, us *units.Statement) *link.Health {
|
|
if us == nil || us.State == "" {
|
|
return h
|
|
}
|
|
h.Units = &link.UnitsHealth{State: us.State, Failed: []link.FailedUnit{}, Unread: us.Unread,
|
|
Away: us.Away}
|
|
for _, f := range us.Failed {
|
|
if f.Module == "" {
|
|
h.Units.Failed = append(h.Units.Failed, link.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load,
|
|
Result: f.Result, Resource: f.Resource, Since: f.Since.UTC()})
|
|
continue
|
|
}
|
|
reason := "failed"
|
|
if f.Scope == units.ScopeUser {
|
|
reason += " in the account's own service manager"
|
|
}
|
|
if f.Result != "" {
|
|
reason += " (" + f.Result + ")"
|
|
}
|
|
h.Resources = append(h.Resources, link.ResourceHealth{Module: f.Module, Resource: f.Resource,
|
|
Kind: link.KindUnit, Target: f.Unit, State: link.StateUnhealthy, Reason: reason, Since: f.Since.UTC(),
|
|
Streak: f.Streak, Account: userOf(f.Scope, f.User)})
|
|
}
|
|
return h
|
|
}
|
|
|
|
// withAccounts adds to a statement every account a module put in a group (novox/hq ADR 0252), as that
|
|
// module's resource of kind account: healthy, or unhealthy with why — "relogin needed" when the account's
|
|
// running session began before it was put in the group, or "can become root without a person" for one declared
|
|
// never to (novox/hq ADR 0266), which is also marked root never. Nil says nothing of them.
|
|
func withAccounts(h *link.Health, as *accounts.Statement) *link.Health {
|
|
if as == nil {
|
|
return h
|
|
}
|
|
for _, v := range as.Accounts {
|
|
rh := link.ResourceHealth{Module: v.Module, Resource: v.ID,
|
|
Kind: link.KindAccount, Target: v.Name, State: v.State, Reason: v.Reason, Since: v.Since.UTC(),
|
|
Streak: v.Streak, Account: v.Name}
|
|
// Said, so the controller knows healthy here also means no way to root was found (novox/hq ADR 0266).
|
|
if v.Root {
|
|
rh.Root = declaration.RootNever
|
|
}
|
|
// How far the setuid search got, while the verdict waits for it (novox/hq issue 361): in the reason the
|
|
// root-free verb repeats, and as fields. Never in the journal's line, which says a change alone.
|
|
if v.Search != nil {
|
|
rh.Reason += "; " + v.Search.Words()
|
|
rh.Search = searchAsReported(v.Search)
|
|
}
|
|
h.Resources = append(h.Resources, rh)
|
|
}
|
|
return h
|
|
}
|
|
|
|
// searchAsReported is a search's progress as the statement carries it.
|
|
func searchAsReported(sp *accounts.SearchProgress) *link.SetuidSearch {
|
|
out := &link.SetuidSearch{Started: sp.Started.UTC(), Filesystems: sp.Filesystems, Done: sp.Done,
|
|
Walking: sp.Walking, Visited: sp.Visited, Found: sp.Found}
|
|
if !sp.Kept.IsZero() {
|
|
kept := sp.Kept.UTC()
|
|
out.Kept = &kept
|
|
}
|
|
for _, f := range sp.Finished {
|
|
sf := link.SearchedFilesystem{Mount: f.Mount}
|
|
if f.Took >= 0 {
|
|
secs := int64(f.Took.Round(time.Second) / time.Second)
|
|
sf.TookSeconds = &secs
|
|
}
|
|
out.Finished = append(out.Finished, sf)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// userOf is the account whose own service manager a unit is in (novox/hq ADR 0254), empty for the
|
|
// machine's own manager.
|
|
func userOf(scope, user string) string {
|
|
if scope == declaration.ScopeUser {
|
|
return user
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// failedUnitWords is a failed unit as the console says it.
|
|
func failedUnitWords(f units.Failed) string {
|
|
whose := "no module places it"
|
|
if f.Module != "" {
|
|
whose = fmt.Sprintf("%s's, by its %s %s", f.Module, f.Via, f.Resource)
|
|
}
|
|
return fmt.Sprintf("the unit %s (%s) failed%s: %s", f.Unit, f.Scope, orNothing(" ("+f.Result+")", f.Result), whose)
|
|
}
|
|
|
|
func orNothing(s, unless string) string {
|
|
if unless == "" {
|
|
return ""
|
|
}
|
|
return s
|
|
}
|
|
|
|
// busyWaiting is whether the last apply left a container running because it said busy (novox/hq ADR 0335).
|
|
var busyWaiting atomic.Bool
|
|
|
|
// busyBy is, per module, whether its last apply left a container running because it said busy; busyWaiting is any of them.
|
|
var (
|
|
busyMu sync.Mutex
|
|
busyBy = map[string]bool{}
|
|
)
|
|
|
|
func noteBusy(module string, waiting bool) {
|
|
busyMu.Lock()
|
|
defer busyMu.Unlock()
|
|
busyBy[module] = waiting
|
|
any := false
|
|
for _, w := range busyBy {
|
|
any = any || w
|
|
}
|
|
busyWaiting.Store(any)
|
|
}
|
|
|
|
// BusyAskEvery is how often a container waiting because it said busy is asked again (novox/hq ADR 0335 §2):
|
|
// each ask is a pass of its own, so a build that ends is followed by its replacement within this, not
|
|
// within the reconcile's five minutes — and the bound of 15 minutes is met within this of its end.
|
|
const BusyAskEvery = 20 * time.Second
|
|
|
|
// busyPasses asks for a pass every so often while a container waits because it said busy (ADR 0360 §11). **It asks; it
|
|
// does not apply**: the node agent does, when its turn comes, so a pass never holds the machine for the wait.
|
|
func busyPasses(ctx context.Context, ask func(), every time.Duration) {
|
|
ticker := time.NewTicker(every)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
if busyWaiting.Load() {
|
|
ask()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// announceOr is what the apply writes its detail with, given what the caller has to say things with.
|
|
//
|
|
// **Never nil.** This argument was nil on the serving path, and nil is silence: everything the apply
|
|
// says — a file held, a container replaced, the found firewall retired — was visible when a person ran
|
|
// the one-shot command and discarded in the way the host actually runs (novox/hq 04-ISSUES/143).
|
|
//
|
|
// Named rather than written inline at the call site so both paths reach the apply the same way, and so
|
|
// a reader asking "where does the apply's output go" finds one answer.
|
|
func announceOr(say link.Announce) func(string) {
|
|
if say == nil {
|
|
return func(string) {}
|
|
}
|
|
return say
|
|
}
|
|
|
|
// applying serialises applies within this process.
|
|
//
|
|
// **One thing applies here: the apply queue's worker** (novox/hq to-be 45 §6), for a delivery and for
|
|
// the reconcile alike. This lock was the only order between two that applied — the link and the
|
|
// reconcile loop — and each order it allowed was met as a fault (issues 257, 261, 267); the queue is
|
|
// the order now. It stays as the guard for what may be added beside the worker: two applies
|
|
// interleaved lose what one of them recorded — a hold, the firewall found here, a resource just
|
|
// applied — and the machine is then one thing and its record another. Across processes — `reconcile`
|
|
// run by hand beside this service — the lock beside the state does the same (store.Lock).
|
|
var applying sync.Mutex
|
|
|
|
// appliedModule is how the apply of one module's declaration came out.
|
|
type appliedModule struct {
|
|
// Refused is why nothing was applied, when nothing was.
|
|
Refused string
|
|
Outcome apply.Report
|
|
Updated store.State
|
|
// Err is the apply's own error, when it failed part way.
|
|
Err error
|
|
Probe *unanswered
|
|
}
|
|
|
|
// applyModuleHeld applies one module's declaration (novox/hq ADR 0539 §9): the resources of its composition, with the
|
|
// compositions the node's other modules run held out of any removal. A nil scope is a whole declaration, as the carried
|
|
// bundle is. One at a time, and the state is read and saved under the lock of the node's state.
|
|
func applyModuleHeld(ctx context.Context, opts options, declared *declaration.Declaration, scope *apply.Scope,
|
|
sched *apply.Scheduler, say link.Announce) appliedModule {
|
|
applying.Lock()
|
|
defer applying.Unlock()
|
|
unlock, err := store.Lock(opts.state, nil)
|
|
if err != nil {
|
|
return appliedModule{Refused: err.Error()}
|
|
}
|
|
defer unlock()
|
|
|
|
known, err := store.Load(opts.state)
|
|
if err != nil {
|
|
return appliedModule{Refused: err.Error()}
|
|
}
|
|
node := scope == nil || scope.Module == store.NodeModule
|
|
// The node's own composition is the controller's word on the node's mode — the flip arrives as exactly that, the
|
|
// first converged composition after adopted ones — and becomes the record (novox/hq issue 104).
|
|
if node && known.Mode != "" && known.Mode != apply.ModeOf(declared) {
|
|
announceOr(say)(fmt.Sprintf("this node's record said %s; what the mesh last said, signed, says %s — the record is repaired",
|
|
known.Mode, apply.ModeOf(declared)))
|
|
known.Mode = apply.ModeOf(declared)
|
|
}
|
|
|
|
// **A file given by digest is fetched and verified before anything of the composition is applied** (novox/hq ADR
|
|
// 0486 §1, issue 1695): one object missing or differing refuses the module whole.
|
|
if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil {
|
|
return appliedModule{Refused: "a file of this composition could not be had by its digest: " + err.Error()}
|
|
}
|
|
|
|
built, err := system.For(builtFor)
|
|
if err != nil {
|
|
return appliedModule{Refused: err.Error()}
|
|
}
|
|
if err := system.Check(built, declared); err != nil {
|
|
return appliedModule{Refused: err.Error()}
|
|
}
|
|
|
|
// A package manager that does not answer fails only what needs it, and the report says so (novox/hq issue 233).
|
|
probe, err := probePackageManager(ctx, built)
|
|
if err != nil {
|
|
return appliedModule{Refused: err.Error()}
|
|
}
|
|
if probe != nil {
|
|
announceOr(say)(probe.said())
|
|
}
|
|
|
|
// **What the apply says goes to the console, which is the journal when this runs as a service** (novox/hq
|
|
// 04-ISSUES/143). Once more when the package manager did not answer and this apply wrote something (issue 233).
|
|
outcome, updated, applyErr, probe, firstPass := applyMindingProbe(ctx, built, probe, known,
|
|
func(known store.State) (apply.Report, store.State, error) {
|
|
if scope == nil {
|
|
return apply.ApplyMindingWindows(ctx, built, declared, known, store.OriginDeclared,
|
|
apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)),
|
|
apply.WindowsIn(filepath.Dir(opts.state)))
|
|
}
|
|
return apply.ApplyModule(ctx, built, declared, *scope, known, store.OriginDeclared,
|
|
apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)),
|
|
apply.WindowsIn(filepath.Dir(opts.state)))
|
|
}, announceOr(say))
|
|
|
|
if node {
|
|
updated.Mode = apply.ModeOf(declared)
|
|
}
|
|
|
|
// Root acted on what an agent account's judgement reads — an account, a package, an action, sudo's rules: the search
|
|
// for setuid programs runs again (novox/hq issue 361). A failed apply counts what it did before it stopped.
|
|
moved := append(append([]apply.Outcome(nil), firstPass...), outcome.Outcomes...)
|
|
var partial *apply.Error
|
|
if errors.As(applyErr, &partial) {
|
|
moved = append(append([]apply.Outcome(nil), moved...), partial.Done.Outcomes...)
|
|
}
|
|
if apply.MovesRoot(moved) {
|
|
setuidSearch.SearchAgain()
|
|
}
|
|
|
|
// Saved whichever way it went: the footprint of a failed apply is on the machine either way.
|
|
if saveErr := store.Save(opts.state, updated); saveErr != nil {
|
|
return appliedModule{Refused: "applied, and the node's state could not be saved: " + saveErr.Error()}
|
|
}
|
|
|
|
// Re-establish the scheduled steps of this declaration (novox/hq ADR 0053): a schedule newly declared is armed, one
|
|
// whose image, environment or cadence changed is re-armed, one no longer declared is forgotten, and the other
|
|
// modules' are left as they are. A nil scheduler is a test.
|
|
held := map[string]bool{}
|
|
for _, h := range updated.Held {
|
|
held[h.ID] = true
|
|
}
|
|
if sched != nil {
|
|
if scope == nil {
|
|
sched.Sync(declared, held)
|
|
} else {
|
|
sched.SyncModule(declared, held, func(id string) bool {
|
|
m, ok := declaration.ModuleGuess(id)
|
|
return ok && m == scope.Module || (!ok && scope.Module == store.NodeModule)
|
|
})
|
|
}
|
|
}
|
|
name := ""
|
|
if scope != nil {
|
|
name = scope.Module
|
|
}
|
|
noteBusy(name, outcome.Waiting())
|
|
return appliedModule{Outcome: outcome, Updated: updated, Err: applyErr, Probe: probe}
|
|
}
|
|
|
|
// factsOf is what this node says of itself: its ports, its profile, how what runs here is, which links face outside,
|
|
// what it holds, its filters and, when adopted, its firewall and what is reachable. The facts are an event of the node
|
|
// to the controller (novox/hq ADR 0539 §9); declared is the declaration the node was last held to, updated its state,
|
|
// outcome what the last apply did (empty when nothing was applied), and held the ids it holds.
|
|
func factsOf(ctx context.Context, opts options, declared *declaration.Declaration, updated store.State,
|
|
outcome apply.Report, held map[string]bool) link.Report {
|
|
report := link.Report{Carried: carriedPorts(updated), Host: runningVersion(),
|
|
Rollbacks: standingRollbacks(opts.state), Witness: link.WitnessContract,
|
|
Profile: profileAsReported(profile.Detect(ctx, profile.Default(nil), opts.timeout))}
|
|
// What runs here for a module, and how each is (novox/hq ADR 0240): judged from what was just
|
|
// applied, so every resource this apply started says `starting` and the gate waits for it.
|
|
if j := judging; j != nil {
|
|
// A container that waits for a library whose disk or mount is not there was not created: the library says
|
|
// why, and liveness judges it neither alive nor dead (novox/hq ADR 0328 §9).
|
|
j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnLibrary(declared))))
|
|
st, _ := j.Look(ctx)
|
|
netJudge.declare(declaredResolvConf(declared))
|
|
var netSt *network.Statement
|
|
if n := netJudge.get(); n != nil {
|
|
ns := n.Last()
|
|
netSt = &ns
|
|
}
|
|
var unitSt *units.Statement
|
|
if u := unitJudge; u != nil {
|
|
u.Set(units.OwnedBy(declared, held))
|
|
us := u.Last()
|
|
unitSt = &us
|
|
}
|
|
// Looked at now, not taken from the last look: an apply that just put the account in a group is
|
|
// said with it, relogin needed included, in the report that says the apply.
|
|
var accountSt *accounts.Statement
|
|
if a := accountJudge; a != nil {
|
|
a.Set(accounts.Of(declared, held))
|
|
as, _ := a.Look(ctx)
|
|
accountSt = &as
|
|
}
|
|
accesses.set(declared, held)
|
|
// A take act stopped at its bound goes on, beside (ADR 0328 §7).
|
|
keeping.poke(ctx)
|
|
accessSt, _ := accesses.look(time.Now())
|
|
report.Health = withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt),
|
|
accountSt), foundDirectories(updated)), accessSt)
|
|
}
|
|
// Which of this machine's links face outside, for the filter the mesh writes around them
|
|
// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
|
|
// and an adopted one becomes converged without a further round trip. A machine that cannot read
|
|
// its own routing table says nothing rather than guessing, and is sent no filter.
|
|
if links, err := outward.Links("", ""); err != nil {
|
|
fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read which links face outside: %v\n", err)
|
|
} else {
|
|
report.Outward = links
|
|
}
|
|
// What this node found and holds, its firewall, and what is reachable on it — so an adopted
|
|
// node never reads as converged (novox/hq ADR 0100).
|
|
for _, h := range updated.Held {
|
|
report.Held = append(report.Held, link.Held{ID: h.ID, Module: h.Module, Kind: h.Kind,
|
|
Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept, Facts: factsAsReported(h.Facts)})
|
|
}
|
|
// And a maintenance window open as the apply ended (novox/hq issue 224): a server stopped because
|
|
// its collector is running reads as working, not broken.
|
|
for _, w := range outcome.Windows {
|
|
report.Windows = append(report.Windows, link.Window{Step: w.Step, Holds: w.Holds, Since: w.Opened, Until: w.Until})
|
|
}
|
|
// And a container left running because it said busy, or replaced after it waited (novox/hq ADR 0335):
|
|
// the walk waiting on this machine reads it. While one waits, this machine asks it again sooner than its
|
|
// reconcile (busyPasses).
|
|
for _, b := range outcome.Busy {
|
|
report.Busy = append(report.Busy, link.Busy{ID: b.ID, Container: b.Container, State: b.State,
|
|
Since: b.Since, Until: b.Until, Said: b.Said})
|
|
}
|
|
for _, m := range outcome.MovedOut {
|
|
report.MovedOut = append(report.MovedOut, link.MovedOut{Resource: m.Resource, From: m.From, To: m.To})
|
|
}
|
|
for _, l := range outcome.Left {
|
|
report.Left = append(report.Left, link.Left{Resource: l.Resource, Path: l.Path})
|
|
}
|
|
busyWaiting.Store(outcome.Waiting())
|
|
// And what runs here that nobody asked for (novox/hq ADR 0163).
|
|
if strays, err := apply.Strays(ctx, apply.ExecRunner, updated); err != nil {
|
|
fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not list what else runs here: %v\n", err)
|
|
} else {
|
|
for _, s := range strays {
|
|
report.Strays = append(report.Strays, link.Stray{Kind: s.Kind, Name: s.Name, Detail: s.Detail})
|
|
}
|
|
}
|
|
// What filters this machine, with owners, whatever its mode (novox/hq ADR 0168): the mesh says
|
|
// truthfully what filters a converged machine, and names what it did not write.
|
|
ufwActive := firewall.Active(ctx, apply.ExecRunner)
|
|
if filters, err := firewall.Collect(ctx, apply.ExecRunner, ufwActive); err != nil {
|
|
fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read what filters this machine: %v\n", err)
|
|
} else {
|
|
for _, f := range filters {
|
|
report.Filters = append(report.Filters, link.Filter{Where: f.Where, Owner: f.Owner, Refuses: f.Refuses})
|
|
}
|
|
}
|
|
if declared.Adoption == nil && updated.Firewall != nil && updated.Firewall.Kind == string(firewall.UFW) && updated.Firewall.WasActive {
|
|
// And, converged, the state of the firewall it was found with and who retired it.
|
|
report.FoundFirewall = &link.FoundFirewall{Kind: updated.Firewall.Kind, Active: ufwActive,
|
|
RetiredBy: updated.Firewall.RetiredBy}
|
|
}
|
|
if declared.Adoption != nil {
|
|
if updated.Firewall != nil {
|
|
report.Firewall = updated.Firewall.Kind
|
|
}
|
|
// And the tunnel the private network took over, as this apply found it (novox/hq ADR 0105).
|
|
if t := outcome.Tunnel; t != nil {
|
|
report.Tunnel = &link.CarriedTunnel{Interface: t.Interface, Port: t.Port, Range: t.Range,
|
|
Peers: t.Peers, State: t.State, Note: t.Note, Kept: t.Kept}
|
|
}
|
|
reached, err := reachable.Collect(ctx, apply.ExecRunner)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read what is reachable here: %v\n", err)
|
|
}
|
|
report.Reachable = reached
|
|
}
|
|
for _, change := range outcome.Outcomes {
|
|
// What is held is not what this machine owns: it was found, and is kept as it was until
|
|
// its module is taken (novox/hq ADR 0100).
|
|
if change.Action == "held" {
|
|
continue
|
|
}
|
|
// Nor is a unit waiting for its account's manager that this machine never applied (novox/hq
|
|
// ADR 0177); one applied before and waiting now is still the machine's, recorded as it was.
|
|
if _, recorded := updated.Find(change.ID); change.Action == "waiting" && !recorded {
|
|
continue
|
|
}
|
|
report.Applied = append(report.Applied, change.ID)
|
|
}
|
|
return report
|
|
}
|
|
|
|
// sayFacts says this node's facts on the link open now (novox/hq ADR 0539 §9): what the node is made of, how what runs
|
|
// here is, and its filters and ports. What runs is the compositions the node agent holds running (ag), or, before it
|
|
// has run, the declaration the node kept before the switch. It applies nothing; the haves are the agent's.
|
|
func sayFacts(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, say link.Announce,
|
|
ag *agent.Agent) {
|
|
known, err := store.Load(opts.state)
|
|
if err != nil {
|
|
say("cannot say this node's facts: " + err.Error())
|
|
return
|
|
}
|
|
declared := &declaration.Declaration{}
|
|
if d, err := declaredNow(opts.state, signer); err == nil {
|
|
declared = d
|
|
}
|
|
held := map[string]bool{}
|
|
for _, h := range known.Held {
|
|
held[h.ID] = true
|
|
}
|
|
report := factsOf(ctx, opts, declared, known, apply.Report{}, held)
|
|
session.SayFacts(ctx, report)
|
|
}
|
|
|
|
// waitForEnrolment returns this node's identity, waiting for one if it has none.
|
|
//
|
|
// It applies the carried bundle first, if there is one, because that is what a first node does
|
|
// before there is a mesh at all — and a machine that has been installed and not yet enrolled
|
|
// should still be whatever its bundle says it is.
|
|
func waitForEnrolment(ctx context.Context, opts options) (identity.Identity, error) {
|
|
const look = 5 * time.Second
|
|
|
|
said := false
|
|
for {
|
|
mine, err := identity.Load(identity.Path(opts.state))
|
|
if err == nil {
|
|
return mine, nil
|
|
}
|
|
if !errors.Is(err, identity.ErrNoIdentity) {
|
|
// An identity that exists and cannot be read is a fault, not a wait. Treating it as
|
|
// "not enrolled yet" would leave a node sitting quietly for ever while the mesh
|
|
// believes it is a member.
|
|
return identity.Identity{}, err
|
|
}
|
|
|
|
if !said {
|
|
fmt.Println("this machine has not joined a mesh, and is waiting to be told which one.")
|
|
fmt.Println(" enrol it with: mesh-host enrol --token <token> --name <name>")
|
|
said = true
|
|
}
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return identity.Identity{}, nil
|
|
case <-time.After(look):
|
|
}
|
|
}
|
|
}
|
|
|
|
// sealOpener is how a sealed file is opened.
|
|
//
|
|
// Looked up per file rather than held, because most declarations contain no sealed file at all
|
|
// and a node with no key must fail on the one that needs it rather than on every apply.
|
|
func sealOpener(statePath string) apply.Unseal {
|
|
return func(sealed string) ([]byte, error) {
|
|
key, err := identity.LoadSealingKey(identity.SealingKeyPath(statePath))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return key.Open(sealed)
|
|
}
|
|
}
|
|
|
|
// carriedPorts is every machine port held by what this host raised from its own bundle.
|
|
//
|
|
// **What the mesh must assign around** (novox/hq ADR 0038). The foundation is not a module: a node
|
|
// raises it before any mesh exists, so the control plane has never heard of the store or the
|
|
// broker. Told this, it can put a module somewhere else; not told, it hands out a port one of them
|
|
// holds and finds out from a container runtime.
|
|
//
|
|
// Only what was carried. What the mesh itself put here it already knows about, and reporting it
|
|
// back would make the machine an authority on the mesh's own bookkeeping.
|
|
// digestOf names a declaration by its bytes, exactly as the mesh names what it sends. The two
|
|
// sides never exchange the digest of different things: this hashes the same raw bytes the mesh
|
|
// hashed when it recorded the send.
|
|
func digestOf(body []byte) string {
|
|
sum := sha256.Sum256(body)
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func carriedPorts(state store.State) []int {
|
|
seen := map[int]bool{}
|
|
var out []int
|
|
for _, applied := range state.Resources {
|
|
if applied.Origin == store.OriginDeclared {
|
|
continue
|
|
}
|
|
for _, port := range applied.Holds {
|
|
if !seen[port] {
|
|
seen[port] = true
|
|
out = append(out, port)
|
|
}
|
|
}
|
|
}
|
|
sort.Ints(out)
|
|
return out
|
|
}
|
|
|
|
// MembershipNextPath is where the mesh delivers this machine's membership for the bus it is moving
|
|
// to: a sealed file in a declaration, written by the host like any secret, read here after the
|
|
// declaration has applied.
|
|
const MembershipNextPath = "/var/lib/mesh/membership-next.json"
|
|
|
|
// adoptDeliveredMembership moves this machine to the bus a delivered membership names.
|
|
//
|
|
// **Saved, then restarted — not swapped in place.** The link holds one membership for the life of
|
|
// the process, and every reconnect path assumes the bus did not change under it; a process that
|
|
// found itself half on one bus and half on another would be a new kind of state nothing was written
|
|
// for. Exiting cleanly hands the machine to the service manager's restart, and the process that
|
|
// comes back reads the identity file the way it always has and dials the bus it names. That is the
|
|
// same path a machine takes after a reboot, which is why nothing new has to be right for it to work.
|
|
//
|
|
// A membership identical to the one held is nothing: the file stays and is read again next time.
|
|
func adoptDeliveredMembership(identityPath string, mine *identity.Identity, say func(string)) {
|
|
raw, err := os.ReadFile(MembershipNextPath)
|
|
if err != nil {
|
|
return
|
|
}
|
|
var next identity.Membership
|
|
if err := json.Unmarshal(raw, &next); err != nil {
|
|
say(fmt.Sprintf("a membership was delivered at %s and could not be read: %v", MembershipNextPath, err))
|
|
return
|
|
}
|
|
if next.Broker == "" || next.Password == "" || next.Fingerprint == "" {
|
|
say(fmt.Sprintf("a membership was delivered at %s with no broker, fingerprint or password; ignored", MembershipNextPath))
|
|
return
|
|
}
|
|
same := next.Broker == mine.Membership.Broker && next.Fingerprint == mine.Membership.Fingerprint &&
|
|
next.Password == mine.Membership.Password && next.Transport == mine.Membership.Transport
|
|
if same {
|
|
return
|
|
}
|
|
// The signer is the mesh's, not the bus's: a delivered membership that names none keeps the one
|
|
// this machine already trusts, because a change of bus is not a change of who signs declarations.
|
|
if len(next.Signer) == 0 {
|
|
next.Signer = mine.Membership.Signer
|
|
}
|
|
mine.Membership = next
|
|
if err := identity.Save(identityPath, *mine); err != nil {
|
|
say(fmt.Sprintf("a membership for another bus was delivered and could not be saved: %v", err))
|
|
return
|
|
}
|
|
say(fmt.Sprintf("moving to the %s bus at %s — restarting to dial it", next.Transport, next.Broker))
|
|
os.Exit(0)
|
|
}
|
|
|
|
// profileAsReported is the profile as the mesh reads it — the same bytes enrolment sends, so a
|
|
// report's profile and an enrolment's are one shape on the controller's side (novox/hq ADR 0161).
|
|
func profileAsReported(detected profile.Profile) map[string]any {
|
|
reported := map[string]any{}
|
|
if raw, err := json.Marshal(detected); err == nil {
|
|
_ = json.Unmarshal(raw, &reported)
|
|
}
|
|
return reported
|
|
}
|
|
|
|
// factsAsReported is a held thing's facts as the mesh reads them: the same bytes the host keeps.
|
|
func factsAsReported(f *store.Facts) map[string]any {
|
|
if f == nil {
|
|
return nil
|
|
}
|
|
out := map[string]any{}
|
|
if raw, err := json.Marshal(f); err == nil {
|
|
_ = json.Unmarshal(raw, &out)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// unhealthyWords is what the engine says of an unhealthy resource. Its restart count is the judge's own
|
|
// (kept.Counted): automatic restarts since it began watching the resource, kept across recreates and starts
|
|
// by hand, so it is said as that and never read as since the state began (novox/hq issue 458).
|
|
func unhealthyWords(r liveness.State) string {
|
|
return fmt.Sprintf("%s (%s %s) is unhealthy: %s, %d automatic restart(s) since this engine began watching it",
|
|
r.ID, r.Kind, r.Target, r.Reason, r.Restarts)
|
|
}
|
|
|
|
// moduleFacts is how each module of a declaration came out of an apply, with the build the controller sent for it
|
|
// (novox/hq ADR 0429 §4, issue 1132): sorted by module so the facts do not move between applies. A module an adopted
|
|
// node has not taken is held as found and not applied, so it is no fact.
|
|
func moduleFacts(d *declaration.Declaration, outcome apply.Report, applyErr error) []link.ModuleFact {
|
|
if d == nil {
|
|
return nil
|
|
}
|
|
results := apply.ModuleResults(d, outcome, applyErr)
|
|
facts := make([]link.ModuleFact, 0, len(results))
|
|
for module, result := range results {
|
|
if _, untaken := untakenModules(d)[module]; untaken {
|
|
continue
|
|
}
|
|
facts = append(facts, link.ModuleFact{Module: module, Build: d.Builds[module], Failed: result.Failed, Detail: result.Detail})
|
|
}
|
|
sort.Slice(facts, func(i, j int) bool { return facts[i].Module < facts[j].Module })
|
|
return facts
|
|
}
|
|
|
|
func untakenModules(d *declaration.Declaration) map[string][]string {
|
|
if d.Adoption == nil {
|
|
return nil
|
|
}
|
|
return d.Adoption.Untaken
|
|
}
|
|
|
|
// missingModules are the modules of the resources a statement says missing, each once, sorted.
|
|
func missingModules(st liveness.Statement) []string {
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
for _, r := range st.Resources {
|
|
if r.Missing && r.Module != "" && !seen[r.Module] {
|
|
seen[r.Module] = true
|
|
out = append(out, r.Module)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// newlyMissing are the modules of the resources a statement says missing that the last statement acted on did not, each
|
|
// once, sorted, and the missing resources now: a resource still missing is not acted on again, so a module whose
|
|
// resource cannot be made is not applied at every change of the statement (the review of issue 1945).
|
|
func newlyMissing(last map[string]bool, st liveness.Statement) ([]string, map[string]bool) {
|
|
now := map[string]bool{}
|
|
var fresh liveness.Statement
|
|
for _, r := range st.Resources {
|
|
if !r.Missing {
|
|
continue
|
|
}
|
|
now[r.ID] = true
|
|
if !last[r.ID] {
|
|
fresh.Resources = append(fresh.Resources, r)
|
|
}
|
|
}
|
|
return missingModules(fresh), now
|
|
}
|