Refuse a declaration for the other mode, or older than the mesh's last, and preview before applying (hq issue 104) #23
@@ -55,12 +55,13 @@ connects to nothing and listens on nothing — what it applies comes from a file
|
||||
mesh-host profile what this machine can be asked to do
|
||||
mesh-host inventory what this machine is, and what it holds
|
||||
mesh-host apply FILE make this machine match a declaration from a file
|
||||
mesh-host reconcile make this machine match the declaration this host carries
|
||||
mesh-host reconcile make this machine match what the mesh last told it — or, before
|
||||
any mesh has, the bundle this host carries
|
||||
mesh-host bundle show what this host carries
|
||||
mesh-host owned what this host has applied and still owns
|
||||
--json machine-readable
|
||||
--state where this node keeps what it knows
|
||||
--dry-run read and check the declaration, change nothing
|
||||
--dry-run say what applying would change, and change nothing
|
||||
```
|
||||
|
||||
```
|
||||
@@ -114,8 +115,16 @@ A host built for a machine carries its declaration **inside the binary**:
|
||||
make host BUNDLE=path/to/foundation.lock
|
||||
```
|
||||
|
||||
`mesh-host reconcile` then applies it. That is the first node's path — no mesh present, nothing
|
||||
fetched, nothing else copied onto the machine. `copy it and run it` stops being true the moment
|
||||
`mesh-host reconcile` then applies it, on a machine the mesh has told nothing yet. That is the
|
||||
first node's path — no mesh present, nothing fetched, nothing else copied onto the machine. Once
|
||||
the mesh has spoken, `reconcile` holds the machine to what it last said and never to the bundle,
|
||||
which genesis consumed; a bundle or a file is refused when it says the other mode than the node
|
||||
is in; and `apply FILE` is refused altogether once the mesh has spoken — a file is applied as the
|
||||
bundle is, its resources recorded as the machine's own, so on an enrolled node it would plan to
|
||||
remove the foundation. `apply FILE` is for a machine the mesh has not spoken to. (hq's to-be
|
||||
node lifecycle describes `apply repair.json` as a rescue on an enrolled node; that line is being
|
||||
amended in hq, and no rescue path exists here yet.) Both commands say what they would change
|
||||
before changing anything, and `--dry-run` is that alone. `copy it and run it` stops being true the moment
|
||||
a second file has to arrive with it, which is why the bundle is embedded rather than beside it.
|
||||
|
||||
**A default build carries nothing and refuses to reconcile**, saying so. A host that applied
|
||||
|
||||
+274
-34
@@ -15,6 +15,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
@@ -53,7 +54,8 @@ const usage = `mesh-host — the node host
|
||||
profile what this machine can be asked to do
|
||||
inventory what this machine is, and what it holds
|
||||
apply FILE make this machine match a declaration from a file
|
||||
reconcile make this machine match the declaration this host carries
|
||||
reconcile make this machine match what the mesh last told it — or, before any
|
||||
mesh has, the bundle this host carries
|
||||
bundle show what this host carries
|
||||
owned what this host has applied and still owns
|
||||
version
|
||||
@@ -61,7 +63,10 @@ const usage = `mesh-host — the node host
|
||||
--json machine-readable output
|
||||
--timeout how long any single probe may take (default 10s)
|
||||
--state where this node keeps what it knows (default /var/lib/mesh-host/state.json)
|
||||
--dry-run read the declaration and refuse it if wrong, but change nothing
|
||||
--dry-run say what applying would change, and change nothing
|
||||
|
||||
Both apply and reconcile say what they would change before changing anything, and refuse a
|
||||
declaration for the other mode than this node is in, or one older than what the mesh last said.
|
||||
|
||||
It connects to nothing and listens on nothing. What it applies comes from a file.
|
||||
`
|
||||
@@ -90,6 +95,8 @@ type options struct {
|
||||
nodeName string
|
||||
dryRun bool
|
||||
file string
|
||||
// out is where what a command says goes. Stdout, and a buffer under test.
|
||||
out io.Writer
|
||||
}
|
||||
|
||||
// parseArgs takes the subcommand first, then its flags.
|
||||
@@ -99,7 +106,7 @@ type options struct {
|
||||
// passed, silently ignored, with a successful exit. That is the fault this whole project keeps
|
||||
// naming, so the parser takes the subcommand off the front and parses what follows.
|
||||
func parseArgs(args []string) (string, options, error) {
|
||||
opts := options{timeout: 10 * time.Second, state: store.DefaultPath}
|
||||
opts := options{timeout: 10 * time.Second, state: store.DefaultPath, out: os.Stdout}
|
||||
|
||||
command := ""
|
||||
if len(args) > 0 {
|
||||
@@ -184,18 +191,14 @@ func run(ctx context.Context, command string, opts options) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return runApply(ctx, opts, d, opts.file)
|
||||
return runApply(ctx, opts, d, raw, fromFile)
|
||||
|
||||
case "reconcile":
|
||||
// The first node's path. novox/hq ADR 0004: no mesh reachable means the declaration
|
||||
// comes from the bundle the host carries. There is no link yet, so this is currently
|
||||
// the only source — which is a stage, not a design, and saying so beats implying the
|
||||
// other source exists.
|
||||
d, err := bundle.Load(builtFor)
|
||||
d, raw, from, err := reconcileSource(opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return runApply(ctx, opts, d, "the carried bundle")
|
||||
return runApply(ctx, opts, d, raw, from)
|
||||
|
||||
case "bundle":
|
||||
if bundle.IsEmpty(builtFor) {
|
||||
@@ -247,8 +250,10 @@ func run(ctx context.Context, command string, opts options) error {
|
||||
}
|
||||
}
|
||||
|
||||
func writeJSON(v any) error {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
func writeJSON(v any) error { return writeJSONTo(os.Stdout, v) }
|
||||
|
||||
func writeJSONTo(w io.Writer, v any) error {
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(v)
|
||||
}
|
||||
@@ -305,20 +310,221 @@ func writeInventory(inv inventory.Inventory) {
|
||||
}
|
||||
}
|
||||
|
||||
// runApply reads a declaration and makes the machine match it.
|
||||
// provenance is where a declaration a command applies came from. It decides the origin its
|
||||
// resources are recorded under, what it is held against, and what is recorded once it applied.
|
||||
type provenance int
|
||||
|
||||
const (
|
||||
// fromFile is `apply FILE`: handed to the host by someone already running it as root.
|
||||
fromFile provenance = iota
|
||||
// fromBundle is `reconcile` on a machine the mesh has told nothing yet: the bundle carried in
|
||||
// the binary, the first node's path before its mesh is up (novox/hq ADR 0004).
|
||||
fromBundle
|
||||
// fromDeclared is `reconcile` on a node the mesh has spoken to: what it last said, kept
|
||||
// signed beside the state (declared.json), verified again before it is applied.
|
||||
fromDeclared
|
||||
)
|
||||
|
||||
// reconcileSource is which declaration `reconcile` holds this machine to.
|
||||
//
|
||||
// **What the mesh last said, never the bundle, once the mesh has said anything** (novox/hq issue
|
||||
// 104). The bundle is right at genesis and stale a minute later — genesis rewrites it for the
|
||||
// machine before applying it, and every declaration since came from the controller — and on an
|
||||
// adopted node it is a converged declaration for a machine that is not converged. A node that
|
||||
// has been told something and cannot prove it is the mesh's is refused, with the reason; the
|
||||
// bundle is not applied in its place.
|
||||
func reconcileSource(opts options) (*declaration.Declaration, []byte, provenance, error) {
|
||||
path := store.DeclaredPath(opts.state)
|
||||
_, err := store.ReadDeclared(path)
|
||||
switch {
|
||||
case err == nil:
|
||||
mine, err := identity.Load(identity.Path(opts.state))
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("this node was told a declaration by the mesh, kept at %s, "+
|
||||
"and cannot prove it is the mesh's: %w\n\nIt is not applied unproven, and the bundle "+
|
||||
"this host carries is not applied in its place: that was consumed at genesis", path, err)
|
||||
}
|
||||
raw, err := store.LoadDeclared(path, mine.Membership.Signer)
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("%w\n\nThe bundle this host carries is not applied in its "+
|
||||
"place: that was consumed at genesis", err)
|
||||
}
|
||||
d, err := declaration.Parse(raw)
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
return d, raw, fromDeclared, nil
|
||||
case errors.Is(err, store.ErrNothingDeclared):
|
||||
d, err := bundle.Load(builtFor)
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
return d, bundle.Raw(builtFor), fromBundle, nil
|
||||
default:
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
func (p provenance) origin() string {
|
||||
if p == fromDeclared {
|
||||
return store.OriginDeclared
|
||||
}
|
||||
// A file handed to the host is applied as the bundle is: what it puts here is invisible to
|
||||
// the removal pass of a declaration that later arrives from the mesh (novox/hq issue 010).
|
||||
return store.OriginCarried
|
||||
}
|
||||
|
||||
func (p provenance) name(opts options) string {
|
||||
switch p {
|
||||
case fromBundle:
|
||||
return "the carried bundle"
|
||||
case fromDeclared:
|
||||
return "what the mesh last told this node (" + store.DeclaredName + ")"
|
||||
}
|
||||
return opts.file
|
||||
}
|
||||
|
||||
// short is a digest as a person reads one aloud.
|
||||
func short(digest string) string {
|
||||
if len(digest) > 12 {
|
||||
return digest[:12]
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
// refuseStale refuses a declaration that is not the one this node should be held to (novox/hq
|
||||
// issue 104), naming both.
|
||||
//
|
||||
// **A declaration carries no order.** The controller signs a version — the vocabulary — the
|
||||
// node it is for, the mode, and the resources: no sequence, no issued-at. What exists is
|
||||
// identity: the mesh names what it sends by the digest of the bytes, the node keeps the last one
|
||||
// it was sent, and genesis records the digest of what it consumed. So the bundle is held to
|
||||
// genesis's digest, and a file is not applied at all once the mesh has spoken: a file is applied
|
||||
// as the bundle is, its resources recorded as this machine's own — so the mesh could never remove
|
||||
// them again — and everything the mesh declared read as no longer declared and removed. Even the
|
||||
// very declaration the mesh last sent, applied from a file, would plan to remove the foundation.
|
||||
// `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so.
|
||||
func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance) error {
|
||||
switch from {
|
||||
case fromDeclared:
|
||||
return nil
|
||||
case fromBundle:
|
||||
if known.Genesis == nil || known.Genesis.Digest == digest {
|
||||
return nil
|
||||
}
|
||||
// By digest. Genesis applies the bundle rewritten for this machine — its foundation
|
||||
// ports, root credentials, mode — and writes that lock out; a host built from the
|
||||
// carried template does not match it, and one built from the written lock does.
|
||||
return fmt.Errorf("the bundle this host carries (%s) is not the one genesis applied here on %s "+
|
||||
"(%s), which was the bundle rewritten for this machine. It was consumed then, and nothing "+
|
||||
"the mesh has said is kept on this node yet, so there is nothing to reconcile against: "+
|
||||
"enrol this node, or push to it from the controller",
|
||||
short(digest), known.Genesis.At.Format(time.RFC3339), short(known.Genesis.Digest))
|
||||
}
|
||||
// A file.
|
||||
switch {
|
||||
case errors.Is(keptErr, store.ErrNothingDeclared):
|
||||
// The mesh has said nothing here: a machine a file is for.
|
||||
return nil
|
||||
case keptErr != nil:
|
||||
return fmt.Errorf("%w\n\nNothing is applied over what this node cannot read back", keptErr)
|
||||
}
|
||||
last := apply.DigestOf(kept.Declaration)
|
||||
what := fmt.Sprintf("this file (%s) is not it", short(digest))
|
||||
switch {
|
||||
case digest == last:
|
||||
what = "this file is that declaration, and from a file it would still be applied as a bundle is"
|
||||
case known.Genesis != nil && digest == known.Genesis.Digest:
|
||||
what = fmt.Sprintf("this file is the bundle genesis consumed on %s (%s), older than it",
|
||||
known.Genesis.At.Format(time.RFC3339), short(digest))
|
||||
}
|
||||
return fmt.Errorf("`apply FILE` is for a machine the mesh has not spoken to. The mesh has told this "+
|
||||
"node declaration %s, kept as %s, and %s. A file is applied as the bundle is — its resources "+
|
||||
"recorded as this machine's own, which the mesh could then never remove, and what the mesh "+
|
||||
"declared read as no longer declared — so no file is applied here: `reconcile` applies what "+
|
||||
"the mesh last said, and `push` from the controller changes it",
|
||||
short(last), store.DeclaredName, what)
|
||||
}
|
||||
|
||||
// applied is what an apply says in machine-readable form: what it planned, then what it did.
|
||||
type applied struct {
|
||||
Plan []apply.Step `json:"plan"`
|
||||
// Report is absent on a dry run, which is the plan and nothing more.
|
||||
Report *apply.Report `json:"report,omitempty"`
|
||||
}
|
||||
|
||||
// runApply makes the machine match a declaration — after refusing one this node must not apply,
|
||||
// and after saying what it would change.
|
||||
//
|
||||
// Refused first, and before anything is read from the machine: a declaration for the other
|
||||
// mode than this node is in, or one older than what the mesh last said (novox/hq issue 104).
|
||||
// Then the plan, printed whole before a single resource is touched; `--dry-run` is that and
|
||||
// nothing more.
|
||||
//
|
||||
// The state is loaded before anything is touched and saved after, including when the apply
|
||||
// fails part-way: what was applied before the failure is on the machine, and a host that did
|
||||
// not record it would believe it owns less than it does and leave that behind forever.
|
||||
func runApply(ctx context.Context, opts options, d *declaration.Declaration, source string) error {
|
||||
func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw []byte, from provenance) error {
|
||||
out := opts.out
|
||||
if out == nil {
|
||||
out = os.Stdout
|
||||
}
|
||||
// One apply at a time on this machine, whichever process asks: the link service applies too,
|
||||
// and two saves of the state interleaved lose what one of them recorded.
|
||||
unlock, err := store.Lock(opts.state, func() {
|
||||
fmt.Fprintln(out, "another apply holds this node's state — mesh-host run, or the installer; waiting for it to finish")
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
known, err := store.Load(opts.state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
source, origin, digest := from.name(opts), from.origin(), apply.DigestOf(raw)
|
||||
|
||||
// The mode this node is in. What the mesh last said is signed and was verified on load; the
|
||||
// state's note of it is this host's own, and where they disagree the note is what is wrong
|
||||
// — a genesis re-run over a node the mesh converged since, a state saved when the kept
|
||||
// declaration could not be — and is repaired, not obeyed. A bundle or a file is held to the
|
||||
// note, or to the kept declaration when the note predates it.
|
||||
kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state))
|
||||
if from == fromDeclared {
|
||||
if known.Mode != "" && known.Mode != apply.ModeOf(d) {
|
||||
fmt.Fprintf(out, "this node's record said %s; what the mesh last said, signed, says %s — the record is repaired\n",
|
||||
known.Mode, apply.ModeOf(d))
|
||||
}
|
||||
known.Mode = apply.ModeOf(d)
|
||||
} else if known.Mode == "" && keptErr == nil {
|
||||
if last, err := declaration.Parse(kept.Declaration); err == nil {
|
||||
known.Mode = apply.ModeOf(last)
|
||||
}
|
||||
}
|
||||
if err := apply.CheckMode(known, d); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := refuseStale(known, kept, keptErr, digest, from); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Said before anything is done.
|
||||
steps := apply.Plan(d, known, origin)
|
||||
if opts.json && opts.dryRun {
|
||||
return writeJSONTo(out, applied{Plan: steps})
|
||||
}
|
||||
mode := known.Mode
|
||||
if mode == "" {
|
||||
mode = "in no mode yet — nothing has said one"
|
||||
}
|
||||
fmt.Fprintf(out, "%s (%s): %d resource(s), version %d\n", source, short(digest), len(d.Resources), d.Version)
|
||||
fmt.Fprintf(out, "this node is %s; the declaration says %s\n", mode, apply.ModeOf(d))
|
||||
fmt.Fprintf(out, "\nwould change, in this order:\n")
|
||||
for _, step := range steps {
|
||||
fmt.Fprintln(out, " "+step.String())
|
||||
}
|
||||
if opts.dryRun {
|
||||
fmt.Printf("%s: %d resource(s), version %d — accepted, nothing applied\n",
|
||||
source, len(d.Resources), d.Version)
|
||||
fmt.Fprintf(out, "\n--dry-run: nothing applied\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -338,12 +544,20 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, sou
|
||||
return err
|
||||
}
|
||||
|
||||
report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried,
|
||||
fmt.Fprintf(out, "\napplying:\n")
|
||||
report, updated, applyErr := apply.ApplyKeeping(ctx, sys, d, known, origin,
|
||||
apply.ExecRunner, func(line string) {
|
||||
if !opts.json {
|
||||
fmt.Println(line)
|
||||
fmt.Fprintln(out, line)
|
||||
}
|
||||
}, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
|
||||
|
||||
// What this apply settles about the node, whichever way it went. The mode is what the
|
||||
// declaration said and the check above agreed with; the bundle, once applied, is consumed.
|
||||
updated.Mode = apply.ModeOf(d)
|
||||
if from == fromBundle {
|
||||
updated.Genesis = &store.Genesis{Digest: digest, At: time.Now().UTC()}
|
||||
}
|
||||
}, sealOpener(opts.state))
|
||||
|
||||
// Saved whichever way it went. Recording only on success would lose the footprint of a
|
||||
// failed apply, and that footprint is on the machine either way.
|
||||
@@ -380,13 +594,13 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, sou
|
||||
}
|
||||
|
||||
if opts.json {
|
||||
return writeJSON(report)
|
||||
return writeJSONTo(out, applied{Plan: steps, Report: &report})
|
||||
}
|
||||
if !report.Changed() {
|
||||
fmt.Printf("%s: already matches — %d resource(s) checked\n", source, len(report.Outcomes))
|
||||
fmt.Fprintf(out, "%s: already matches — %d resource(s) checked\n", source, len(report.Outcomes))
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s: applied — %d resource(s)\n", source, len(report.Outcomes))
|
||||
fmt.Fprintf(out, "%s: applied — %d resource(s)\n", source, len(report.Outcomes))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -629,7 +843,7 @@ func runLink(ctx context.Context, opts options) error {
|
||||
go sched.Run(ctx)
|
||||
|
||||
applier := func(ctx context.Context, raw, signature []byte) link.Report {
|
||||
return applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched)
|
||||
return applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
|
||||
}
|
||||
|
||||
// Two things at once, and the second is what makes disconnection ordinary. The link brings
|
||||
@@ -792,7 +1006,7 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
|
||||
continue
|
||||
}
|
||||
|
||||
report := applyDeclared(ctx, opts, declared, sched)
|
||||
report := applyDeclared(ctx, opts, declared, sched, say)
|
||||
// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
|
||||
// its firewall changed, because that is how a predecessor still writing is caught
|
||||
// (novox/hq ADR 0100); publish decides whether anything did.
|
||||
@@ -813,31 +1027,58 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
|
||||
// Signature checking happens before this is called, in the link. By the time anything here runs,
|
||||
// the question "is this from the mesh I joined" is settled — which is why this can treat the
|
||||
// bytes as instructions.
|
||||
func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.Scheduler) link.Report {
|
||||
return applyAndKeep(ctx, opts, raw, nil, sched)
|
||||
func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.Scheduler, say link.Announce) link.Report {
|
||||
return applyAndKeep(ctx, opts, raw, nil, sched, say)
|
||||
}
|
||||
|
||||
// applying serialises applies on this node.
|
||||
// applying serialises applies within this process.
|
||||
//
|
||||
// **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
|
||||
// acts on the machine, and writes the state back. Run at the same time they interleave, and the
|
||||
// one that saves last writes a state read before the other acted — losing what the first recorded:
|
||||
// a hold, the firewall found here, a resource just applied. The machine would then be one thing
|
||||
// and its record another, which is the fault every read-back in this package exists to prevent.
|
||||
// Across processes — `reconcile` run by hand beside this service — the lock beside the state does
|
||||
// the same (store.Lock).
|
||||
var applying sync.Mutex
|
||||
|
||||
// applyAndKeep applies a declaration and, when it came from the mesh, keeps it so this node can
|
||||
// go on obeying it while disconnected. One at a time, whoever asks.
|
||||
// go on obeying it while disconnected. One at a time, whoever asks. Given unsigned, the bytes are
|
||||
// what this node kept, already verified against the mesh's key on load.
|
||||
func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared,
|
||||
sched *apply.Scheduler) link.Report {
|
||||
sched *apply.Scheduler, say link.Announce) link.Report {
|
||||
applying.Lock()
|
||||
defer applying.Unlock()
|
||||
unlock, err := store.Lock(opts.state, nil)
|
||||
if err != nil {
|
||||
return link.Report{Refused: err.Error()}
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
declared, err := declaration.Parse(raw)
|
||||
if err != nil {
|
||||
return link.Report{Refused: err.Error()}
|
||||
}
|
||||
|
||||
known, err := store.Load(opts.state)
|
||||
if err != nil {
|
||||
return link.Report{Refused: err.Error()}
|
||||
}
|
||||
// A declaration from the mesh is the controller's word on the node's mode — the flip arrives
|
||||
// as exactly that, the first converged declaration after adopted ones — and becomes the
|
||||
// record. So does what this node kept: it is signed by the mesh and verified on load, where
|
||||
// the state's note of the mode is this host's own. Where they disagree the note is wrong — a
|
||||
// genesis re-run over a node the mesh converged since, a state saved when the kept declaration
|
||||
// could not be — and it is repaired and said, not obeyed: refusing would hold this node off what
|
||||
// the mesh said until a delivery that comes only when something changes (novox/hq issue 104).
|
||||
if signed == nil && known.Mode != "" && known.Mode != apply.ModeOf(declared) {
|
||||
if say != nil {
|
||||
say(fmt.Sprintf("this node's record said %s; what the mesh last said, signed, says %s — the record is repaired",
|
||||
known.Mode, apply.ModeOf(declared)))
|
||||
}
|
||||
known.Mode = apply.ModeOf(declared)
|
||||
}
|
||||
|
||||
built, err := system.For(builtFor)
|
||||
if err != nil {
|
||||
return link.Report{Refused: err.Error()}
|
||||
@@ -846,11 +1087,6 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
|
||||
return link.Report{Refused: err.Error()}
|
||||
}
|
||||
|
||||
known, err := store.Load(opts.state)
|
||||
if err != nil {
|
||||
return link.Report{Refused: err.Error()}
|
||||
}
|
||||
|
||||
if err := built.Confirm(ctx, apply.ExecRunner); err != nil {
|
||||
return link.Report{Refused: err.Error()}
|
||||
}
|
||||
@@ -860,6 +1096,10 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
|
||||
outcome, updated, applyErr := apply.ApplyKeeping(ctx, built, declared, known, store.OriginDeclared,
|
||||
apply.ExecRunner, nil, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)))
|
||||
|
||||
// The mode the mesh said, recorded whichever way the apply went: the declaration is kept
|
||||
// either way, and the node is held to it from the next reconcile (novox/hq ADR 0100).
|
||||
updated.Mode = apply.ModeOf(declared)
|
||||
|
||||
// Saved whichever way it went. Recording only on success would lose the footprint of a
|
||||
// failed apply, and that footprint is on the machine either way.
|
||||
if saveErr := store.Save(opts.state, updated); saveErr != nil {
|
||||
|
||||
+240
-1
@@ -1,14 +1,21 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/bundle"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
@@ -218,7 +225,7 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) {
|
||||
// Whatever else is applying — the link, while this is the reconcile — this waits for it.
|
||||
applying.Lock()
|
||||
done := make(chan link.Report, 1)
|
||||
go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil) }()
|
||||
go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil, nil) }()
|
||||
select {
|
||||
case report := <-done:
|
||||
applying.Unlock()
|
||||
@@ -262,3 +269,235 @@ func TestAChangeThatNeverReachedTheMeshIsSaidAgain(t *testing.T) {
|
||||
t.Error("a change the mesh was told was said again")
|
||||
}
|
||||
}
|
||||
|
||||
// Defends novox/hq issue 104: a declaration for the other mode than this node is in is refused at
|
||||
// the point of application, whichever command delivered it, naming both — an adopted control-node
|
||||
// once applied its converged genesis bundle and closed itself for forty-five minutes.
|
||||
|
||||
func stateWithMode(t *testing.T, mode string) options {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
opts := options{state: filepath.Join(dir, "state.json"), out: &bytes.Buffer{}}
|
||||
if err := store.Save(opts.state, store.State{Mode: mode}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func TestAConvergedDeclarationIsRefusedOnAnAdoptedNode(t *testing.T) {
|
||||
opts := stateWithMode(t, store.ModeAdopted)
|
||||
path := filepath.Join(filepath.Dir(opts.state), "filter.conf")
|
||||
raw := []byte(`{"declaration":1,"resources":[{"id":"filter","type":"file","path":"` + path +
|
||||
`","content":"table inet filter { chain input { policy drop; } }\n"}]}`)
|
||||
d, err := declaration.ParseFileTrusted(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, from := range []provenance{fromFile, fromBundle} {
|
||||
err := runApply(context.Background(), opts, d, raw, from)
|
||||
if err == nil {
|
||||
t.Fatalf("a converged declaration was applied to an adopted node (from %d)", from)
|
||||
}
|
||||
want := "this node is adopted; the declaration says converged"
|
||||
if !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "`converge`") {
|
||||
t.Errorf("the refusal does not name both modes and the act that changes it: %v", err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Error("the refused declaration touched the machine")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheKeptDeclarationRepairsARecordThatDisagrees(t *testing.T) {
|
||||
// What a node kept is signed by the mesh and verified on load; the state's note of the mode is
|
||||
// the host's own. A genesis re-run after `converge`, or a state saved when the kept declaration
|
||||
// could not be, leaves them apart — and a loop that refused every five minutes would hold the
|
||||
// node off what the mesh said until a delivery that comes only when something changes. The
|
||||
// kept declaration wins, and the repair is said.
|
||||
opts := stateWithMode(t, store.ModeConverged)
|
||||
raw := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"` +
|
||||
filepath.Join(filepath.Dir(opts.state), "a.conf") + `","content":"x\n"}]}`)
|
||||
var said []string
|
||||
report := applyAndKeep(context.Background(), opts, raw, nil, nil, func(line string) { said = append(said, line) })
|
||||
if strings.Contains(report.Refused, "the declaration says") {
|
||||
t.Fatalf("what the node kept was refused against its own note: %s", report.Refused)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "record said converged") ||
|
||||
!strings.Contains(said[0], "says adopted") || !strings.Contains(said[0], "repaired") {
|
||||
t.Errorf("the repair was not said: %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheMeshItselfMayChangeTheMode(t *testing.T) {
|
||||
// The flip is a declaration: `converge` on the controller records the mode and sends the
|
||||
// first converged declaration. Delivered by the link, signed, it is not held to the record —
|
||||
// it becomes it. (With no system linked in, the apply is refused later for that; what this
|
||||
// checks is that the refusal is not the mode's.)
|
||||
opts := stateWithMode(t, store.ModeAdopted)
|
||||
raw := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` +
|
||||
filepath.Join(filepath.Dir(opts.state), "a.conf") + `","content":"x\n"}]}`)
|
||||
report := applyAndKeep(context.Background(), opts, raw, &store.Declared{Declaration: raw}, nil, nil)
|
||||
if strings.Contains(report.Refused, "the declaration says") {
|
||||
t.Errorf("the mesh's own flip was refused for its mode: %s", report.Refused)
|
||||
}
|
||||
}
|
||||
|
||||
// Defends novox/hq issue 104: a declaration older than what the mesh last said is refused, naming
|
||||
// both — and `apply FILE` is refused altogether once the mesh has spoken, the last declaration
|
||||
// itself included: from a file it is applied as the bundle is, which would record the mesh's
|
||||
// resources as this machine's own and remove the foundation as undeclared.
|
||||
func TestAnOlderDeclarationIsRefused(t *testing.T) {
|
||||
opts := stateWithMode(t, "")
|
||||
genesis := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"genesis\n"}]}`)
|
||||
since := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"since\n"}]}`)
|
||||
other := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"other\n"}]}`)
|
||||
if err := store.Save(opts.state, store.State{Genesis: &store.Genesis{Digest: apply.DigestOf(genesis),
|
||||
At: time.Now(), Rewritten: true}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), store.Declared{Declaration: since,
|
||||
Signature: []byte("unverified here")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parsed := func(raw []byte) *declaration.Declaration {
|
||||
d, err := declaration.ParseFileTrusted(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
err := runApply(context.Background(), opts, parsed(genesis), genesis, fromFile)
|
||||
if err == nil {
|
||||
t.Fatal("the bundle genesis consumed was applied over what the mesh said since")
|
||||
}
|
||||
for _, want := range []string{"older", short(apply.DigestOf(genesis)), short(apply.DigestOf(since))} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = runApply(context.Background(), opts, parsed(other), other, fromFile)
|
||||
if err == nil {
|
||||
t.Fatal("a declaration that is not what the mesh last said was applied")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "for a machine the mesh has not spoken to") ||
|
||||
!strings.Contains(err.Error(), short(apply.DigestOf(since))) {
|
||||
t.Errorf("the refusal does not say what a file is for and what was last said: %v", err)
|
||||
}
|
||||
|
||||
// The very declaration the mesh last sent, from a file: still a file.
|
||||
err = runApply(context.Background(), opts, parsed(since), since, fromFile)
|
||||
if err == nil || !strings.Contains(err.Error(), "applied as the bundle is") {
|
||||
t.Errorf("the last declaration, from a file, was not refused as a file: %v", err)
|
||||
}
|
||||
if known, _ := store.Load(opts.state); len(known.Resources) != 0 {
|
||||
t.Errorf("a refused file recorded %d resource(s)", len(known.Resources))
|
||||
}
|
||||
|
||||
// A bundle other than the one genesis consumed: what genesis applied was rewritten for this
|
||||
// machine, so the carried bytes never are.
|
||||
err = runApply(context.Background(), opts, parsed(other), other, fromBundle)
|
||||
if err == nil || !strings.Contains(err.Error(), "consumed") ||
|
||||
!strings.Contains(err.Error(), short(apply.DigestOf(genesis))) {
|
||||
t.Errorf("a bundle other than the consumed one was not refused naming it: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Defends novox/hq issue 104: what an apply would change is said before anything is, and
|
||||
// `--dry-run` is that and nothing else — an action listed as the action it is.
|
||||
func TestADryRunChangesNothingAndListsTheActions(t *testing.T) {
|
||||
opts := stateWithMode(t, "")
|
||||
opts.dryRun = true
|
||||
out := &bytes.Buffer{}
|
||||
opts.out = out
|
||||
path := filepath.Join(filepath.Dir(opts.state), "a.conf")
|
||||
raw := []byte(`{"declaration":1,"resources":[
|
||||
{"id":"a","type":"file","path":"` + path + `","content":"x\n"},
|
||||
{"id":"init","type":"action","command":["createdb","mesh"],"verify":["psql","-c","select 1"]}]}`)
|
||||
d, err := declaration.ParseFileTrusted(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil {
|
||||
t.Fatalf("a dry run failed: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Error("a dry run wrote the file")
|
||||
}
|
||||
for _, want := range []string{"would change", "create file a", "run action init",
|
||||
"`createdb mesh`", "--dry-run: nothing applied"} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("the preview does not say %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
if strings.Contains(out.String(), "applying:") {
|
||||
t.Errorf("a dry run went on to apply:\n%s", out.String())
|
||||
}
|
||||
|
||||
// Machine-readable, the same shape as an apply's: the plan, and no report.
|
||||
out.Reset()
|
||||
opts.json = true
|
||||
if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var shape struct {
|
||||
Plan []apply.Step `json:"plan"`
|
||||
Report *json.RawMessage `json:"report"`
|
||||
}
|
||||
if err := json.Unmarshal(out.Bytes(), &shape); err != nil || len(shape.Plan) != 2 || shape.Report != nil {
|
||||
t.Errorf("a json dry run is not {plan} alone: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Defends novox/hq issue 104: once the mesh has told this node anything, `reconcile` holds it to
|
||||
// that — never to the bundle the host carries, which genesis consumed.
|
||||
func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.T) {
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
opts := stateWithMode(t, store.ModeAdopted)
|
||||
|
||||
controllerPublic, controllerPrivate, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`)
|
||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), store.Declared{Declaration: said,
|
||||
Signature: ed25519.Sign(controllerPrivate, said)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Told, and unable to prove by whom: refused, and the bundle is not applied in its place.
|
||||
_, _, _, err = reconcileSource(opts)
|
||||
if err == nil || !strings.Contains(err.Error(), "not applied in its place") {
|
||||
t.Errorf("a node that cannot prove what it was told fell back to something: %v", err)
|
||||
}
|
||||
|
||||
mine, err := identity.Generate("workstation")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mine.Membership = identity.Membership{Broker: "198.51.100.10:5671", Fingerprint: "sha256:0",
|
||||
Signer: controllerPublic, Password: "issued"}
|
||||
if err := identity.Save(identity.Path(opts.state), mine); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, raw, from, err := reconcileSource(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if from != fromDeclared || !bytes.Equal(raw, said) || d.Adoption == nil {
|
||||
t.Errorf("reconcile chose %d with %d bytes, not what the mesh last said", from, len(raw))
|
||||
}
|
||||
|
||||
// And before the mesh has said anything: the bundle, as it always was. A test binary carries
|
||||
// only the placeholder, and asking for it is what proves the path.
|
||||
if err := os.Remove(store.DeclaredPath(opts.state)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, _, err = reconcileSource(opts)
|
||||
if !errors.Is(err, bundle.ErrEmpty) {
|
||||
t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1611,6 +1611,13 @@ func digestOf(content string) string {
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// DigestOf names a declaration by its bytes, exactly as the mesh names what it sends: sha256 of
|
||||
// the raw bytes, hex. The two sides never digest different things.
|
||||
func DigestOf(raw []byte) string {
|
||||
sum := sha256.Sum256(raw)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// holds is the machine's own ports a resource occupies.
|
||||
//
|
||||
// **What the declaration binds, not what is open.** A machine's open ports are a moving target —
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// A node is adopted or converged, and a declaration says which it is for (novox/hq ADR 0100).
|
||||
//
|
||||
// **The two are not interchangeable, and the host knows which it is.** A converged declaration
|
||||
// carries the mesh's drop-by-default filter and retires the firewall the node was found with; on
|
||||
// an adopted node that closes the machine to everything the predecessor still serves — which is
|
||||
// what happened when an operator ran `reconcile` on an adopted control-node and it applied the
|
||||
// converged genesis bundle (novox/hq issue 104). The host reported "adopted" in every report and
|
||||
// compared nothing. Now it compares, at the point of application, whichever command delivered
|
||||
// the declaration.
|
||||
//
|
||||
// Only the mesh changes a node's mode, and it does so by sending a declaration: the flip arrives
|
||||
// over the link as the first converged declaration after adopted ones (`converge` on the
|
||||
// controller), and the way back as the first adopted one (`adopt`). So a declaration the link
|
||||
// delivers, signed and verified, is the mode's authority and is not checked against the record —
|
||||
// it becomes the record. Everything else — the carried bundle, a file, the kept declaration a
|
||||
// disconnected node re-applies — is held to the mode already recorded.
|
||||
|
||||
// ModeOf says which mode a declaration is for.
|
||||
func ModeOf(d *declaration.Declaration) string {
|
||||
if d.Adoption != nil {
|
||||
return store.ModeAdopted
|
||||
}
|
||||
return store.ModeConverged
|
||||
}
|
||||
|
||||
// CheckMode refuses a declaration whose mode is not the one this node has recorded. A node with
|
||||
// no recorded mode — a machine nothing has said a mode to yet — takes either.
|
||||
func CheckMode(known store.State, d *declaration.Declaration) error {
|
||||
if known.Mode == "" || known.Mode == ModeOf(d) {
|
||||
return nil
|
||||
}
|
||||
act := "`converge`"
|
||||
if ModeOf(d) == store.ModeAdopted {
|
||||
act = "`adopt`"
|
||||
}
|
||||
return fmt.Errorf("this node is %s; the declaration says %s — %s declaration is not applied "+
|
||||
"to %s node; %s on the controller is the act that changes it, and it sends the "+
|
||||
"declaration that does", known.Mode, ModeOf(d), article(ModeOf(d)), article(known.Mode), act)
|
||||
}
|
||||
|
||||
func article(mode string) string {
|
||||
if mode == store.ModeAdopted {
|
||||
return "an adopted"
|
||||
}
|
||||
return "a converged"
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// Defends novox/hq issue 104: a declaration is refused for the other mode than the node is in,
|
||||
// naming both and the act that changes it; a node no mode has been said to takes either.
|
||||
func TestADeclarationForTheOtherModeIsRefused(t *testing.T) {
|
||||
converged := parse(t, `{"declaration":1,"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`)
|
||||
adopted := parse(t, `{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`)
|
||||
|
||||
err := CheckMode(store.State{Mode: store.ModeAdopted}, converged)
|
||||
if err == nil || !strings.Contains(err.Error(), "this node is adopted; the declaration says converged") ||
|
||||
!strings.Contains(err.Error(), "`converge`") {
|
||||
t.Errorf("a converged declaration on an adopted node: %v", err)
|
||||
}
|
||||
err = CheckMode(store.State{Mode: store.ModeConverged}, adopted)
|
||||
if err == nil || !strings.Contains(err.Error(), "this node is converged; the declaration says adopted") ||
|
||||
!strings.Contains(err.Error(), "`adopt`") {
|
||||
t.Errorf("an adopted declaration on a converged node: %v", err)
|
||||
}
|
||||
if err := CheckMode(store.State{Mode: store.ModeAdopted}, adopted); err != nil {
|
||||
t.Errorf("the node's own mode was refused: %v", err)
|
||||
}
|
||||
if err := CheckMode(store.State{}, converged); err != nil {
|
||||
t.Errorf("a node in no mode yet refused a declaration: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/firewall"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// A declaration is said before it is done.
|
||||
//
|
||||
// An apply that prints what it did after it did it is a report; what an operator reaching for
|
||||
// `reconcile` under pressure needs is a preview — the base filter that closed an adopted
|
||||
// control-node for forty-five minutes was listed nowhere until it was on disk (novox/hq issue
|
||||
// 104). Converging already previews on the controller; the host's own commands now do too, and
|
||||
// `--dry-run` is the preview and nothing else.
|
||||
|
||||
// Step is one thing an apply would do to this machine.
|
||||
type Step struct {
|
||||
// Verb is create · update · check · hold · run · remove · forget · disable · enable.
|
||||
Verb string `json:"verb"`
|
||||
Type string `json:"type,omitempty"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Target string `json:"target,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
func (s Step) String() string {
|
||||
line := fmt.Sprintf("%-8s %-10s %s", s.Verb, s.Type, s.ID)
|
||||
if s.Target != "" && s.Target != s.ID {
|
||||
line += " (" + s.Target + ")"
|
||||
}
|
||||
if s.Why != "" {
|
||||
line += " — " + s.Why
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
// Plan says what applying a declaration would change, in the order ApplyKeeping would do it,
|
||||
// before anything on the machine is touched.
|
||||
//
|
||||
// **Read from the declaration and the node's own record, not from the machine.** What the
|
||||
// record cannot settle — whether a file recorded here has since drifted, whether a container
|
||||
// runs the spec it was made from — is said as a check, because that is what the apply does: it
|
||||
// reads the machine and corrects it. What the record does settle is said as it is: a resource
|
||||
// with no record is created; a plain file whose declared content differs from what this host
|
||||
// last wrote is updated; a hold is kept; an action is run — an action is a command, and a
|
||||
// preview that folded it into "check" would hide the one kind of step that is not read back
|
||||
// from state.
|
||||
func Plan(d *declaration.Declaration, known store.State, origin string) []Step {
|
||||
var steps []Step
|
||||
|
||||
declared := map[string]bool{}
|
||||
for _, r := range d.Resources {
|
||||
declared[r.Identity()] = true
|
||||
}
|
||||
rec := known.Firewall
|
||||
ufw := rec != nil && rec.Kind == string(firewall.UFW)
|
||||
|
||||
if d.Adoption != nil && ufw && rec.DisabledByMesh {
|
||||
steps = append(steps, Step{Verb: "enable", Type: "firewall", ID: "ufw",
|
||||
Why: "this node is adopted again, so the firewall found on it is put back in force"})
|
||||
}
|
||||
|
||||
// What is held and no longer declared is let go of on paper only (ApplyKeeping does this
|
||||
// before the resources); the file or container itself is left as found.
|
||||
if origin == store.OriginDeclared {
|
||||
for _, h := range known.Held {
|
||||
if declared[h.ID] {
|
||||
continue
|
||||
}
|
||||
steps = append(steps, Step{Verb: "forget", Type: h.Kind, ID: h.ID, Target: h.Target,
|
||||
Why: "held for " + h.Module + " and no longer declared; left as found"})
|
||||
}
|
||||
}
|
||||
|
||||
var protecting, orphans []Step
|
||||
for _, orphan := range known.Orphans(declared, origin) {
|
||||
step := Step{Verb: "remove", Type: orphan.Type, ID: orphan.ID, Target: orphan.Target,
|
||||
Why: "recorded here and no longer declared"}
|
||||
if d.Adoption == nil && strings.HasPrefix(orphan.ID, declaration.AdoptionPrefix) {
|
||||
step.Why = "what protected this node while adopted; removed last, once everything else applied"
|
||||
protecting = append(protecting, step)
|
||||
continue
|
||||
}
|
||||
orphans = append(orphans, step)
|
||||
}
|
||||
|
||||
// The guard goes up before anything is removed on an adopted node; on a converged one the
|
||||
// removals go first (novox/hq ADR 0103).
|
||||
var guard, rest []declaration.Resource
|
||||
for _, r := range d.Resources {
|
||||
if d.Adoption != nil && strings.HasPrefix(r.Identity(), guardPrefix) {
|
||||
guard = append(guard, r)
|
||||
continue
|
||||
}
|
||||
rest = append(rest, r)
|
||||
}
|
||||
for _, r := range guard {
|
||||
steps = append(steps, planned(r, d, known))
|
||||
}
|
||||
steps = append(steps, orphans...)
|
||||
for _, r := range rest {
|
||||
steps = append(steps, planned(r, d, known))
|
||||
}
|
||||
|
||||
// Only a declaration from the mesh converges a node; a bundle or a file never retires the
|
||||
// firewall found here, and neither says so in a plan.
|
||||
if d.Adoption == nil && origin == store.OriginDeclared && ufw && rec.WasActive && !rec.DisabledByMesh {
|
||||
steps = append(steps, Step{Verb: "disable", Type: "firewall", ID: "ufw",
|
||||
Why: "this node converges: retired once the mesh's own filter is loaded, never before; " +
|
||||
"its configuration stays on disk"})
|
||||
}
|
||||
steps = append(steps, protecting...)
|
||||
return steps
|
||||
}
|
||||
|
||||
// planned is what one declared resource would come to.
|
||||
//
|
||||
// **In the order holdOnAdopted decides it**, because the one cutover ADR 0100 says must be
|
||||
// previewed is the one a plan gets backwards if it looks at the record first: a resource this
|
||||
// node holds for a module the declaration now says is taken is not held any longer — it is
|
||||
// applied, and what was found is replaced. The declaration's word on which modules are untaken
|
||||
// comes first; the record of what is held only says what that replacement replaces.
|
||||
func planned(r declaration.Resource, d *declaration.Declaration, known store.State) Step {
|
||||
step := Step{Type: string(r.Kind()), ID: r.Identity(), Target: r.Target()}
|
||||
|
||||
if d.Adoption != nil {
|
||||
// Something run inside a held container is held with it, while that container's module
|
||||
// is untaken; once the module is taken the container is replaced before this runs.
|
||||
if in := runsIn(r); in != "" {
|
||||
if container, isHeld := heldContainer(known, in); isHeld {
|
||||
if _, untaken := d.Adoption.Untaken[container.Module]; untaken {
|
||||
step.Verb = "hold"
|
||||
step.Why = "runs in " + in + ", which is held as found; not run until " + container.Module + " is taken"
|
||||
return step
|
||||
}
|
||||
}
|
||||
}
|
||||
// A file written into replaces nothing that was found, so it is never held (ADR 0102).
|
||||
into := false
|
||||
if f, ok := r.(*declaration.File); ok && f.Into != "" {
|
||||
into = true
|
||||
}
|
||||
h, held := known.HeldAt(r.Identity())
|
||||
module, untaken := d.Adoption.UntakenModuleOf(r.Identity())
|
||||
switch {
|
||||
case into:
|
||||
case untaken && held:
|
||||
step.Verb, step.Why = "hold", "found on this machine and kept as it is until "+module+" is taken"
|
||||
return step
|
||||
case untaken:
|
||||
step.Verb = "create"
|
||||
step.Why = "unless it is found on this machine — then held as it is until " + module + " is taken"
|
||||
return step
|
||||
case held:
|
||||
// The cutover: the module is taken, and what was held for it is replaced.
|
||||
step.Verb = "create"
|
||||
if _, recorded := known.Find(r.Identity()); recorded {
|
||||
step.Verb = "update"
|
||||
}
|
||||
step.Why = h.Module + " is taken: replaces what was found and held"
|
||||
if h.Kept != "" {
|
||||
step.Why += "; the original stays at " + h.Kept
|
||||
}
|
||||
return step
|
||||
}
|
||||
}
|
||||
|
||||
if a, ok := r.(*declaration.Action); ok {
|
||||
// Named as what it is. Its verify decides whether it runs, and that is read from the
|
||||
// machine, not the record.
|
||||
step.Verb = "run"
|
||||
step.Target = ""
|
||||
step.Why = fmt.Sprintf("an action: `%s`, unless its verify `%s` already passes; if it fails, "+
|
||||
"nothing after it is attempted", strings.Join(a.Command, " "), strings.Join(a.Verify, " "))
|
||||
if a.In != "" {
|
||||
step.Why = "in " + a.In + ", " + step.Why
|
||||
}
|
||||
return step
|
||||
}
|
||||
|
||||
was, recorded := known.Find(r.Identity())
|
||||
if !recorded {
|
||||
step.Verb, step.Why = "create", "no record of it on this node"
|
||||
return step
|
||||
}
|
||||
if want := wouldWrite(r); want != "" && was.Wrote != "" && want != was.Wrote {
|
||||
step.Verb, step.Why = "update", "the declaration changed since this host applied it"
|
||||
return step
|
||||
}
|
||||
step.Verb, step.Why = "check", "recorded here; corrected if this machine drifted from it"
|
||||
return step
|
||||
}
|
||||
|
||||
// wouldWrite is the digest a plain file would be recorded under, or empty where only the apply
|
||||
// can know: a sealed file, one with secrets in it, one written into, one carrying bytes.
|
||||
func wouldWrite(r declaration.Resource) string {
|
||||
f, ok := r.(*declaration.File)
|
||||
if !ok || f.Into != "" || f.Bytes != "" || f.Secret() || len(f.Secrets) > 0 {
|
||||
return ""
|
||||
}
|
||||
return digestOf(f.Content)
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// Defends novox/hq issue 104: what an apply would change is said before anything is, from the
|
||||
// declaration and the node's record — and an action is named as the action it is.
|
||||
|
||||
func trusted(t *testing.T, raw string) *declaration.Declaration {
|
||||
t.Helper()
|
||||
d, err := declaration.ParseFileTrusted([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("fixture is not a valid declaration: %v", err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func verbs(steps []Step) string {
|
||||
var out []string
|
||||
for _, s := range steps {
|
||||
out = append(out, s.Verb+" "+s.ID)
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
func TestAPlanNamesAnActionAsAnAction(t *testing.T) {
|
||||
d := trusted(t, `{"declaration":1,"resources":[
|
||||
{"id":"init","type":"action","command":["createdb","mesh"],"verify":["psql","-c","select 1"]}]}`)
|
||||
steps := Plan(d, store.State{}, store.OriginCarried)
|
||||
if len(steps) != 1 || steps[0].Verb != "run" {
|
||||
t.Fatalf("an action was planned as %s", verbs(steps))
|
||||
}
|
||||
if !strings.Contains(steps[0].Why, "createdb mesh") || !strings.Contains(steps[0].Why, "nothing after it") {
|
||||
t.Errorf("the plan does not say what the action runs and what failing it means: %q", steps[0].Why)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPlanSaysWhatIsRecordedAndWhatIsNot(t *testing.T) {
|
||||
d := parse(t, `{"declaration":1,"resources":[
|
||||
{"id":"new","type":"file","path":"/tmp/new","content":"a\n"},
|
||||
{"id":"same","type":"file","path":"/tmp/same","content":"b\n"},
|
||||
{"id":"moved","type":"file","path":"/tmp/moved","content":"c\n"},
|
||||
{"id":"sealed","type":"file","path":"/tmp/sealed","sealed":"AAAA","mode":"0600"}]}`)
|
||||
known := store.State{}
|
||||
known.Record(store.Applied{ID: "same", Type: "file", Target: "/tmp/same", Wrote: digestOf("b\n")})
|
||||
known.Record(store.Applied{ID: "moved", Type: "file", Target: "/tmp/moved", Wrote: digestOf("old\n")})
|
||||
known.Record(store.Applied{ID: "sealed", Type: "file", Target: "/tmp/sealed", Wrote: digestOf("secret")})
|
||||
known.Record(store.Applied{ID: "gone", Type: "file", Target: "/tmp/gone"})
|
||||
|
||||
got := verbs(Plan(d, known, store.OriginCarried))
|
||||
want := "remove gone, create new, check same, update moved, check sealed"
|
||||
if got != want {
|
||||
t.Errorf("planned %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPlanSaysTheFirewallAConvergingNodeRetires(t *testing.T) {
|
||||
d := parse(t, `{"declaration":1,"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`)
|
||||
known := store.State{Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, FoundAt: time.Now()}}
|
||||
known.Record(store.Applied{ID: "adoption.guard.table", Type: "file", Target: "/etc/guard", Origin: store.OriginDeclared})
|
||||
|
||||
got := verbs(Plan(d, known, store.OriginDeclared))
|
||||
// The firewall goes last but for what protected the node, which goes after it.
|
||||
if got != "create a, disable ufw, remove adoption.guard.table" {
|
||||
t.Errorf("a converging node planned %q", got)
|
||||
}
|
||||
// A file or the bundle never retires the firewall found here, and says nothing about it.
|
||||
if got := verbs(Plan(d, known, store.OriginCarried)); strings.Contains(got, "ufw") {
|
||||
t.Errorf("a carried declaration planned to touch the firewall: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPlanHoldsWhatAnAdoptedNodeFound(t *testing.T) {
|
||||
d := parse(t, `{"declaration":1,"adoption":{"taken":[],"untaken":{"hello-web":["hello-web.page","hello-web.server"]}},
|
||||
"resources":[
|
||||
{"id":"hello-web.page","type":"file","path":"/srv/index.html","content":"x\n"},
|
||||
{"id":"hello-web.server","type":"container","name":"hello-web","image":"example/web@sha256:0000000000000000000000000000000000000000000000000000000000000000"}]}`)
|
||||
known := store.State{}
|
||||
known.RecordHeld(store.Held{ID: "hello-web.page", Module: "hello-web", Kind: "file", Target: "/srv/index.html"})
|
||||
|
||||
steps := Plan(d, known, store.OriginDeclared)
|
||||
if len(steps) != 2 || steps[0].Verb != "hold" || steps[1].Verb != "create" {
|
||||
t.Fatalf("an adopted node planned %s", verbs(steps))
|
||||
}
|
||||
if !strings.Contains(steps[1].Why, "held as it is until hello-web is taken") {
|
||||
t.Errorf("the plan does not say an untaken module's resource is held if found: %q", steps[1].Why)
|
||||
}
|
||||
}
|
||||
|
||||
// both is a machine with a container runtime and ufw on it at once.
|
||||
type both struct {
|
||||
m *machine
|
||||
u *ufwMachine
|
||||
}
|
||||
|
||||
func (b *both) run(ctx context.Context, name string, args ...string) (string, error) {
|
||||
switch name {
|
||||
case "nft", "ufw", "iptables", "ip6tables", "firewall-cmd":
|
||||
return b.u.run(ctx, name, args...)
|
||||
}
|
||||
return b.m.run(ctx, name, args...)
|
||||
}
|
||||
|
||||
func ids(steps []Step) []string {
|
||||
var out []string
|
||||
for _, s := range steps {
|
||||
if s.Type == "firewall" {
|
||||
continue // said, not an outcome
|
||||
}
|
||||
out = append(out, s.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func outcomeIDs(r Report) []string {
|
||||
var out []string
|
||||
for _, o := range r.Outcomes {
|
||||
out = append(out, o.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func write(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Defends novox/hq issue 104: the plan is the apply, said first — the same resources in the same
|
||||
// order, and the one cutover ADR 0100 says must be previewed said as a cutover, not a hold.
|
||||
func TestThePlanIsTheApplyInOrder(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
guard, page, keep, old := filepath.Join(dir, "guard.nft"), filepath.Join(dir, "index.html"),
|
||||
filepath.Join(dir, "keep.html"), filepath.Join(dir, "old.conf")
|
||||
for _, p := range []string{page, keep, old} {
|
||||
write(t, p, "the predecessor's\n")
|
||||
}
|
||||
|
||||
// Returning to adopted, with a guard to raise, an orphan, a hold that stays, a hold whose
|
||||
// module is now taken, and a hold no longer declared.
|
||||
back := adopted(t, `{"taken":["hello-web"],"untaken":{"keep":["keep.page"]}}`,
|
||||
`{"id":"adoption.guard.table","type":"file","path":"`+guard+`","content":"table inet mesh-guard {}\n"},
|
||||
{"id":"hello-web.page","type":"file","path":"`+page+`","content":"the mesh's page\n"},
|
||||
{"id":"keep.page","type":"file","path":"`+keep+`","content":"the mesh's keep\n"}`)
|
||||
known := store.State{Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, DisabledByMesh: true, FoundAt: time.Now()}}
|
||||
known.Record(store.Applied{ID: "old.conf", Type: "file", Target: old, Origin: store.OriginDeclared})
|
||||
for id, module := range map[string]string{"hello-web.page": "hello-web", "keep.page": "keep", "gone.page": "gone"} {
|
||||
known.RecordHeld(store.Held{ID: id, Module: module, Kind: "file", Target: filepath.Join(dir, id), Since: time.Now()})
|
||||
}
|
||||
fake := &both{m: &machine{containers: map[string]*fakeContainer{}}, u: &ufwMachine{installed: true}}
|
||||
|
||||
plan := Plan(back, known, store.OriginDeclared)
|
||||
report, state, err := ApplyKeeping(context.Background(), archHost(t), back, known, store.OriginDeclared,
|
||||
fake.run, nil, nil, KeepIn(dir))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := verbs(plan), "enable ufw, forget gone.page, create adoption.guard.table, remove old.conf, "+
|
||||
"create hello-web.page, hold keep.page"; got != want {
|
||||
t.Errorf("planned %q, want %q", got, want)
|
||||
}
|
||||
if got, want := strings.Join(ids(plan), " "), strings.Join(outcomeIDs(report), " "); got != want {
|
||||
t.Errorf("the plan said %q and the apply did %q", got, want)
|
||||
}
|
||||
taken := outcomeOf(report, "hello-web.page")
|
||||
if !strings.Contains(taken.Detail, "taken") || !strings.Contains(plan[4].Why, "hello-web is taken: replaces what was found") {
|
||||
t.Errorf("the cutover was applied as %q and planned as %q", taken.Detail, plan[4].Why)
|
||||
}
|
||||
if !fake.u.active || state.Firewall.DisabledByMesh {
|
||||
t.Error("returning to adopted did not enable ufw again")
|
||||
}
|
||||
|
||||
// Converged, from the mesh: an orphan, a new file, ufw retired, and what protected the node
|
||||
// removed last.
|
||||
flip := parse(t, `{"declaration":1,"resources":[{"id":"a","type":"file","path":"`+filepath.Join(dir, "a.conf")+`","content":"a\n"}]}`)
|
||||
write(t, old, "again\n")
|
||||
known = store.State{Firewall: &store.FoundFirewall{Kind: "ufw", WasActive: true, FoundAt: time.Now()}}
|
||||
known.Record(store.Applied{ID: "adoption.guard.table", Type: "file", Target: guard, Origin: store.OriginDeclared})
|
||||
known.Record(store.Applied{ID: "old.conf", Type: "file", Target: old, Origin: store.OriginDeclared})
|
||||
fake = &both{m: &machine{containers: map[string]*fakeContainer{}}, u: &ufwMachine{installed: true, active: true,
|
||||
ruleset: "table inet mesh {\n}\n"}}
|
||||
|
||||
plan = Plan(flip, known, store.OriginDeclared)
|
||||
report, state, err = ApplyKeeping(context.Background(), archHost(t), flip, known, store.OriginDeclared,
|
||||
fake.run, nil, nil, KeepIn(dir))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := verbs(plan), "remove old.conf, create a, disable ufw, remove adoption.guard.table"; got != want {
|
||||
t.Errorf("planned %q, want %q", got, want)
|
||||
}
|
||||
if got, want := strings.Join(ids(plan), " "), strings.Join(outcomeIDs(report), " "); got != want {
|
||||
t.Errorf("the plan said %q and the apply did %q", got, want)
|
||||
}
|
||||
if fake.u.active || !state.Firewall.DisabledByMesh {
|
||||
t.Error("converging did not retire ufw")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAResourceRunInAHeldContainerIsPlannedAsItIsApplied(t *testing.T) {
|
||||
// A run-once step sharing a container's namespace runs in it, as an action's `in` does.
|
||||
img := "example/x@sha256:0000000000000000000000000000000000000000000000000000000000000000"
|
||||
d := parse(t, `{"declaration":1,"adoption":{"taken":["web"],"untaken":{"db":["db.server"]}},"resources":[
|
||||
{"id":"web.server","type":"container","name":"web","image":"`+img+`"},
|
||||
{"id":"db.server","type":"container","name":"db","image":"`+img+`"},
|
||||
{"id":"db.init","type":"container","name":"db-init","image":"`+img+`","run-once":true,"network":"container:db"},
|
||||
{"id":"web.warm","type":"container","name":"web-warm","image":"`+img+`","run-once":true,"network":"container:web"}]}`)
|
||||
known := store.State{}
|
||||
known.RecordHeld(store.Held{ID: "db.server", Module: "db", Kind: "container", Target: "db", Container: "predecessor"})
|
||||
known.RecordHeld(store.Held{ID: "web.server", Module: "web", Kind: "container", Target: "web", Container: "predecessor"})
|
||||
got := verbs(Plan(d, known, store.OriginDeclared))
|
||||
// db is untaken, so what runs in it waits; web is taken, so its held container is replaced (the
|
||||
// cutover) and what runs in it runs.
|
||||
if got != "create web.server, hold db.server, hold db.init, create web.warm" {
|
||||
t.Errorf("planned %q", got)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
@@ -33,8 +34,16 @@ type Runner = apply.Runner
|
||||
// What it does not do is the host's own lifecycle bookkeeping — recording a known-good version,
|
||||
// clearing the launcher's start counter. Those are facts about a running `mesh-host`, and this is
|
||||
// not one.
|
||||
//
|
||||
// What it does record is that the bundle was consumed, and in which mode the operator raised
|
||||
// the machine. `raw` is the exact bytes of what is applied — the bundle as rewritten for this
|
||||
// machine — and its digest is what `mesh-host reconcile` later holds the carried bundle against,
|
||||
// by digest: a host built from the carried template does not match it, since genesis rewrote the
|
||||
// ports, root credentials and adoption; a host built from the lock genesis wrote out does.
|
||||
// Applying the unrewritten template on a raised node recreated the store and loaded the converged
|
||||
// filter on an adopted one (novox/hq issue 104).
|
||||
func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declaration.Declaration,
|
||||
run Runner, say func(string)) (apply.Report, error) {
|
||||
raw []byte, run Runner, say func(string)) (apply.Report, error) {
|
||||
|
||||
// Refuse a shape this host cannot apply before anything is applied, exactly as `mesh-host`
|
||||
// does: finding out half way through is the half-configured machine tier 0 exists to prevent.
|
||||
@@ -42,6 +51,12 @@ func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declarati
|
||||
return apply.Report{}, err
|
||||
}
|
||||
|
||||
// One apply at a time on this machine: a host already running here applies too.
|
||||
unlock, err := store.Lock(o.State, func() { say(" waiting another apply holds this node's state") })
|
||||
if err != nil {
|
||||
return apply.Report{}, err
|
||||
}
|
||||
defer unlock()
|
||||
known, err := store.Load(o.State)
|
||||
if err != nil {
|
||||
return apply.Report{}, err
|
||||
@@ -55,6 +70,20 @@ func ApplyBundle(ctx context.Context, o Options, sys system.System, d *declarati
|
||||
func(line string) { say(" " + strings.TrimPrefix(line, " ")) }, refuseSealed,
|
||||
apply.KeepIn(filepath.Dir(o.State)))
|
||||
|
||||
// The bundle is consumed, whichever way the apply went: what is on the machine came from these
|
||||
// bytes, and the carried ones must not be applied over it. The mode is the operator's word at
|
||||
// genesis, and only at genesis: the controller records the same and says it in every
|
||||
// declaration from then on (novox/hq ADR 0100), so a node the mesh has spoken to — this
|
||||
// installer re-run on it, or finishing a pivot — keeps the mode it has, which may since have
|
||||
// been flipped.
|
||||
updated.Genesis = &store.Genesis{Digest: apply.DigestOf(raw), At: time.Now().UTC(), Rewritten: true}
|
||||
if updated.Mode == "" {
|
||||
updated.Mode = store.ModeConverged
|
||||
if o.Adopted {
|
||||
updated.Mode = store.ModeAdopted
|
||||
}
|
||||
}
|
||||
|
||||
// Saved whichever way it went, for the reason `mesh-host` gives: what was applied before a
|
||||
// failure is on the machine either way, and a host that did not record it would believe it
|
||||
// owns less than it does and leave that behind for ever.
|
||||
|
||||
@@ -3,12 +3,15 @@ package bootstrap
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
@@ -113,7 +116,7 @@ func TestTheBundleKeepsTheOriginalOfWhatItWritesOver(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o := Options{State: filepath.Join(dir, "state.json")}
|
||||
report, err := ApplyBundle(context.Background(), o, sys, d, nil, quietly)
|
||||
report, err := ApplyBundle(context.Background(), o, sys, d, nil, nil, quietly)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -127,3 +130,53 @@ func TestTheBundleKeepsTheOriginalOfWhatItWritesOver(t *testing.T) {
|
||||
t.Errorf("the kept original is %q (%v)", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Defends novox/hq issue 104: genesis consumes the bundle, recording the digest of what it applied
|
||||
// and the mode the operator raised the machine in, so the host's own `reconcile` never applies the
|
||||
// carried bytes over it.
|
||||
func TestGenesisConsumesTheBundleAndRecordsTheMode(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
raw := []byte(`{"declaration":1,"resources":[
|
||||
{"id":"a","type":"file","path":"` + filepath.Join(dir, "a.conf") + `","content":"x\n"}]}`)
|
||||
d, err := declaration.ParseFileTrusted(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sys, err := system.For("arch")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, adopted := range []bool{false, true} {
|
||||
o := Options{State: filepath.Join(dir, fmt.Sprintf("state-%v.json", adopted)), Adopted: adopted}
|
||||
if _, err := ApplyBundle(context.Background(), o, sys, d, raw, nil, quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
known, err := store.Load(o.State)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if known.Genesis == nil || known.Genesis.Digest != apply.DigestOf(raw) || !known.Genesis.Rewritten {
|
||||
t.Errorf("adopted=%v: genesis did not record the bundle it consumed: %+v", adopted, known.Genesis)
|
||||
}
|
||||
want := store.ModeConverged
|
||||
if adopted {
|
||||
want = store.ModeAdopted
|
||||
}
|
||||
if known.Mode != want {
|
||||
t.Errorf("adopted=%v: genesis recorded the mode as %q, want %q", adopted, known.Mode, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Re-run on a node the mesh has spoken to since — and converged — genesis leaves the mode
|
||||
// alone: it is the operator's word at genesis, and the controller's from then on.
|
||||
o := Options{State: filepath.Join(dir, "state-flipped.json"), Adopted: true}
|
||||
if err := store.Save(o.State, store.State{Mode: store.ModeConverged}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ApplyBundle(context.Background(), o, sys, d, raw, nil, quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if known, _ := store.Load(o.State); known.Mode != store.ModeConverged {
|
||||
t.Errorf("a genesis re-run set the mode back to %q over the mesh's converged", known.Mode)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -566,7 +566,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
||||
|
||||
// ---- 4. apply -----------------------------------------------------------------------
|
||||
say("apply — raising the foundation")
|
||||
report, err := ApplyBundle(ctx, o, sys, rewritten.Declaration, d.Run, say)
|
||||
report, err := ApplyBundle(ctx, o, sys, rewritten.Declaration, rewritten.Bundle, d.Run, say)
|
||||
result.Applied, result.Changed = len(report.Outcomes), report.Changed()
|
||||
if err != nil {
|
||||
return result, failed(StepApply, err)
|
||||
|
||||
@@ -89,7 +89,7 @@ func RetireTheTemporaryControlPlane(ctx context.Context, o Options, sys system.S
|
||||
// same state file. What makes this a removal rather than a no-op is that the state file
|
||||
// records the container as something this installer applied, and the declaration no longer
|
||||
// asks for it.
|
||||
report, err := ApplyBundle(ctx, o, sys, without, run, say)
|
||||
report, err := ApplyBundle(ctx, o, sys, without, bundle, run, say)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf(
|
||||
"%w\n\nThe permanent control plane is running and the temporary one is still here. "+
|
||||
|
||||
@@ -80,6 +80,25 @@ func SaveDeclared(path string, d Declared) error {
|
||||
return os.Rename(tmp.Name(), path)
|
||||
}
|
||||
|
||||
// ReadDeclared reads what was kept without proving it is the mesh's.
|
||||
//
|
||||
// For naming and comparing only — which declaration this node was last told, and what mode it
|
||||
// said — never for applying. A declaration to apply goes through LoadDeclared, which verifies.
|
||||
func ReadDeclared(path string) (Declared, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return Declared{}, ErrNothingDeclared
|
||||
}
|
||||
if err != nil {
|
||||
return Declared{}, fmt.Errorf("this node was told something and cannot read it back: %w", err)
|
||||
}
|
||||
var d Declared
|
||||
if err := json.Unmarshal(raw, &d); err != nil {
|
||||
return Declared{}, fmt.Errorf("what this node was told is unreadable at %s: %w", path, err)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// LoadDeclared reads it back and proves it is still the mesh's.
|
||||
//
|
||||
// Verified against the signing key this node holds, which came from its token. A declaration on
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// One apply at a time on a machine, whoever asks.
|
||||
//
|
||||
// Three things apply here and each reads the state, acts, and writes the state back: the link
|
||||
// and the reconcile loop inside `mesh-host run`, the host's own `reconcile` and `apply` run by
|
||||
// hand, and the installer at genesis. Inside one process a mutex serialises them; across
|
||||
// processes nothing did, and two applies interleaved leave the last saver writing a state read
|
||||
// before the other acted — a hold, a firewall record, a resource just applied, lost (novox/hq
|
||||
// issue 104 review). A lock on a file beside the state is what every one of them can take, however
|
||||
// it was started: a `reconcile` run against a service, or against a `mesh-host run` somebody
|
||||
// started by hand, waits the same way. Refusing while a named service is active would have known
|
||||
// the service's name and missed the hand-started one.
|
||||
//
|
||||
// An advisory lock, held for the life of the open file and released by the kernel when the
|
||||
// process ends, so a host that dies mid-apply leaves no lock behind for the next one to clear.
|
||||
|
||||
// LockName is the file the lock is taken on, beside the state.
|
||||
const LockName = "state.lock"
|
||||
|
||||
// Lock takes the machine's apply lock and returns what releases it. When another process holds
|
||||
// it, wait is told once — so a person running a command knows what they are waiting for — and
|
||||
// Lock blocks until it is free.
|
||||
func Lock(statePath string, wait func()) (func(), error) {
|
||||
dir := filepath.Dir(statePath)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := os.OpenFile(filepath.Join(dir, LockName), os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot take this node's apply lock: %w", err)
|
||||
}
|
||||
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
||||
if !errors.Is(err, syscall.EWOULDBLOCK) {
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("cannot take this node's apply lock: %w", err)
|
||||
}
|
||||
if wait != nil {
|
||||
wait()
|
||||
}
|
||||
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
|
||||
f.Close()
|
||||
return nil, fmt.Errorf("waiting for this node's apply lock: %w", err)
|
||||
}
|
||||
}
|
||||
return func() {
|
||||
_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
||||
f.Close()
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Defends the node's record across processes: a `reconcile` run by hand beside the link service,
|
||||
// or the installer beside either, waits for the other's apply rather than saving over it.
|
||||
func TestASecondApplyWaitsForTheFirst(t *testing.T) {
|
||||
state := filepath.Join(t.TempDir(), "state.json")
|
||||
release, err := Lock(state, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
waited := make(chan struct{}, 1)
|
||||
got := make(chan struct{})
|
||||
go func() {
|
||||
unlock, err := Lock(state, func() { waited <- struct{}{} })
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
close(got)
|
||||
unlock()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-waited:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the second apply was not told it is waiting")
|
||||
}
|
||||
select {
|
||||
case <-got:
|
||||
t.Fatal("the second apply took the lock while the first held it")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case <-got:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the second apply never got the lock once the first let go")
|
||||
}
|
||||
}
|
||||
@@ -104,6 +104,34 @@ type State struct {
|
||||
// Firewall is the firewall found on this machine when it was first adopted, and whether the
|
||||
// mesh has since retired it (novox/hq ADR 0100). Nil on a node that was never adopted.
|
||||
Firewall *FoundFirewall `json:"firewall,omitempty"`
|
||||
|
||||
// Mode is the node's mode as this host last recorded it: adopted or converged (novox/hq ADR
|
||||
// 0100). Empty on a machine nothing has said a mode to yet. Recorded from every declaration
|
||||
// the mesh sends and at genesis from what the operator said, so a declaration that says the
|
||||
// other mode can be refused before it is applied (novox/hq issue 104).
|
||||
Mode string `json:"mode,omitempty"`
|
||||
|
||||
// Genesis is the bundle this host consumed raising the foundation, if it has. Once recorded,
|
||||
// the bundle carried in the binary is not applied again: what genesis applied was rewritten
|
||||
// for this machine, and the mesh has said more since (novox/hq issue 104).
|
||||
Genesis *Genesis `json:"genesis,omitempty"`
|
||||
}
|
||||
|
||||
// Modes a node can be in (novox/hq ADR 0100).
|
||||
const (
|
||||
ModeAdopted = "adopted"
|
||||
ModeConverged = "converged"
|
||||
)
|
||||
|
||||
// Genesis is the bundle a host consumed raising this machine's foundation.
|
||||
type Genesis struct {
|
||||
// Digest is sha256 of the exact bytes applied.
|
||||
Digest string `json:"digest"`
|
||||
At time.Time `json:"at"`
|
||||
// Rewritten is true when the bytes applied were the carried bundle rewritten for this
|
||||
// machine — its foundation ports, root credentials, and adoption — so the bundle the binary
|
||||
// carries is not what was applied.
|
||||
Rewritten bool `json:"rewritten,omitempty"`
|
||||
}
|
||||
|
||||
// FoundFirewall is what the host found filtering this machine, and what it did about it.
|
||||
|
||||
Reference in New Issue
Block a user