novox/mesh-host
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refuse every file once the mesh has spoken, plan the cutover as one, and let the kept declaration repair the mode

Browse Source 
Review of the fix for hq issue 104 found three faults in it. A file applied
on an enrolled node — the mesh's own last declaration included — is applied
as the bundle is, so its resources are recorded as the machine's own and
what the mesh declared reads as undeclared: the plan removed the foundation.
`apply FILE` is for a machine the mesh has not spoken to, and is now refused
saying so whenever declared.json exists. The plan looked at what is held
before what the declaration says is taken, so the one cutover ADR 0100 says
must be previewed read as a hold; it now decides in holdOnAdopted's order,
models a step run inside a held container, and a test holds the plan's
sequence to the apply's outcomes. Genesis wrote the mode on every run, so a
re-run after `converge` left the state saying adopted while the kept,
signed declaration said converged, and the reconcile loop refused every five
minutes with no delivery coming to end it: genesis now writes the mode only
when none is recorded, and where the state and the verified kept declaration
disagree, the kept declaration wins and the repair is said.

Also: a file lock beside the state, taken by the link service, the host's
own commands and the installer alike, so a `reconcile` run by hand no
longer races the loop's save — chosen over refusing while a named service is
active, which would miss a `mesh-host run` started by hand; `--json
--dry-run` emits {plan} like an apply emits {plan, report}; the README's
duplicate flag line; and the bundle refusal is about the digest, not a claim
the carried bytes can never match what genesis applied.
This commit is contained in:
jschoubben
2026-09-23 23:35:49 +02:00
parent 27c4b765b2
commit f08a8ea3f7
9 changed files with 441 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/main.go
+80 -44
View File
@@ -398,10 +398,12 @@ func short(digest string) string {
// **A declaration carries no order.** The controller signs a version — the vocabulary — the
// node it is for, the mode, and the resources: no sequence, no issued-at. What exists is
// identity: the mesh names what it sends by the digest of the bytes, the node keeps the last one
// it was sent, and genesis records the digest of what it consumed. So "older" can be said of
// exactly two things — the bundle genesis consumed, and a bundle other than the one it consumed —
// and of anything else only that it is not what the mesh last said, which is refused too: a
// host that cannot tell older from newer and applies anyway is the fault this issue names.
// it was sent, and genesis records the digest of what it consumed. So the bundle is held to
// genesis's digest, and a file is not applied at all once the mesh has spoken: a file is applied
// as the bundle is, its resources recorded as this machine's own — so the mesh could never remove
// them again — and everything the mesh declared read as no longer declared and removed. Even the
// very declaration the mesh last sent, applied from a file, would plan to remove the foundation.
// `apply FILE` is for a machine the mesh has not spoken to, and is refused saying so.
func refuseStale(known store.State, kept store.Declared, keptErr error, digest string, from provenance) error {
switch from {
case fromDeclared:
@@ -410,43 +412,45 @@ func refuseStale(known store.State, kept store.Declared, keptErr error, digest s
if known.Genesis == nil || known.Genesis.Digest == digest {
return nil
}
how := ""
if known.Genesis.Rewritten {
how = ", rewritten for this machine — its foundation ports, root credentials and mode"
}
// By digest. Genesis applies the bundle rewritten for this machine — its foundation
// ports, root credentials, mode — and writes that lock out; a host built from the
// carried template does not match it, and one built from the written lock does.
return fmt.Errorf("the bundle this host carries (%s) is not the one genesis applied here on %s "+
"(%s%s). The bundle was consumed then, and nothing the mesh has said is kept on this node "+
"yet, so there is nothing to reconcile against: enrol this node, or push to it from the "+
"controller", short(digest), known.Genesis.At.Format(time.RFC3339), short(known.Genesis.Digest), how)
"(%s), which was the bundle rewritten for this machine. It was consumed then, and nothing "+
"the mesh has said is kept on this node yet, so there is nothing to reconcile against: "+
"enrol this node, or push to it from the controller",
short(digest), known.Genesis.At.Format(time.RFC3339), short(known.Genesis.Digest))
}
// A file.
switch {
case errors.Is(keptErr, store.ErrNothingDeclared):
// The mesh has said nothing here. Nothing to be older than.
// The mesh has said nothing here: a machine a file is for.
return nil
case keptErr != nil:
return fmt.Errorf("%w\n\nNothing is applied over what this node cannot read back", keptErr)
}
last := apply.DigestOf(kept.Declaration)
if digest == last {
return nil
what := fmt.Sprintf("this file (%s) is not it", short(digest))
switch {
case digest == last:
what = "this file is that declaration, and from a file it would still be applied as a bundle is"
case known.Genesis != nil && digest == known.Genesis.Digest:
what = fmt.Sprintf("this file is the bundle genesis consumed on %s (%s), older than it",
known.Genesis.At.Format(time.RFC3339), short(digest))
}
if known.Genesis != nil && digest == known.Genesis.Digest {
return fmt.Errorf("this file is the bundle genesis consumed on %s (%s), and the mesh has since told "+
"this node declaration %s, kept as %s. An older declaration is not applied over a newer one: "+
"`reconcile` applies what the mesh last said", known.Genesis.At.Format(time.RFC3339), short(digest),
short(last), store.DeclaredName)
}
return fmt.Errorf("this file (%s) is not the declaration the mesh last told this node (%s, kept as %s). "+
"A declaration carries no sequence and no issued-at, so this host cannot tell an older one from a "+
"newer, and it applies only what the mesh last said: `reconcile` applies that, and `push` from the "+
"controller changes it", short(digest), short(last), store.DeclaredName)
return fmt.Errorf("`apply FILE` is for a machine the mesh has not spoken to. The mesh has told this "+
"node declaration %s, kept as %s, and %s. A file is applied as the bundle is — its resources "+
"recorded as this machine's own, which the mesh could then never remove, and what the mesh "+
"declared read as no longer declared — so no file is applied here: `reconcile` applies what "+
"the mesh last said, and `push` from the controller changes it",
short(last), store.DeclaredName, what)
}
// applied is what an apply says in machine-readable form: what it planned, then what it did.
type applied struct {
Plan []apply.Step `json:"plan"`
Report apply.Report `json:"report"`
Plan []apply.Step `json:"plan"`
// Report is absent on a dry run, which is the plan and nothing more.
Report *apply.Report `json:"report,omitempty"`
}
// runApply makes the machine match a declaration — after refusing one this node must not apply,
@@ -465,16 +469,34 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
if out == nil {
out = os.Stdout
}
// One apply at a time on this machine, whichever process asks: the link service applies too,
// and two saves of the state interleaved lose what one of them recorded.
unlock, err := store.Lock(opts.state, func() {
fmt.Fprintln(out, "another apply holds this node's state — mesh-host run, or the installer; waiting for it to finish")
})
if err != nil {
return err
}
defer unlock()
known, err := store.Load(opts.state)
if err != nil {
return err
}
source, origin, digest := from.name(opts), from.origin(), apply.DigestOf(raw)
// The mode this node is in. Recorded in the state since this check existed; a state written
// before then has it in what the mesh last said, which this node kept.
// The mode this node is in. What the mesh last said is signed and was verified on load; the
// state's note of it is this host's own, and where they disagree the note is what is wrong
// — a genesis re-run over a node the mesh converged since, a state saved when the kept
// declaration could not be — and is repaired, not obeyed. A bundle or a file is held to the
// note, or to the kept declaration when the note predates it.
kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state))
if known.Mode == "" && keptErr == nil {
if from == fromDeclared {
if known.Mode != "" && known.Mode != apply.ModeOf(d) {
fmt.Fprintf(out, "this node's record said %s; what the mesh last said, signed, says %s — the record is repaired\n",
known.Mode, apply.ModeOf(d))
}
known.Mode = apply.ModeOf(d)
} else if known.Mode == "" && keptErr == nil {
if last, err := declaration.Parse(kept.Declaration); err == nil {
known.Mode = apply.ModeOf(last)
}
@@ -489,7 +511,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
// Said before anything is done.
steps := apply.Plan(d, known, origin)
if opts.json && opts.dryRun {
return writeJSONTo(out, steps)
return writeJSONTo(out, applied{Plan: steps})
}
mode := known.Mode
if mode == "" {
@@ -572,7 +594,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
}
if opts.json {
return writeJSONTo(out, applied{Plan: steps, Report: report})
return writeJSONTo(out, applied{Plan: steps, Report: &report})
}
if !report.Changed() {
fmt.Fprintf(out, "%s: already matches — %d resource(s) checked\n", source, len(report.Outcomes))
@@ -821,7 +843,7 @@ func runLink(ctx context.Context, opts options) error {
go sched.Run(ctx)
applier := func(ctx context.Context, raw, signature []byte) link.Report {
return applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched)
return applyAndKeep(ctx, opts, raw, &store.Declared{Declaration: raw, Signature: signature}, sched, say)
}
// Two things at once, and the second is what makes disconnection ordinary. The link brings
@@ -984,7 +1006,7 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
continue
}
report := applyDeclared(ctx, opts, declared, sched)
report := applyDeclared(ctx, opts, declared, sched, say)
// A reconcile is otherwise silent. On an adopted node it speaks when what it holds or
// its firewall changed, because that is how a predecessor still writing is caught
// (novox/hq ADR 0100); publish decides whether anything did.
@@ -1005,25 +1027,33 @@ func holdTheMachine(ctx context.Context, opts options, mine identity.Identity, s
// Signature checking happens before this is called, in the link. By the time anything here runs,
// the question "is this from the mesh I joined" is settled — which is why this can treat the
// bytes as instructions.
func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.Scheduler) link.Report {
return applyAndKeep(ctx, opts, raw, nil, sched)
func applyDeclared(ctx context.Context, opts options, raw []byte, sched *apply.Scheduler, say link.Announce) link.Report {
return applyAndKeep(ctx, opts, raw, nil, sched, say)
}
// applying serialises applies on this node.
// applying serialises applies within this process.
//
// **Two things apply here: the link and the reconcile loop**, and each reads the node's state,
// acts on the machine, and writes the state back. Run at the same time they interleave, and the
// one that saves last writes a state read before the other acted — losing what the first recorded:
// a hold, the firewall found here, a resource just applied. The machine would then be one thing
// and its record another, which is the fault every read-back in this package exists to prevent.
// Across processes — `reconcile` run by hand beside this service — the lock beside the state does
// the same (store.Lock).
var applying sync.Mutex
// applyAndKeep applies a declaration and, when it came from the mesh, keeps it so this node can
// go on obeying it while disconnected. One at a time, whoever asks.
// go on obeying it while disconnected. One at a time, whoever asks. Given unsigned, the bytes are
// what this node kept, already verified against the mesh's key on load.
func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared,
sched *apply.Scheduler) link.Report {
sched *apply.Scheduler, say link.Announce) link.Report {
applying.Lock()
defer applying.Unlock()
unlock, err := store.Lock(opts.state, nil)
if err != nil {
return link.Report{Refused: err.Error()}
}
defer unlock()
declared, err := declaration.Parse(raw)
if err != nil {
@@ -1034,13 +1064,19 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D
if err != nil {
return link.Report{Refused: err.Error()}
}
// A declaration the link delivered is the controller's word on the node's mode — the flip
// arrives as exactly that, the first converged declaration after adopted ones — and becomes
// the record. What this node re-applies on its own is held to the record (novox/hq issue 104).
if signed == nil {
if err := apply.CheckMode(known, declared); err != nil {
return link.Report{Refused: err.Error()}
// A declaration from the mesh is the controller's word on the node's mode — the flip arrives
// as exactly that, the first converged declaration after adopted ones — and becomes the
// record. So does what this node kept: it is signed by the mesh and verified on load, where
// the state's note of the mode is this host's own. Where they disagree the note is wrong — a
// genesis re-run over a node the mesh converged since, a state saved when the kept declaration
// could not be — and it is repaired and said, not obeyed: refusing would hold this node off what
// the mesh said until a delivery that comes only when something changes (novox/hq issue 104).
if signed == nil && known.Mode != "" && known.Mode != apply.ModeOf(declared) {
if say != nil {
say(fmt.Sprintf("this node's record said %s; what the mesh last said, signed, says %s — the record is repaired",
known.Mode, apply.ModeOf(declared)))
}
known.Mode = apply.ModeOf(declared)
}
built, err := system.For(builtFor)
cmd/mesh-host/main_test.go
+45 -13
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"os"
"path/filepath"
@@ -224,7 +225,7 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) {
// Whatever else is applying — the link, while this is the reconcile — this waits for it.
applying.Lock()
done := make(chan link.Report, 1)
go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil) }()
go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil, nil) }()
select {
case report := <-done:
applying.Unlock()
@@ -307,16 +308,23 @@ func TestAConvergedDeclarationIsRefusedOnAnAdoptedNode(t *testing.T) {
}
}
func TestAnAdoptedDeclarationIsRefusedOnAConvergedNode(t *testing.T) {
// Only the mesh can say a node is adopted, so this one arrives the way a disconnected node
// re-applies what it kept: through the reconcile loop, unsigned.
func TestTheKeptDeclarationRepairsARecordThatDisagrees(t *testing.T) {
// What a node kept is signed by the mesh and verified on load; the state's note of the mode is
// the host's own. A genesis re-run after `converge`, or a state saved when the kept declaration
// could not be, leaves them apart — and a loop that refused every five minutes would hold the
// node off what the mesh said until a delivery that comes only when something changes. The
// kept declaration wins, and the repair is said.
opts := stateWithMode(t, store.ModeConverged)
raw := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"` +
filepath.Join(filepath.Dir(opts.state), "a.conf") + `","content":"x\n"}]}`)
report := applyAndKeep(context.Background(), opts, raw, nil, nil)
want := "this node is converged; the declaration says adopted"
if !strings.Contains(report.Refused, want) || !strings.Contains(report.Refused, "`adopt`") {
t.Errorf("refused = %q, want it to name both modes and the act that changes it", report.Refused)
var said []string
report := applyAndKeep(context.Background(), opts, raw, nil, nil, func(line string) { said = append(said, line) })
if strings.Contains(report.Refused, "the declaration says") {
t.Fatalf("what the node kept was refused against its own note: %s", report.Refused)
}
if len(said) != 1 || !strings.Contains(said[0], "record said converged") ||
!strings.Contains(said[0], "says adopted") || !strings.Contains(said[0], "repaired") {
t.Errorf("the repair was not said: %q", said)
}
}
@@ -328,15 +336,16 @@ func TestTheMeshItselfMayChangeTheMode(t *testing.T) {
opts := stateWithMode(t, store.ModeAdopted)
raw := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` +
filepath.Join(filepath.Dir(opts.state), "a.conf") + `","content":"x\n"}]}`)
report := applyAndKeep(context.Background(), opts, raw, &store.Declared{Declaration: raw}, nil)
report := applyAndKeep(context.Background(), opts, raw, &store.Declared{Declaration: raw}, nil, nil)
if strings.Contains(report.Refused, "the declaration says") {
t.Errorf("the mesh's own flip was refused for its mode: %s", report.Refused)
}
}
// Defends novox/hq issue 104: a declaration older than what the mesh last said is refused, naming
// both — and a declaration carries no order, so one that is merely not the last is refused too,
// saying what is missing.
// both — and `apply FILE` is refused altogether once the mesh has spoken, the last declaration
// itself included: from a file it is applied as the bundle is, which would record the mesh's
// resources as this machine's own and remove the foundation as undeclared.
func TestAnOlderDeclarationIsRefused(t *testing.T) {
opts := stateWithMode(t, "")
genesis := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"genesis\n"}]}`)
@@ -372,9 +381,18 @@ func TestAnOlderDeclarationIsRefused(t *testing.T) {
if err == nil {
t.Fatal("a declaration that is not what the mesh last said was applied")
}
if !strings.Contains(err.Error(), "no sequence and no issued-at") ||
if !strings.Contains(err.Error(), "for a machine the mesh has not spoken to") ||
!strings.Contains(err.Error(), short(apply.DigestOf(since))) {
t.Errorf("the refusal does not say what is missing and what was last said: %v", err)
t.Errorf("the refusal does not say what a file is for and what was last said: %v", err)
}
// The very declaration the mesh last sent, from a file: still a file.
err = runApply(context.Background(), opts, parsed(since), since, fromFile)
if err == nil || !strings.Contains(err.Error(), "applied as the bundle is") {
t.Errorf("the last declaration, from a file, was not refused as a file: %v", err)
}
if known, _ := store.Load(opts.state); len(known.Resources) != 0 {
t.Errorf("a refused file recorded %d resource(s)", len(known.Resources))
}
// A bundle other than the one genesis consumed: what genesis applied was rewritten for this
@@ -416,6 +434,20 @@ func TestADryRunChangesNothingAndListsTheActions(t *testing.T) {
if strings.Contains(out.String(), "applying:") {
t.Errorf("a dry run went on to apply:\n%s", out.String())
}
// Machine-readable, the same shape as an apply's: the plan, and no report.
out.Reset()
opts.json = true
if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil {
t.Fatal(err)
}
var shape struct {
Plan []apply.Step `json:"plan"`
Report *json.RawMessage `json:"report"`
}
if err := json.Unmarshal(out.Bytes(), &shape); err != nil || len(shape.Plan) != 2 || shape.Report != nil {
t.Errorf("a json dry run is not {plan} alone: %v\n%s", err, out.String())
}
}
// Defends novox/hq issue 104: once the mesh has told this node anything, `reconcile` holds it to