novox/mesh-host
6
0
Fork 0
Code Issues Pull Requests 2 Releases Wiki Activity

Merge pull request 'Refuse a declaration composed from an older assignment generation than the machine applied (issue 234)' (#82) from fix/234-refuse-a-declaration-from-an-older-generation into main

Browse Source
This commit was merged in pull request #82.
This commit is contained in:
mesh-admin
2026-10-11 16:00:01 +00:00
parent 2833caf3d6 20d5af9b2d
commit 6defec8f68
9 changed files with 365 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/generation_floor_test.go
+40
View File
@@ -0,0 +1,40 @@
package main
import (
"context"
"testing"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// **The generation applied is kept, and a put-back never lowers it**: after a put-back of generation 5
// over 7, a send from generation 6 is still older than what this machine applied.
func TestThePutBackDoesNotLowerTheGenerationApplied(t *testing.T) {
built, err := system.For("arch")
if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil {
t.Skip("this machine is not one these tests can apply on")
}
was := builtFor
builtFor = "arch"
t.Cleanup(func() { builtFor = was })
f := machineThatApplied(t, 7)
newer := f.declared(t, map[string]any{"sequence": 9, "generation": 8}, "docker.conf", f.stays)
if r := applyAndKeep(context.Background(), f.opts, newer.Declaration, &newer, nil, nil); r.Refused != "" {
t.Fatalf("a newer generation was refused: %s", r.Refused)
}
back := f.declared(t, map[string]any{"sequence": 10, "generation": 5, "put_back": true}, "docker.conf", f.stays)
if r := applyAndKeep(context.Background(), f.opts, back.Declaration, &back, nil, nil); r.Refused != "" {
t.Fatalf("a put-back was refused: %s", r.Refused)
}
state, err := store.Load(f.opts.state)
if err != nil || state.Generation != 8 {
t.Fatalf("the generation applied is %d after a put-back of 5 over 8 (%v)", state.Generation, err)
}
stale := f.declared(t, map[string]any{"sequence": 11, "generation": 7}, "docker.conf", f.stays)
if r := applyAndKeep(context.Background(), f.opts, stale.Declaration, &stale, nil, nil); r.OlderGeneration == nil {
t.Fatalf("a send from generation 7 after 8 was applied because a put-back came between: %+v", r)
}
}
cmd/mesh-host/generation_test.go
+170
View File
@@ -0,0 +1,170 @@
package main
import (
"context"
"crypto/ed25519"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
)
// The assignment generation a declaration was composed from (novox/hq issue 234).
//
// On 2026-10-04 a declaration newer in sequence than every other, and composed from a view of the
// assignments older than the one this machine had applied, named four fewer modules — and the host,
// ordering by sequence alone, undeclared all four. The sequence orders arrival; it cannot tell a later
// send carrying earlier content. The generation can: the node-engine refuses a declaration composed from
// an older generation than the one it applied, before anything is read from the machine.
//
// These read what the report says through its JSON, as the controller does, so they run against a
// node-engine that has never heard of a generation and fail there for that reason.
// generationFixture is a machine that applied generation 7: a file the mesh declared, kept in its state,
// and the declaration it kept — signed — that named it.
type generationFixture struct {
opts options
private ed25519.PrivateKey
kept store.Declared
// stays is the file the older generation's declaration does not name.
stays string
}
func machineThatApplied(t *testing.T, generation int64) generationFixture {
t.Helper()
dir := t.TempDir()
opts := options{state: filepath.Join(dir, "state.json")}
_, private, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
stays := filepath.Join(dir, "docker.conf")
if err := os.WriteFile(stays, []byte("kept\n"), 0o644); err != nil {
t.Fatal(err)
}
// The state as a node-engine that applied generation 7 keeps it: the file it owns, and the highest
// generation it applied. Written as JSON, so the fixture is the same bytes for any build under test.
state := map[string]any{"resources": []any{map[string]any{"id": "docker.conf", "type": "file",
"origin": store.OriginDeclared, "target": stays}}, "generation": generation}
raw, err := json.Marshal(state)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(opts.state, raw, 0o600); err != nil {
t.Fatal(err)
}
f := generationFixture{opts: opts, private: private, stays: stays}
f.kept = f.declared(t, map[string]any{"sequence": 3, "generation": generation}, "docker.conf", stays)
if err := store.SaveDeclared(store.DeclaredPath(opts.state), f.kept); err != nil {
t.Fatal(err)
}
return f
}
// declared is a signed declaration with the envelope's keys given and one file.
func (f generationFixture) declared(t *testing.T, envelope map[string]any, id, path string) store.Declared {
t.Helper()
body := map[string]any{"declaration": 1,
"resources": []any{map[string]any{"id": id, "type": "file", "path": path, "content": "x\n"}}}
for k, v := range envelope {
body[k] = v
}
raw, err := json.Marshal(body)
if err != nil {
t.Fatal(err)
}
return store.Declared{Declaration: raw, Signature: ed25519.Sign(f.private, raw)}
}
// reportJSON is a report as the controller reads it.
func reportJSON(t *testing.T, r link.Report) map[string]any {
t.Helper()
raw, err := json.Marshal(r)
if err != nil {
t.Fatal(err)
}
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
t.Fatal(err)
}
return out
}
// **A newer sequence from an older generation is refused, and nothing is undeclared** (issue 234).
func TestANewerSequenceFromAnOlderGenerationIsRefusedAndNothingIsUndeclared(t *testing.T) {
f := machineThatApplied(t, 7)
other := filepath.Join(filepath.Dir(f.opts.state), "sudo.conf")
stale := f.declared(t, map[string]any{"sequence": 9, "generation": 6}, "sudo.conf", other)
report := applyAndKeep(context.Background(), f.opts, stale.Declaration, &stale, nil, nil)
said := reportJSON(t, report)
older, _ := said["older_generation"].(map[string]any)
if older == nil || older["generation"] != float64(6) || older["applied"] != float64(7) {
t.Fatalf("the refusal does not name the generation refused and the one applied: %v", said)
}
if report.Refused == "" || !strings.Contains(report.Refused, "generation 6") ||
!strings.Contains(report.Refused, "generation 7") {
t.Errorf("the refusal is not said in words naming both generations: %q", report.Refused)
}
if report.Declared != digestOf(stale.Declaration) || report.Sequence != 9 {
t.Errorf("the refusal does not name the declaration refused, so the mesh cannot name who sent it: %+v", report)
}
if _, err := os.Stat(f.stays); err != nil {
t.Fatalf("a refused declaration undeclared what the machine applied: %v", err)
}
if _, err := os.Stat(other); err == nil {
t.Error("the refused declaration touched the machine")
}
state, err := store.Load(f.opts.state)
if err != nil {
t.Fatal(err)
}
if _, owned := state.Find("docker.conf"); !owned {
t.Error("the refused declaration took a resource out of what this machine owns")
}
kept, err := store.ReadDeclared(store.DeclaredPath(f.opts.state))
if err != nil || string(kept.Declaration) != string(f.kept.Declaration) {
t.Error("the refused declaration replaced what this node kept, so the next reconcile would apply it")
}
}
// refusedForItsGeneration says a report refused its declaration for the generation it came from.
func refusedForItsGeneration(t *testing.T, r link.Report) bool {
t.Helper()
return reportJSON(t, r)["older_generation"] != nil || strings.Contains(r.Refused, "generation")
}
// refusedAsUnread says a report refused its declaration as one this node-engine cannot read.
func refusedAsUnread(r link.Report) bool { return strings.Contains(r.Refused, "not a declaration") }
// **A gate's put-back is applied, whatever generation it carries**: it carries the generation of what it
// puts back, and says it is a put-back (issue 234).
func TestAPutBackFromAnOlderGenerationIsNotRefused(t *testing.T) {
f := machineThatApplied(t, 7)
back := f.declared(t, map[string]any{"sequence": 9, "generation": 5, "put_back": true}, "docker.conf", f.stays)
report := applyAndKeep(context.Background(), f.opts, back.Declaration, &back, nil, nil)
if refusedForItsGeneration(t, report) || refusedAsUnread(report) {
t.Fatalf("a put-back was refused: %s", report.Refused)
}
}
// **A declaration that carries no generation is applied as it always was**: an older controller sends
// none, and a node-engine that refused it would strand every machine the moment the controller is older.
func TestADeclarationWithoutAGenerationIsNotRefusedForOne(t *testing.T) {
f := machineThatApplied(t, 7)
older := f.declared(t, map[string]any{"sequence": 9}, "docker.conf", f.stays)
report := applyAndKeep(context.Background(), f.opts, older.Declaration, &older, nil, nil)
if refusedForItsGeneration(t, report) || refusedAsUnread(report) {
t.Fatalf("a declaration without a generation was refused: %s", report.Refused)
}
// And the same generation again is no older.
same := f.declared(t, map[string]any{"sequence": 10, "generation": 7}, "docker.conf", f.stays)
report = applyAndKeep(context.Background(), f.opts, same.Declaration, &same, nil, nil)
if refusedForItsGeneration(t, report) || refusedAsUnread(report) {
t.Fatalf("a declaration of the generation applied was refused: %s", report.Refused)
}
}
cmd/mesh-host/main.go
+47
View File
@@ -595,6 +595,11 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
if err := refuseStale(known, kept, keptErr, digest, from, link.Order{Epoch: d.Epoch, Sequence: d.Sequence}); err != nil {
return err
}
// The generation, as the link's deliveries are held to it (novox/hq issue 234): one composed from an
// older view of the assignments than this machine applied would undeclare what the mesh still assigns.
if refused := olderGeneration(known, d); refused != nil {
return errors.New(olderGenerationWords(*refused))
}
// Said before anything is done.
steps := apply.Plan(d, known, origin)
@@ -646,6 +651,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
// What this apply settles about the node, whichever way it went. The mode is what the
// declaration said and the check above agreed with; the bundle, once applied, is consumed.
updated.Mode = apply.ModeOf(d)
updated.Generation = appliedGeneration(known, d) // raised, never lowered (novox/hq issue 234)
if from == fromBundle {
updated.Genesis = &store.Genesis{Digest: digest, At: time.Now().UTC()}
}
@@ -1887,6 +1893,16 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
if err != nil {
return link.Report{Refused: err.Error()}
}
// **Composed from an older view of the assignments than this machine applied is refused, before
// anything is read from the machine** (novox/hq issue 234). A newer sequence carrying an older view
// undeclared four assigned modules on 2026-10-04; the sequence cannot tell it, the generation can.
// Only a delivery: what a reconcile applies is what was kept.
if signed != nil {
if refused := olderGeneration(known, declared); refused != nil {
return link.Report{Declared: digestOf(raw), Order: order, OlderGeneration: refused,
Refused: olderGenerationWords(*refused)}
}
}
// A declaration from the mesh is the controller's word on the node's mode — the flip arrives
// as exactly that, the first converged declaration after adopted ones — and becomes the
// record. So does what this node kept: it is signed by the mesh and verified on load, where
@@ -1933,6 +1949,8 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
// The mode the mesh said, recorded whichever way the apply went: the declaration is kept
// either way, and the node is held to it from the next reconcile (novox/hq ADR 0100).
updated.Mode = apply.ModeOf(declared)
// And the generation it came from, the same way, raised and never lowered (novox/hq issue 234).
updated.Generation = appliedGeneration(known, declared)
// Root acted on what an agent account's judgement reads — an account, a package, an action, sudo's
// rules: the search for setuid programs runs again (novox/hq issue 361). A failed apply counts what it
@@ -2253,6 +2271,35 @@ func refuseOlder(kept store.Declared, keptErr error, order link.Order) error {
return nil
}
// olderGeneration is the refusal of a declaration composed from an older assignment generation than
// this machine applied, nil when it is not one (novox/hq issue 234).
//
// Only when both claim a generation: a declaration without one is from a controller that claims none,
// and a machine that applied none has nothing to hold it to — refusing on a guess would strand every
// machine the moment the controller is older than the node-engine. Equal is not older: a push of the
// same assignments is the ordinary case. A put-back carries the generation of what it puts back and
// says so; a gate putting a machine back is the one sender that may say something older.
func olderGeneration(known store.State, d *declaration.Declaration) *link.GenerationRefusal {
if d.PutBack || d.Generation <= 0 || known.Generation <= 0 || d.Generation >= known.Generation {
return nil
}
return &link.GenerationRefusal{Generation: d.Generation, Applied: known.Generation}
}
// olderGenerationWords is a generation refusal as the report and a person read it.
func olderGenerationWords(r link.GenerationRefusal) string {
return fmt.Sprintf("this declaration was composed from assignment generation %d, and this node applied "+
"generation %d: it was composed from a view of the assignments the mesh has moved past, and applying it "+
"would undeclare what the mesh still assigns here (novox/hq issue 234). Refused whole; nothing was "+
"applied", r.Generation, r.Applied)
}
// appliedGeneration is the highest generation applied once this declaration is: raised by it, never
// lowered — a put-back of an older generation leaves it where it was (novox/hq issue 234).
func appliedGeneration(known store.State, d *declaration.Declaration) int64 {
return max(known.Generation, d.Generation)
}
// olderThanKept says whether a declaration of this order is older than the one this node kept — the
// last it applied — and the kept one's order (link.Order.Older: only when both claim an epoch). A kept
// declaration that cannot be read claims no order: what it is compared with is unknown, and refusing
internal/declaration/declaration.go
+29 -1
View File
@@ -1378,6 +1378,20 @@ type Declaration struct {
// what it wrote for it — its resources are absent from the declaration, and absence would
// otherwise read as removal.
LeftOut []string
// Generation is the assignment generation this declaration was composed from: the controller's
// counter, raised in the same transaction as every change to what is assigned where (novox/hq
// issue 234). The sequence orders arrival and cannot tell a later send that carries an older view
// of the assignments; this can. A node-engine refuses a declaration composed from a generation
// older than the highest it applied — unless it is a put-back — because applying it would
// undeclare what the mesh still assigns. Zero is a declaration from a controller that claims none,
// and is applied as before.
Generation int64
// PutBack says this declaration is a gate putting a machine back (novox/hq issue 234): it carries
// the generation of what it puts back, which may be older than what the machine applied, and is
// not refused for it.
PutBack bool
}
// LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the
@@ -1542,6 +1556,11 @@ type envelope struct {
Epoch int64 `json:"epoch,omitempty"`
// LeftOut is optional on the wire too, and absent when nothing was left out (ADR 0163).
LeftOut []string `json:"left_out,omitempty"`
// Generation and PutBack are optional on the wire too (novox/hq issue 234): absent is a controller
// that claims no generation. **An older host refuses these keys**, decoding strictly; a controller
// sends them only to a host whose reports say `reads_generation`.
Generation int64 `json:"generation,omitempty"`
PutBack bool `json:"put_back,omitempty"`
}
func parse(raw []byte, allowActions bool) (*Declaration, error) {
@@ -1559,8 +1578,17 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
}
d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence,
Epoch: env.Epoch, LeftOut: env.LeftOut}
Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack}
var problems []string
if env.Generation < 0 {
// Below zero would read as "none claimed" and pass every refusal of an older generation.
problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+
"mesh assigns", env.Generation))
}
if env.PutBack && allowActions {
// A put-back is a gate's, sent by the mesh; a carried bundle puts nothing back.
problems = append(problems, "a carried bundle says it is a put-back, and only the mesh's gate can say that")
}
if env.Sequence < 0 || env.Epoch < 0 {
// Below zero is no order any controller assigns, and read as "none claimed" it would let the
// declaration past every refusal of what is older.
internal/declaration/order_test.go
+22
View File
@@ -35,3 +35,25 @@ func TestAnOrderBelowZeroIsRefused(t *testing.T) {
}
}
}
// **The assignment generation it was composed from, and whether it is a put-back** (novox/hq issue 234):
// absent is no claim; below zero is none the mesh assigns; a carried bundle puts nothing back.
func TestADeclarationCarriesItsGenerationAndWhetherItIsAPutBack(t *testing.T) {
d, err := Parse([]byte(withOrder(`"sequence":3,"generation":41,"put_back":true,`)))
if err != nil {
t.Fatalf("a declaration carrying its generation was refused: %v", err)
}
if d.Generation != 41 || !d.PutBack {
t.Fatalf("read generation %d, put-back %v", d.Generation, d.PutBack)
}
if d, err = Parse([]byte(withOrder(``))); err != nil || d.Generation != 0 || d.PutBack {
t.Fatalf("a declaration with no generation read as %+v, %v", d, err)
}
if refusal := refusalFor(t, withOrder(`"generation":-2,`)); !strings.Contains(refusal.Error(), "below zero") {
t.Errorf("a generation below zero was refused for something else: %v", refusal)
}
if _, err := ParseTrusted([]byte(withOrder(`"generation":3,"put_back":true,`))); err == nil ||
!strings.Contains(err.Error(), "put-back") {
t.Errorf("a carried bundle said it was a put-back and was not refused for it: %v", err)
}
}
internal/link/messages.go
+18
View File
@@ -128,6 +128,17 @@ type Report struct {
// read from the next one: what the mesh's stale-writer watchdog (to-be 45 §3, S13) counts.
RefusedOlder int64 `json:"refused_older,omitempty"`
// ReadsGeneration says this node-engine reads a declaration's assignment generation and its
// put-back mark (novox/hq issue 234), on every report it makes: the controller sends them only to a
// machine that said so, because an older node-engine decodes strictly and refuses a key it does not
// know, whole. A node-engine rolled back says so no longer.
ReadsGeneration bool `json:"reads_generation,omitempty"`
// OlderGeneration is set on a report refusing a declaration composed from an older assignment
// generation than this machine applied (novox/hq issue 234). The refused declaration is the report's
// own Declared and Order, so the controller can name who sent it from its record of sends.
OlderGeneration *GenerationRefusal `json:"older_generation,omitempty"`
// Held is what this adopted node found and is keeping as it was until its module is taken
// (novox/hq ADR 0100). Without it an adopted node reads as converged.
Held []Held `json:"held,omitempty"`
@@ -634,3 +645,10 @@ type FoundFirewall struct {
Active bool `json:"active"`
RetiredBy string `json:"retired_by,omitempty"`
}
// GenerationRefusal is a declaration refused for the assignment generation it was composed from: that
// generation, and the highest this machine applied (novox/hq issue 234).
type GenerationRefusal struct {
Generation int64 `json:"generation"`
Applied int64 `json:"applied"`
}
internal/link/queue.go
+10 -1
View File
@@ -211,6 +211,12 @@ func (q *Queue) step(ctx context.Context) bool {
q.Say(fmt.Sprintf("refused declaration %s (%s): this node applied %s, from a newer lease "+
"holder — %d refused as older so far", short(report.Declared), report.Order.Words(),
report.OlderThan.Words(), total))
case report.OlderGeneration != nil:
// Said here, and the report names it, so the controller raises it naming the sender (novox/hq
// issue 234). Not counted with the lease's refusals: those are S13's, by epoch.
q.Say(fmt.Sprintf("refused declaration %s (%s): it was composed from assignment generation %d, and "+
"this node applied generation %d", short(report.Declared), report.Order.Words(),
report.OlderGeneration.Generation, report.OlderGeneration.Applied))
case report.Refused != "":
q.Say("refused a declaration: " + report.Refused)
case len(report.Failed) > 0:
@@ -228,7 +234,7 @@ func (q *Queue) step(ctx context.Context) bool {
// A refusal as older is not kept to be said again: what the mesh waits for from this node is the
// account of the last apply, and that is still what is kept.
kind := applied
if report.OlderThan != nil {
if report.OlderThan != nil || report.OlderGeneration != nil {
kind = refusedOlder
}
q.tell(ctx, report, kind, latest)
@@ -336,6 +342,9 @@ func (q *Queue) stamp(r Report) Report {
r.Node = q.Membership.Node
r.ReportSequence = q.numbers.sequence
r.RefusedOlder = q.numbers.refused
// Every report of a node-engine that orders its reports says it reads a generation (novox/hq issue
// 234): this build does, and the controller sends one only to a machine that said so.
r.ReadsGeneration = true
return r
}
internal/link/queue_test.go
+23
View File
@@ -540,3 +540,26 @@ func TestALinkIsNotHeldForAReconcileInHand(t *testing.T) {
t.Fatalf("a report was said on a link let go: %+v", l.said())
}
}
// **Every report says this node-engine reads a declaration's generation** (novox/hq issue 234): the
// controller sends one only to a machine that said so, because an older node-engine refuses the key.
func TestEveryReportSaysItReadsAGeneration(t *testing.T) {
m, key := aMember(t)
l := newQuietLink()
a := &applies{}
q := aQueue(m, a.apply, &keptInMemory{})
q.attach(context.Background(), l)
q.Deliver(ordered(t, key, "new", 7, 12))
ctx, stop := context.WithCancel(context.Background())
worker := runWorker(ctx, q)
eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 })
stop()
<-worker
raw, err := json.Marshal(l.said()[0])
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(raw), `"reads_generation":true`) {
t.Fatalf("the report does not say it reads a generation: %s", raw)
}
}
internal/store/store.go
+6
View File
@@ -187,6 +187,12 @@ type Into struct {
// State is the whole of what a node knows about what it has done.
type State struct {
// Generation is the highest assignment generation this machine applied a declaration from
// (novox/hq issue 234): a declaration from an older one is refused unless it is a put-back. Only
// ever raised — a put-back of an older generation leaves it — so a put-back cannot open the door to
// the stale send it protects against. Zero on a machine never sent one.
Generation int64 `json:"generation,omitempty"`
// Resources, keyed by identity, in the order they were applied. Order matters for removal:
// undoing in reverse is the only ordering the host can derive without deciding anything.
Resources []Applied `json:"resources"`