diff --git a/cmd/mesh-host/generation_floor_test.go b/cmd/mesh-host/generation_floor_test.go new file mode 100644 index 0000000..81ec418 --- /dev/null +++ b/cmd/mesh-host/generation_floor_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "context" + "testing" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// **The generation applied is kept, and a put-back never lowers it**: after a put-back of generation 5 +// over 7, a send from generation 6 is still older than what this machine applied. +func TestThePutBackDoesNotLowerTheGenerationApplied(t *testing.T) { + built, err := system.For("arch") + if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil { + t.Skip("this machine is not one these tests can apply on") + } + was := builtFor + builtFor = "arch" + t.Cleanup(func() { builtFor = was }) + f := machineThatApplied(t, 7) + + newer := f.declared(t, map[string]any{"sequence": 9, "generation": 8}, "docker.conf", f.stays) + if r := applyAndKeep(context.Background(), f.opts, newer.Declaration, &newer, nil, nil); r.Refused != "" { + t.Fatalf("a newer generation was refused: %s", r.Refused) + } + back := f.declared(t, map[string]any{"sequence": 10, "generation": 5, "put_back": true}, "docker.conf", f.stays) + if r := applyAndKeep(context.Background(), f.opts, back.Declaration, &back, nil, nil); r.Refused != "" { + t.Fatalf("a put-back was refused: %s", r.Refused) + } + state, err := store.Load(f.opts.state) + if err != nil || state.Generation != 8 { + t.Fatalf("the generation applied is %d after a put-back of 5 over 8 (%v)", state.Generation, err) + } + stale := f.declared(t, map[string]any{"sequence": 11, "generation": 7}, "docker.conf", f.stays) + if r := applyAndKeep(context.Background(), f.opts, stale.Declaration, &stale, nil, nil); r.OlderGeneration == nil { + t.Fatalf("a send from generation 7 after 8 was applied because a put-back came between: %+v", r) + } +} diff --git a/cmd/mesh-host/generation_test.go b/cmd/mesh-host/generation_test.go new file mode 100644 index 0000000..a9b02f5 --- /dev/null +++ b/cmd/mesh-host/generation_test.go @@ -0,0 +1,170 @@ +package main + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// The assignment generation a declaration was composed from (novox/hq issue 234). +// +// On 2026-10-04 a declaration newer in sequence than every other, and composed from a view of the +// assignments older than the one this machine had applied, named four fewer modules — and the host, +// ordering by sequence alone, undeclared all four. The sequence orders arrival; it cannot tell a later +// send carrying earlier content. The generation can: the node-engine refuses a declaration composed from +// an older generation than the one it applied, before anything is read from the machine. +// +// These read what the report says through its JSON, as the controller does, so they run against a +// node-engine that has never heard of a generation and fail there for that reason. + +// generationFixture is a machine that applied generation 7: a file the mesh declared, kept in its state, +// and the declaration it kept — signed — that named it. +type generationFixture struct { + opts options + private ed25519.PrivateKey + kept store.Declared + // stays is the file the older generation's declaration does not name. + stays string +} + +func machineThatApplied(t *testing.T, generation int64) generationFixture { + t.Helper() + dir := t.TempDir() + opts := options{state: filepath.Join(dir, "state.json")} + _, private, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + stays := filepath.Join(dir, "docker.conf") + if err := os.WriteFile(stays, []byte("kept\n"), 0o644); err != nil { + t.Fatal(err) + } + // The state as a node-engine that applied generation 7 keeps it: the file it owns, and the highest + // generation it applied. Written as JSON, so the fixture is the same bytes for any build under test. + state := map[string]any{"resources": []any{map[string]any{"id": "docker.conf", "type": "file", + "origin": store.OriginDeclared, "target": stays}}, "generation": generation} + raw, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(opts.state, raw, 0o600); err != nil { + t.Fatal(err) + } + f := generationFixture{opts: opts, private: private, stays: stays} + f.kept = f.declared(t, map[string]any{"sequence": 3, "generation": generation}, "docker.conf", stays) + if err := store.SaveDeclared(store.DeclaredPath(opts.state), f.kept); err != nil { + t.Fatal(err) + } + return f +} + +// declared is a signed declaration with the envelope's keys given and one file. +func (f generationFixture) declared(t *testing.T, envelope map[string]any, id, path string) store.Declared { + t.Helper() + body := map[string]any{"declaration": 1, + "resources": []any{map[string]any{"id": id, "type": "file", "path": path, "content": "x\n"}}} + for k, v := range envelope { + body[k] = v + } + raw, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + return store.Declared{Declaration: raw, Signature: ed25519.Sign(f.private, raw)} +} + +// reportJSON is a report as the controller reads it. +func reportJSON(t *testing.T, r link.Report) map[string]any { + t.Helper() + raw, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatal(err) + } + return out +} + +// **A newer sequence from an older generation is refused, and nothing is undeclared** (issue 234). +func TestANewerSequenceFromAnOlderGenerationIsRefusedAndNothingIsUndeclared(t *testing.T) { + f := machineThatApplied(t, 7) + other := filepath.Join(filepath.Dir(f.opts.state), "sudo.conf") + stale := f.declared(t, map[string]any{"sequence": 9, "generation": 6}, "sudo.conf", other) + + report := applyAndKeep(context.Background(), f.opts, stale.Declaration, &stale, nil, nil) + said := reportJSON(t, report) + older, _ := said["older_generation"].(map[string]any) + if older == nil || older["generation"] != float64(6) || older["applied"] != float64(7) { + t.Fatalf("the refusal does not name the generation refused and the one applied: %v", said) + } + if report.Refused == "" || !strings.Contains(report.Refused, "generation 6") || + !strings.Contains(report.Refused, "generation 7") { + t.Errorf("the refusal is not said in words naming both generations: %q", report.Refused) + } + if report.Declared != digestOf(stale.Declaration) || report.Sequence != 9 { + t.Errorf("the refusal does not name the declaration refused, so the mesh cannot name who sent it: %+v", report) + } + if _, err := os.Stat(f.stays); err != nil { + t.Fatalf("a refused declaration undeclared what the machine applied: %v", err) + } + if _, err := os.Stat(other); err == nil { + t.Error("the refused declaration touched the machine") + } + state, err := store.Load(f.opts.state) + if err != nil { + t.Fatal(err) + } + if _, owned := state.Find("docker.conf"); !owned { + t.Error("the refused declaration took a resource out of what this machine owns") + } + kept, err := store.ReadDeclared(store.DeclaredPath(f.opts.state)) + if err != nil || string(kept.Declaration) != string(f.kept.Declaration) { + t.Error("the refused declaration replaced what this node kept, so the next reconcile would apply it") + } +} + +// refusedForItsGeneration says a report refused its declaration for the generation it came from. +func refusedForItsGeneration(t *testing.T, r link.Report) bool { + t.Helper() + return reportJSON(t, r)["older_generation"] != nil || strings.Contains(r.Refused, "generation") +} + +// refusedAsUnread says a report refused its declaration as one this node-engine cannot read. +func refusedAsUnread(r link.Report) bool { return strings.Contains(r.Refused, "not a declaration") } + +// **A gate's put-back is applied, whatever generation it carries**: it carries the generation of what it +// puts back, and says it is a put-back (issue 234). +func TestAPutBackFromAnOlderGenerationIsNotRefused(t *testing.T) { + f := machineThatApplied(t, 7) + back := f.declared(t, map[string]any{"sequence": 9, "generation": 5, "put_back": true}, "docker.conf", f.stays) + report := applyAndKeep(context.Background(), f.opts, back.Declaration, &back, nil, nil) + if refusedForItsGeneration(t, report) || refusedAsUnread(report) { + t.Fatalf("a put-back was refused: %s", report.Refused) + } +} + +// **A declaration that carries no generation is applied as it always was**: an older controller sends +// none, and a node-engine that refused it would strand every machine the moment the controller is older. +func TestADeclarationWithoutAGenerationIsNotRefusedForOne(t *testing.T) { + f := machineThatApplied(t, 7) + older := f.declared(t, map[string]any{"sequence": 9}, "docker.conf", f.stays) + report := applyAndKeep(context.Background(), f.opts, older.Declaration, &older, nil, nil) + if refusedForItsGeneration(t, report) || refusedAsUnread(report) { + t.Fatalf("a declaration without a generation was refused: %s", report.Refused) + } + // And the same generation again is no older. + same := f.declared(t, map[string]any{"sequence": 10, "generation": 7}, "docker.conf", f.stays) + report = applyAndKeep(context.Background(), f.opts, same.Declaration, &same, nil, nil) + if refusedForItsGeneration(t, report) || refusedAsUnread(report) { + t.Fatalf("a declaration of the generation applied was refused: %s", report.Refused) + } +} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index cc11fc1..a3d69a0 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -595,6 +595,11 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw if err := refuseStale(known, kept, keptErr, digest, from, link.Order{Epoch: d.Epoch, Sequence: d.Sequence}); err != nil { return err } + // The generation, as the link's deliveries are held to it (novox/hq issue 234): one composed from an + // older view of the assignments than this machine applied would undeclare what the mesh still assigns. + if refused := olderGeneration(known, d); refused != nil { + return errors.New(olderGenerationWords(*refused)) + } // Said before anything is done. steps := apply.Plan(d, known, origin) @@ -646,6 +651,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw // What this apply settles about the node, whichever way it went. The mode is what the // declaration said and the check above agreed with; the bundle, once applied, is consumed. updated.Mode = apply.ModeOf(d) + updated.Generation = appliedGeneration(known, d) // raised, never lowered (novox/hq issue 234) if from == fromBundle { updated.Genesis = &store.Genesis{Digest: digest, At: time.Now().UTC()} } @@ -1887,6 +1893,16 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto if err != nil { return link.Report{Refused: err.Error()} } + // **Composed from an older view of the assignments than this machine applied is refused, before + // anything is read from the machine** (novox/hq issue 234). A newer sequence carrying an older view + // undeclared four assigned modules on 2026-10-04; the sequence cannot tell it, the generation can. + // Only a delivery: what a reconcile applies is what was kept. + if signed != nil { + if refused := olderGeneration(known, declared); refused != nil { + return link.Report{Declared: digestOf(raw), Order: order, OlderGeneration: refused, + Refused: olderGenerationWords(*refused)} + } + } // A declaration from the mesh is the controller's word on the node's mode — the flip arrives // as exactly that, the first converged declaration after adopted ones — and becomes the // record. So does what this node kept: it is signed by the mesh and verified on load, where @@ -1933,6 +1949,8 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto // The mode the mesh said, recorded whichever way the apply went: the declaration is kept // either way, and the node is held to it from the next reconcile (novox/hq ADR 0100). updated.Mode = apply.ModeOf(declared) + // And the generation it came from, the same way, raised and never lowered (novox/hq issue 234). + updated.Generation = appliedGeneration(known, declared) // Root acted on what an agent account's judgement reads — an account, a package, an action, sudo's // rules: the search for setuid programs runs again (novox/hq issue 361). A failed apply counts what it @@ -2253,6 +2271,35 @@ func refuseOlder(kept store.Declared, keptErr error, order link.Order) error { return nil } +// olderGeneration is the refusal of a declaration composed from an older assignment generation than +// this machine applied, nil when it is not one (novox/hq issue 234). +// +// Only when both claim a generation: a declaration without one is from a controller that claims none, +// and a machine that applied none has nothing to hold it to — refusing on a guess would strand every +// machine the moment the controller is older than the node-engine. Equal is not older: a push of the +// same assignments is the ordinary case. A put-back carries the generation of what it puts back and +// says so; a gate putting a machine back is the one sender that may say something older. +func olderGeneration(known store.State, d *declaration.Declaration) *link.GenerationRefusal { + if d.PutBack || d.Generation <= 0 || known.Generation <= 0 || d.Generation >= known.Generation { + return nil + } + return &link.GenerationRefusal{Generation: d.Generation, Applied: known.Generation} +} + +// olderGenerationWords is a generation refusal as the report and a person read it. +func olderGenerationWords(r link.GenerationRefusal) string { + return fmt.Sprintf("this declaration was composed from assignment generation %d, and this node applied "+ + "generation %d: it was composed from a view of the assignments the mesh has moved past, and applying it "+ + "would undeclare what the mesh still assigns here (novox/hq issue 234). Refused whole; nothing was "+ + "applied", r.Generation, r.Applied) +} + +// appliedGeneration is the highest generation applied once this declaration is: raised by it, never +// lowered — a put-back of an older generation leaves it where it was (novox/hq issue 234). +func appliedGeneration(known store.State, d *declaration.Declaration) int64 { + return max(known.Generation, d.Generation) +} + // olderThanKept says whether a declaration of this order is older than the one this node kept — the // last it applied — and the kept one's order (link.Order.Older: only when both claim an epoch). A kept // declaration that cannot be read claims no order: what it is compared with is unknown, and refusing diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index ea03cf5..50db892 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -1378,6 +1378,20 @@ type Declaration struct { // what it wrote for it — its resources are absent from the declaration, and absence would // otherwise read as removal. LeftOut []string + + // Generation is the assignment generation this declaration was composed from: the controller's + // counter, raised in the same transaction as every change to what is assigned where (novox/hq + // issue 234). The sequence orders arrival and cannot tell a later send that carries an older view + // of the assignments; this can. A node-engine refuses a declaration composed from a generation + // older than the highest it applied — unless it is a put-back — because applying it would + // undeclare what the mesh still assigns. Zero is a declaration from a controller that claims none, + // and is applied as before. + Generation int64 + + // PutBack says this declaration is a gate putting a machine back (novox/hq issue 234): it carries + // the generation of what it puts back, which may be older than what the machine applied, and is + // not refused for it. + PutBack bool } // LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the @@ -1542,6 +1556,11 @@ type envelope struct { Epoch int64 `json:"epoch,omitempty"` // LeftOut is optional on the wire too, and absent when nothing was left out (ADR 0163). LeftOut []string `json:"left_out,omitempty"` + // Generation and PutBack are optional on the wire too (novox/hq issue 234): absent is a controller + // that claims no generation. **An older host refuses these keys**, decoding strictly; a controller + // sends them only to a host whose reports say `reads_generation`. + Generation int64 `json:"generation,omitempty"` + PutBack bool `json:"put_back,omitempty"` } func parse(raw []byte, allowActions bool) (*Declaration, error) { @@ -1559,8 +1578,17 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { } d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence, - Epoch: env.Epoch, LeftOut: env.LeftOut} + Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack} var problems []string + if env.Generation < 0 { + // Below zero would read as "none claimed" and pass every refusal of an older generation. + problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+ + "mesh assigns", env.Generation)) + } + if env.PutBack && allowActions { + // A put-back is a gate's, sent by the mesh; a carried bundle puts nothing back. + problems = append(problems, "a carried bundle says it is a put-back, and only the mesh's gate can say that") + } if env.Sequence < 0 || env.Epoch < 0 { // Below zero is no order any controller assigns, and read as "none claimed" it would let the // declaration past every refusal of what is older. diff --git a/internal/declaration/order_test.go b/internal/declaration/order_test.go index cdb4ad3..4d2c93d 100644 --- a/internal/declaration/order_test.go +++ b/internal/declaration/order_test.go @@ -35,3 +35,25 @@ func TestAnOrderBelowZeroIsRefused(t *testing.T) { } } } + +// **The assignment generation it was composed from, and whether it is a put-back** (novox/hq issue 234): +// absent is no claim; below zero is none the mesh assigns; a carried bundle puts nothing back. +func TestADeclarationCarriesItsGenerationAndWhetherItIsAPutBack(t *testing.T) { + d, err := Parse([]byte(withOrder(`"sequence":3,"generation":41,"put_back":true,`))) + if err != nil { + t.Fatalf("a declaration carrying its generation was refused: %v", err) + } + if d.Generation != 41 || !d.PutBack { + t.Fatalf("read generation %d, put-back %v", d.Generation, d.PutBack) + } + if d, err = Parse([]byte(withOrder(``))); err != nil || d.Generation != 0 || d.PutBack { + t.Fatalf("a declaration with no generation read as %+v, %v", d, err) + } + if refusal := refusalFor(t, withOrder(`"generation":-2,`)); !strings.Contains(refusal.Error(), "below zero") { + t.Errorf("a generation below zero was refused for something else: %v", refusal) + } + if _, err := ParseTrusted([]byte(withOrder(`"generation":3,"put_back":true,`))); err == nil || + !strings.Contains(err.Error(), "put-back") { + t.Errorf("a carried bundle said it was a put-back and was not refused for it: %v", err) + } +} diff --git a/internal/link/messages.go b/internal/link/messages.go index 68ff0bc..0183930 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -128,6 +128,17 @@ type Report struct { // read from the next one: what the mesh's stale-writer watchdog (to-be 45 §3, S13) counts. RefusedOlder int64 `json:"refused_older,omitempty"` + // ReadsGeneration says this node-engine reads a declaration's assignment generation and its + // put-back mark (novox/hq issue 234), on every report it makes: the controller sends them only to a + // machine that said so, because an older node-engine decodes strictly and refuses a key it does not + // know, whole. A node-engine rolled back says so no longer. + ReadsGeneration bool `json:"reads_generation,omitempty"` + + // OlderGeneration is set on a report refusing a declaration composed from an older assignment + // generation than this machine applied (novox/hq issue 234). The refused declaration is the report's + // own Declared and Order, so the controller can name who sent it from its record of sends. + OlderGeneration *GenerationRefusal `json:"older_generation,omitempty"` + // Held is what this adopted node found and is keeping as it was until its module is taken // (novox/hq ADR 0100). Without it an adopted node reads as converged. Held []Held `json:"held,omitempty"` @@ -634,3 +645,10 @@ type FoundFirewall struct { Active bool `json:"active"` RetiredBy string `json:"retired_by,omitempty"` } + +// GenerationRefusal is a declaration refused for the assignment generation it was composed from: that +// generation, and the highest this machine applied (novox/hq issue 234). +type GenerationRefusal struct { + Generation int64 `json:"generation"` + Applied int64 `json:"applied"` +} diff --git a/internal/link/queue.go b/internal/link/queue.go index 539189c..c2a66dd 100644 --- a/internal/link/queue.go +++ b/internal/link/queue.go @@ -211,6 +211,12 @@ func (q *Queue) step(ctx context.Context) bool { q.Say(fmt.Sprintf("refused declaration %s (%s): this node applied %s, from a newer lease "+ "holder — %d refused as older so far", short(report.Declared), report.Order.Words(), report.OlderThan.Words(), total)) + case report.OlderGeneration != nil: + // Said here, and the report names it, so the controller raises it naming the sender (novox/hq + // issue 234). Not counted with the lease's refusals: those are S13's, by epoch. + q.Say(fmt.Sprintf("refused declaration %s (%s): it was composed from assignment generation %d, and "+ + "this node applied generation %d", short(report.Declared), report.Order.Words(), + report.OlderGeneration.Generation, report.OlderGeneration.Applied)) case report.Refused != "": q.Say("refused a declaration: " + report.Refused) case len(report.Failed) > 0: @@ -228,7 +234,7 @@ func (q *Queue) step(ctx context.Context) bool { // A refusal as older is not kept to be said again: what the mesh waits for from this node is the // account of the last apply, and that is still what is kept. kind := applied - if report.OlderThan != nil { + if report.OlderThan != nil || report.OlderGeneration != nil { kind = refusedOlder } q.tell(ctx, report, kind, latest) @@ -336,6 +342,9 @@ func (q *Queue) stamp(r Report) Report { r.Node = q.Membership.Node r.ReportSequence = q.numbers.sequence r.RefusedOlder = q.numbers.refused + // Every report of a node-engine that orders its reports says it reads a generation (novox/hq issue + // 234): this build does, and the controller sends one only to a machine that said so. + r.ReadsGeneration = true return r } diff --git a/internal/link/queue_test.go b/internal/link/queue_test.go index dd75b69..9443983 100644 --- a/internal/link/queue_test.go +++ b/internal/link/queue_test.go @@ -540,3 +540,26 @@ func TestALinkIsNotHeldForAReconcileInHand(t *testing.T) { t.Fatalf("a report was said on a link let go: %+v", l.said()) } } + +// **Every report says this node-engine reads a declaration's generation** (novox/hq issue 234): the +// controller sends one only to a machine that said so, because an older node-engine refuses the key. +func TestEveryReportSaysItReadsAGeneration(t *testing.T) { + m, key := aMember(t) + l := newQuietLink() + a := &applies{} + q := aQueue(m, a.apply, &keptInMemory{}) + q.attach(context.Background(), l) + q.Deliver(ordered(t, key, "new", 7, 12)) + ctx, stop := context.WithCancel(context.Background()) + worker := runWorker(ctx, q) + eventually(t, "the delivery was never reported", func() bool { return len(l.said()) == 1 }) + stop() + <-worker + raw, err := json.Marshal(l.said()[0]) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), `"reads_generation":true`) { + t.Fatalf("the report does not say it reads a generation: %s", raw) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 3db306e..44ad307 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -187,6 +187,12 @@ type Into struct { // State is the whole of what a node knows about what it has done. type State struct { + // Generation is the highest assignment generation this machine applied a declaration from + // (novox/hq issue 234): a declaration from an older one is refused unless it is a put-back. Only + // ever raised — a put-back of an older generation leaves it — so a put-back cannot open the door to + // the stale send it protects against. Zero on a machine never sent one. + Generation int64 `json:"generation,omitempty"` + // Resources, keyed by identity, in the order they were applied. Order matters for removal: // undoing in reverse is the only ordering the host can derive without deciding anything. Resources []Applied `json:"resources"`