From ab1811d00357a95329870b0c3a7614168a12480b Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 23:42:49 +0200 Subject: [PATCH] Reload, not restart, the huge pages unit when the setting changes (hq issue 746) A restart ran restore then apply, so every change passed back through the found mode and rotated the journal again. Reload applies the new value directly; restore also drops its state; no rotation while the machine boots; the unit is sandboxed. --- modules/kernel/README.md | 14 ++-- .../kernel/cmd/kernel-tools/manifest_test.go | 78 ++++++++++++++++--- modules/kernel/module.json | 8 +- 3 files changed, 81 insertions(+), 19 deletions(-) diff --git a/modules/kernel/README.md b/modules/kernel/README.md index faf4cc94..dddcebd4 100644 --- a/modules/kernel/README.md +++ b/modules/kernel/README.md @@ -22,8 +22,8 @@ lost its lease four times in 30 minutes. With `madvise`, only memory a program a |---|---|---| | setting | `transparent-huge-pages` | `always`, `madvise` or `never`. Default `always`, what the distribution's kernel boots with, so assigning the module changes nothing until a machine sets another value | | file | `${dir:state}/huge-pages` | the script, written from the setting: `apply` writes the mode to `/sys/kernel/mm/transparent_hugepage/enabled` and reads it back (a value that does not take fails the unit); `restore` gives back the mode found before the module first set it, kept in `${dir:state}/found` | -| file | `/etc/systemd/system/mesh-kernel-huge-pages.service` | oneshot, early in boot (`sysinit.target`, before the services that fill the page cache); stopping it runs `restore` | -| service | `mesh-kernel-huge-pages.service` | running and enabled, restarted when the script (so the setting) or the unit changes; its health is the unit's | +| file | `/etc/systemd/system/mesh-kernel-huge-pages.service` | oneshot, early in boot (`sysinit.target`, before the services that fill the page cache); reloading it runs `apply`, stopping it runs `restore`, which also removes `found`. Sandboxed: the file system read-only but for the module's state (`/sys` stays writable), no home, no network, no new privileges; the journal's rotation goes over its local socket, which a private network leaves reachable | +| service | `mesh-kernel-huge-pages.service` | running and enabled; **reloaded** when the script (so the setting) changes, so a new value is set directly and never passes back through the found one; restarted only when the unit changes; its health is the unit's | The per-size controls (`hugepages-kB/enabled`) are left alone: on the machines read, 2048 kB says `inherit` (it follows the mode) and every other size `never`. `kernel_huge_pages` shows each one. @@ -31,8 +31,9 @@ The per-size controls (`hugepages-kB/enabled`) are left alone: on the machine **A folio already in the page cache keeps its size.** A new mode applies to pages faulted or collapsed from then on. The active journal files would keep their 2 MiB folios until archived, so `apply` rotates the journal (`journalctl --rotate`) whenever it moves the mode away from `always`: new files are mapped -under the new mode. A failed rotation is said in the unit's journal and does not fail the setting. At -boot the kernel starts at `always`, so a machine set to `madvise` rotates its journal once per boot. +under the new mode. A failed rotation is said in the unit's journal and does not fail the setting. No +rotation is made while the machine is still starting (`systemctl is-system-running` says `initializing` +or `starting`): nothing is in huge pages yet, and the persistent journal opens after this unit. ## Tools @@ -43,8 +44,9 @@ boot the kernel starts at `always`, so a machine set to `madvise` rotates its jo ## Tests `go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the -manifest's shape (the default, the order, the unit, restart and health); and the script as the mesh +manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); and the script as the mesh writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once, -rotates the journal once and not for `always`, gives back what it found, refuses a value the kernel +rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a +reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel does not take before writing anything, fails when the write does not take, and says when the kernel has no transparent huge pages. diff --git a/modules/kernel/cmd/kernel-tools/manifest_test.go b/modules/kernel/cmd/kernel-tools/manifest_test.go index fd1a0327..bcdc42e3 100644 --- a/modules/kernel/cmd/kernel-tools/manifest_test.go +++ b/modules/kernel/cmd/kernel-tools/manifest_test.go @@ -2,8 +2,9 @@ package main // The module's shape (novox/hq issue 746): the setting's default is what the kernel boots with, so // assigning the module changes nothing until a machine sets another value; the script is written into the -// module's own state and run by a oneshot unit that is restarted whenever the script (and so the setting) -// or the unit changes; and the script itself sets, verifies, gives back and refuses as it says. +// module's own state and run by a sandboxed oneshot unit that is reloaded (applied again) when the script, +// and so the setting, changes and restarted only when the unit does; and the script itself sets, verifies, +// gives back and refuses as it says. import ( "os" @@ -34,7 +35,9 @@ func TestTheDefaultChangesNothingAndTheUnitFollowsTheSetting(t *testing.T) { } unit := m.resource(t, "unit")["content"].(string) for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/huge-pages apply", - "ExecStop=${dir:state}/huge-pages restore", "WantedBy=sysinit.target", "DefaultDependencies=no"} { + "ExecReload=${dir:state}/huge-pages apply", "ExecStop=${dir:state}/huge-pages restore", "WantedBy=sysinit.target", + "DefaultDependencies=no", "ProtectSystem=strict", "ReadWritePaths=${dir:state}", "ProtectHome=yes", "PrivateNetwork=yes", + "NoNewPrivileges=yes"} { if !strings.Contains(unit, line+"\n") { t.Errorf("the unit lacks %q", line) } @@ -43,9 +46,14 @@ func TestTheDefaultChangesNothingAndTheUnitFollowsTheSetting(t *testing.T) { if apply["unit"] != "mesh-kernel-huge-pages.service" || apply["state"] != "running" || apply["boot"] != "enabled" { t.Fatalf("apply: %v", apply) } - if on := apply["restart-on"].([]any); len(on) != 2 || on[0] != "huge-pages" || on[1] != "unit" { + // A new value reloads (apply again); only a new unit restarts, whose stop would pass back through the + // mode the machine was found with. + if on := apply["restart-on"].([]any); len(on) != 1 || on[0] != "unit" { t.Fatalf("restart-on: %v", on) } + if on := apply["reload-on"].([]any); len(on) != 1 || on[0] != "huge-pages" { + t.Fatalf("reload-on: %v", on) + } if h := apply["health"].(map[string]any); h["kind"] != "unit" { t.Fatalf("health: %v", h) } @@ -54,12 +62,17 @@ func TestTheDefaultChangesNothingAndTheUnitFollowsTheSetting(t *testing.T) { // bed is the script as the mesh writes it for one value of the setting, a stand-in transparent_hugepage // directory, the module's state, and a PATH whose journalctl records that it was asked to rotate. type bed struct { - script, sys, state, rotated, path string + script, sys, state, rotated, path, bin string } func newBed(t *testing.T, want, found string) bed { + return newBedIn(t, t.TempDir(), want, found) +} + +// newBedIn writes the script for want into dir; a second bed in the same dir shares the machine and the +// state, as a reload with a new value does. +func newBedIn(t *testing.T, dir, want, found string) bed { t.Helper() - dir := t.TempDir() b := bed{script: filepath.Join(dir, "huge-pages"), sys: filepath.Join(dir, "thp"), state: filepath.Join(dir, "state"), rotated: filepath.Join(dir, "rotated")} content := manifest(t).resource(t, "huge-pages")["content"].(string) @@ -75,12 +88,22 @@ func newBed(t *testing.T, want, found string) bed { } } write(t, b.script, content, 0o755) - write(t, filepath.Join(b.sys, "enabled"), found, 0o644) + if found != "" { + write(t, filepath.Join(b.sys, "enabled"), found, 0o644) + } write(t, filepath.Join(bin, "journalctl"), "#!/bin/sh\necho \"$@\" >>"+b.rotated+"\n", 0o755) + b.bin = bin + b.booted(t, "running") b.path = bin + string(os.PathListSeparator) + os.Getenv("PATH") return b } +// booted makes the service manager say state for is-system-running. +func (b bed) booted(t *testing.T, state string) { + t.Helper() + write(t, filepath.Join(b.bin, "systemctl"), "#!/bin/sh\n[ \"$1\" = is-system-running ] && echo "+state+"\n", 0o755) +} + func write(t *testing.T, p, content string, mode os.FileMode) { t.Helper() if err := os.WriteFile(p, []byte(content), mode); err != nil { @@ -134,6 +157,44 @@ func TestApplySetsTheModeRotatesTheJournalOnceAndRestoreGivesBackWhatWasFound(t if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "always\n" { t.Fatalf("restored: %q", got) } + // Nothing of the module is kept once it is stopped. + if _, err := os.Stat(filepath.Join(b.state, "found")); !os.IsNotExist(err) { + t.Fatalf("found kept after restore: %v", err) + } +} + +func TestAReloadWithANewValueGoesStraightThereAndNeverRewritesFound(t *testing.T) { + dir := t.TempDir() + b := newBedIn(t, dir, "madvise", "[always] madvise never\n") + if out, err := b.run(t, "apply"); err != nil { + t.Fatalf("apply: %v %s", err, out) + } + // The setting changes to never: the mesh rewrites the script and reloads the unit, which applies again. + n := newBedIn(t, dir, "never", "") + out, err := n.run(t, "apply") + if err != nil || !strings.Contains(out, "madvise -> never") { + t.Fatalf("reload: %v %s", err, out) + } + if got := n.file(t, filepath.Join(n.state, "found")); got != "always\n" { + t.Fatalf("found rewritten by a reload: %q", got) + } +} + +func TestNoRotationWhileTheMachineIsStarting(t *testing.T) { + for _, state := range []string{"initializing", "starting"} { + b := newBed(t, "madvise", "[always] madvise never\n") + b.booted(t, state) + out, err := b.run(t, "apply") + if err != nil || !strings.Contains(out, "still starting") { + t.Fatalf("%s: %v %s", state, err, out) + } + if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "madvise\n" { + t.Fatalf("%s: not set: %q", state, got) + } + if got := b.file(t, b.rotated); got != "" { + t.Fatalf("%s: rotated at boot: %q", state, got) + } + } } func TestAlwaysIsSetWithoutRotating(t *testing.T) { @@ -172,9 +233,6 @@ func TestAWriteThatDoesNotTakeFailsTheUnit(t *testing.T) { func TestAKernelWithoutHugePagesIsSaid(t *testing.T) { b := newBed(t, "madvise", "") - if err := os.Remove(filepath.Join(b.sys, "enabled")); err != nil { - t.Fatal(err) - } if out, err := b.run(t, "apply"); err == nil || !strings.Contains(out, "built without transparent huge pages") { t.Fatalf("%v %s", err, out) } diff --git a/modules/kernel/module.json b/modules/kernel/module.json index 68645769..15077979 100644 --- a/modules/kernel/module.json +++ b/modules/kernel/module.json @@ -27,14 +27,14 @@ "path": "${dir:state}/huge-pages", "mode": "0755", "trusted": true, - "content": "#!/bin/sh\n# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746).\n# Written by the mesh from this machine's setting transparent-huge-pages and run by\n# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the\n# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the\n# setting, never this file.\n#\n# huge-pages apply|restore [ []]\n#\n# The two directories are arguments only so the module's tests can run this against a stand-in tree.\nset -eu\n\nwant='${setting:transparent-huge-pages}'\nsys=/sys/kernel/mm/transparent_hugepage\nstate='${dir:state}'\n[ $# -ge 2 ] && sys=$2\n[ $# -ge 3 ] && state=$3\n\ncase \"$want\" in\nalways | madvise | never) ;;\n*)\n\techo \"the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never\" >&2\n\texit 2\n\t;;\nesac\n[ -f \"$sys/enabled\" ] || {\n\techo \"this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set\" >&2\n\texit 1\n}\n\n# The mode in force is the word the kernel brackets: \"always [madvise] never\". A file holding one bare\n# word (the stand-in tree of the tests) is read as that word.\nmode() { sed -n -e 's/.*\\[\\([a-z]*\\)\\].*/\\1/p' -e t -e 's/^ *\\([a-z]*\\) *$/\\1/p' \"$sys/enabled\"; }\n\n[ $# -ge 1 ] || set -- usage\ncase \"$1\" in\napply)\n\tnow=$(mode)\n\t# What the machine had before the mesh first set it, kept once, so `restore` gives it back.\n\t[ -s \"$state/found\" ] || printf '%s\\n' \"$now\" >\"$state/found\"\n\tif [ \"$now\" = \"$want\" ]; then\n\t\techo \"transparent huge pages: $want, as set\"\n\t\texit 0\n\tfi\n\tprintf '%s\\n' \"$want\" >\"$sys/enabled\"\n\tgot=$(mode)\n\t[ \"$got\" = \"$want\" ] || {\n\t\techo \"wrote $want to $sys/enabled and the kernel says $got\" >&2\n\t\texit 1\n\t}\n\techo \"transparent huge pages: $now -> $want\"\n\t# A file folio already in the page cache keeps its size: the active journal files, mapped by journald\n\t# and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new\n\t# files that are mapped under the new mode. A failed rotation is said and does not fail the setting.\n\tif [ \"$want\" != always ] && command -v journalctl >/dev/null 2>&1; then\n\t\tjournalctl --rotate || echo \"the journal could not be rotated: its active files keep their huge pages until they are archived\" >&2\n\tfi\n\t;;\nrestore)\n\tfound=$(cat \"$state/found\" 2>/dev/null || true)\n\tcase \"$found\" in\n\talways | madvise | never)\n\t\tprintf '%s\\n' \"$found\" >\"$sys/enabled\"\n\t\techo \"transparent huge pages: back to $found, as the machine had them\"\n\t\t;;\n\t*) echo \"transparent huge pages: nothing recorded as found, left at $(mode)\" ;;\n\tesac\n\t;;\n*)\n\techo \"usage: huge-pages apply|restore [ []]\" >&2\n\texit 2\n\t;;\nesac\n" + "content": "#!/bin/sh\n# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746).\n# Written by the mesh from this machine's setting transparent-huge-pages and run by\n# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the\n# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the\n# setting, never this file. A change of the setting reloads the unit, which runs `apply` again: the mode\n# moves straight to the new value and never passes back through the one the machine was found with.\n#\n# huge-pages apply|restore [ []]\n#\n# The two directories are arguments only so the module's tests can run this against a stand-in tree.\nset -eu\n\nwant='${setting:transparent-huge-pages}'\nsys=/sys/kernel/mm/transparent_hugepage\nstate='${dir:state}'\n[ $# -ge 2 ] && sys=$2\n[ $# -ge 3 ] && state=$3\n\ncase \"$want\" in\nalways | madvise | never) ;;\n*)\n\techo \"the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never\" >&2\n\texit 2\n\t;;\nesac\n[ -f \"$sys/enabled\" ] || {\n\techo \"this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set\" >&2\n\texit 1\n}\n\n# The mode in force is the word the kernel brackets: \"always [madvise] never\". A file holding one bare\n# word (the stand-in tree of the tests) is read as that word.\nmode() { sed -n -e 's/.*\\[\\([a-z]*\\)\\].*/\\1/p' -e t -e 's/^ *\\([a-z]*\\) *$/\\1/p' \"$sys/enabled\"; }\n\n[ $# -ge 1 ] || set -- usage\ncase \"$1\" in\napply)\n\tnow=$(mode)\n\t# What the machine had before the mesh first set it, kept once, so `restore` gives it back.\n\t[ -s \"$state/found\" ] || printf '%s\\n' \"$now\" >\"$state/found\"\n\tif [ \"$now\" = \"$want\" ]; then\n\t\techo \"transparent huge pages: $want, as set\"\n\t\texit 0\n\tfi\n\tprintf '%s\\n' \"$want\" >\"$sys/enabled\"\n\tgot=$(mode)\n\t[ \"$got\" = \"$want\" ] || {\n\t\techo \"wrote $want to $sys/enabled and the kernel says $got\" >&2\n\t\texit 1\n\t}\n\techo \"transparent huge pages: $now -> $want\"\n\t# A file folio already in the page cache keeps its size: the active journal files, mapped by journald\n\t# and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new\n\t# files that are mapped under the new mode. A failed rotation is said and does not fail the setting.\n\t# At boot nothing is in huge folios yet, and the persistent journal opens only after this unit: no rotation\n\t# while the service manager is still bringing the machine up.\n\tbooting=$(systemctl is-system-running 2>/dev/null || true)\n\tcase \"$booting\" in\n\tinitializing | starting) echo \"the machine is still starting: nothing to rotate yet\" ;;\n\t*)\n\t\tif [ \"$want\" != always ] && command -v journalctl >/dev/null 2>&1; then\n\t\t\tjournalctl --rotate || echo \"the journal could not be rotated: its active files keep their huge pages until they are archived\" >&2\n\t\tfi\n\t\t;;\n\tesac\n\t;;\nrestore)\n\tfound=$(cat \"$state/found\" 2>/dev/null || true)\n\tcase \"$found\" in\n\talways | madvise | never)\n\t\tprintf '%s\\n' \"$found\" >\"$sys/enabled\"\n\t\techo \"transparent huge pages: back to $found, as the machine had them\"\n\t\t;;\n\t*) echo \"transparent huge pages: nothing recorded as found, left at $(mode)\" ;;\n\tesac\n\t# Unassigned or shut down: nothing of the module is kept; the next start records what it finds again.\n\trm -f \"$state/found\"\n\t;;\n*)\n\techo \"usage: huge-pages apply|restore [ []]\" >&2\n\texit 2\n\t;;\nesac\n" }, { "id": "unit", "type": "file", "path": "/etc/systemd/system/mesh-kernel-huge-pages.service", "mode": "0644", - "content": "[Unit]\nDescription=Transparent huge pages as the mesh sets them on this machine\n# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting\n# transparent-huge-pages; stopping it gives back the mode the machine had before. It runs early in boot,\n# before the services that fill the page cache, and again whenever the setting changes.\nDefaultDependencies=no\nAfter=local-fs.target\nBefore=sysinit.target shutdown.target\nConflicts=shutdown.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=${dir:state}/huge-pages apply\nExecStop=${dir:state}/huge-pages restore\n\n[Install]\nWantedBy=sysinit.target\n" + "content": "[Unit]\nDescription=Transparent huge pages as the mesh sets them on this machine\n# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting\n# transparent-huge-pages; reloading it (the setting changed) sets the new mode; stopping it gives back the\n# mode the machine had before. It runs early in boot, before the services that fill the page cache.\nDefaultDependencies=no\nAfter=local-fs.target\nBefore=sysinit.target shutdown.target\nConflicts=shutdown.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=${dir:state}/huge-pages apply\nExecReload=${dir:state}/huge-pages apply\nExecStop=${dir:state}/huge-pages restore\n# It writes one file under /sys and its own state, and asks the journal daemon to rotate over its socket.\nProtectSystem=strict\nReadWritePaths=${dir:state}\nProtectHome=yes\nPrivateNetwork=yes\nNoNewPrivileges=yes\n\n[Install]\nWantedBy=sysinit.target\n" }, { "id": "apply", @@ -43,9 +43,11 @@ "state": "running", "boot": "enabled", "restart-on": [ - "huge-pages", "unit" ], + "reload-on": [ + "huge-pages" + ], "health": { "kind": "unit" }