novox/mesh-catalog
13
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

Feature 57 (config and provisioning): release the catalogue, the reviewed release branch merged with main, on top of main

Browse Source
This commit is contained in:
mesh-tools
2026-10-16 09:42:45 +00:00
parent ee1b15d7f0
commit 0260ae92f6
646 changed files with 37611 additions and 1861 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/kernel/cmd/kernel-tools/manifest_test.go
+6 -6
View File
@@ -16,8 +16,8 @@ import (
func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) {
m := manifest(t)
s := m.Settings["transparent-huge-pages"]
if s["kind"] != "preference" || s["default"] != "madvise" || s["why"] == "" {
s := m.Config["transparent-huge-pages"]
if s["type"] != "string" || s["default"] != "madvise" || s["description"] == "" || s["trusted"] != true {
t.Fatalf("the setting: %v", s)
}
var ids []string
@@ -30,10 +30,10 @@ func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) {
script := m.resource(t, "huge-pages")
// The script runs as root, so the setting it is written from is set at the terminal alone (issue 339).
if script["path"] != "${dir:state}/huge-pages" || script["mode"] != "0755" || script["trusted"] != true ||
!strings.Contains(script["content"].(string), "want='${setting:transparent-huge-pages}'") {
!strings.Contains(script.text(t), "want='${config:transparent-huge-pages}'") {
t.Fatalf("the script: %v", script["path"])
}
unit := m.resource(t, "unit")["content"].(string)
unit := m.resource(t, "unit").text(t)
for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/huge-pages apply",
"ExecReload=${dir:state}/huge-pages apply", "ExecStop=${dir:state}/huge-pages restore", "WantedBy=sysinit.target",
"DefaultDependencies=no", "ProtectSystem=strict", "ReadWritePaths=${dir:state}", "ProtectHome=yes", "PrivateNetwork=yes",
@@ -75,8 +75,8 @@ func newBedIn(t *testing.T, dir, want, found string) bed {
t.Helper()
b := bed{script: filepath.Join(dir, "huge-pages"), sys: filepath.Join(dir, "thp"), state: filepath.Join(dir, "state"),
rotated: filepath.Join(dir, "rotated")}
content := manifest(t).resource(t, "huge-pages")["content"].(string)
content = strings.ReplaceAll(content, "${setting:transparent-huge-pages}", want)
content := manifest(t).resource(t, "huge-pages").text(t)
content = strings.ReplaceAll(content, "${config:transparent-huge-pages}", want)
content = strings.ReplaceAll(content, "${dir:state}", b.state)
if strings.Contains(content, "${") {
t.Fatalf("a substitution the test does not make is left in the script")
modules/kernel/cmd/kernel-tools/shape_test.go
+12 -1
View File
@@ -3,6 +3,7 @@ package main
import (
"encoding/json"
"os"
"path/filepath"
"testing"
)
@@ -11,7 +12,7 @@ type resource map[string]any
type manifestShape struct {
Module string `json:"module"`
Capabilities []string `json:"capabilities"`
Settings map[string]map[string]any `json:"settings"`
Config map[string]map[string]any `json:"config"`
Tools []string `json:"tools"`
Resources []resource `json:"resources"`
Build struct {
@@ -77,3 +78,13 @@ func TestToolsAreTheManifests(t *testing.T) {
t.Fatalf("loads: %v", tools["loads"])
}
}
// text is the file text a resource keeps in its template under files/ (novox/hq ADR 0349 §2).
func (r resource) text(t *testing.T) string {
t.Helper()
raw, err := os.ReadFile(filepath.Join("../..", r["template"].(string)))
if err != nil {
t.Fatal(err)
}
return string(raw)
}
modules/kernel/files/huge-pages.sh
+84
View File
@@ -0,0 +1,84 @@
#!/bin/sh
# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746).
# Written by the mesh from this machine's setting transparent-huge-pages and run by
# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the
# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the
# setting, never this file. A change of the setting reloads the unit, which runs `apply` again: the mode
# moves straight to the new value and never passes back through the one the machine was found with.
#
# huge-pages apply|restore [<the kernel's transparent_hugepage directory> [<this module's state>]]
#
# The two directories are arguments only so the module's tests can run this against a stand-in tree.
set -eu
want='${config:transparent-huge-pages}'
sys=/sys/kernel/mm/transparent_hugepage
state='${dir:state}'
[ $# -ge 2 ] && sys=$2
[ $# -ge 3 ] && state=$3
case "$want" in
always | madvise | never) ;;
*)
echo "the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never" >&2
exit 2
;;
esac
[ -f "$sys/enabled" ] || {
echo "this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set" >&2
exit 1
}
# The mode in force is the word the kernel brackets: "always [madvise] never". A file holding one bare
# word (the stand-in tree of the tests) is read as that word.
mode() { sed -n -e 's/.*\[\([a-z]*\)\].*/\1/p' -e t -e 's/^ *\([a-z]*\) *$/\1/p' "$sys/enabled"; }
[ $# -ge 1 ] || set -- usage
case "$1" in
apply)
now=$(mode)
# What the machine had before the mesh first set it, kept once, so `restore` gives it back.
[ -s "$state/found" ] || printf '%s\n' "$now" >"$state/found"
if [ "$now" = "$want" ]; then
echo "transparent huge pages: $want, as set"
exit 0
fi
printf '%s\n' "$want" >"$sys/enabled"
got=$(mode)
[ "$got" = "$want" ] || {
echo "wrote $want to $sys/enabled and the kernel says $got" >&2
exit 1
}
echo "transparent huge pages: $now -> $want"
# A file folio already in the page cache keeps its size: the active journal files, mapped by journald
# and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new
# files that are mapped under the new mode. A failed rotation is said and does not fail the setting.
# At boot nothing is in huge folios yet, and the persistent journal opens only after this unit: no rotation
# while the service manager is still bringing the machine up.
booting=$(systemctl is-system-running 2>/dev/null || true)
case "$booting" in
initializing | starting) echo "the machine is still starting: nothing to rotate yet" ;;
*)
if [ "$want" != always ] && command -v journalctl >/dev/null 2>&1; then
journalctl --rotate || echo "the journal could not be rotated: its active files keep their huge pages until they are archived" >&2
fi
;;
esac
;;
restore)
found=$(cat "$state/found" 2>/dev/null || true)
case "$found" in
always | madvise | never)
printf '%s\n' "$found" >"$sys/enabled"
echo "transparent huge pages: back to $found, as the machine had them"
;;
*) echo "transparent huge pages: nothing recorded as found, left at $(mode)" ;;
esac
# Unassigned or shut down: nothing of the module is kept; the next start records what it finds again.
rm -f "$state/found"
;;
*)
echo "usage: huge-pages apply|restore [<transparent_hugepage directory> [<state directory>]]" >&2
exit 2
;;
esac
modules/kernel/files/mesh-kernel-huge-pages.service
+25
View File
@@ -0,0 +1,25 @@
[Unit]
Description=Transparent huge pages as the mesh sets them on this machine
# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting
# transparent-huge-pages; reloading it (the setting changed) sets the new mode; stopping it gives back the
# mode the machine had before. It runs early in boot, before the services that fill the page cache.
DefaultDependencies=no
After=local-fs.target
Before=sysinit.target shutdown.target
Conflicts=shutdown.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=${dir:state}/huge-pages apply
ExecReload=${dir:state}/huge-pages apply
ExecStop=${dir:state}/huge-pages restore
# It writes one file under /sys and its own state, and asks the journal daemon to rotate over its socket.
ProtectSystem=strict
ReadWritePaths=${dir:state}
ProtectHome=yes
PrivateNetwork=yes
NoNewPrivileges=yes
[Install]
WantedBy=sysinit.target
modules/kernel/module.json
+12 -6
View File
@@ -4,11 +4,12 @@
"capabilities": [
"service-manager"
],
"settings": {
"config": {
"transparent-huge-pages": {
"kind": "preference",
"default": "madvise",
"why": "madvise is the value the mesh chose for every node (novox/hq ADR 0381), not the always the distribution's kernel boots with; a machine that needs another value is given it as its own setting; madvise gives huge pages only to programs that ask for them, which stops a small write to a mapped file (the journal) dirtying and writing back a whole 2 MiB page on a machine whose disks are slow (novox/hq issue 746)"
"type": "string",
"description": "madvise is the value the mesh chose for every node (novox/hq ADR 0381), not the always the distribution's kernel boots with; a machine that needs another value is given it as its own setting; madvise gives huge pages only to programs that ask for them, which stops a small write to a mapped file (the journal) dirtying and writing back a whole 2 MiB page on a machine whose disks are slow (novox/hq issue 746)",
"trusted": true,
"default": "madvise"
}
},
"tools": [
@@ -27,14 +28,14 @@
"path": "${dir:state}/huge-pages",
"mode": "0755",
"trusted": true,
"content": "#!/bin/sh\n# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746).\n# Written by the mesh from this machine's setting transparent-huge-pages and run by\n# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the\n# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the\n# setting, never this file. A change of the setting reloads the unit, which runs `apply` again: the mode\n# moves straight to the new value and never passes back through the one the machine was found with.\n#\n# huge-pages apply|restore [<the kernel's transparent_hugepage directory> [<this module's state>]]\n#\n# The two directories are arguments only so the module's tests can run this against a stand-in tree.\nset -eu\n\nwant='${setting:transparent-huge-pages}'\nsys=/sys/kernel/mm/transparent_hugepage\nstate='${dir:state}'\n[ $# -ge 2 ] && sys=$2\n[ $# -ge 3 ] && state=$3\n\ncase \"$want\" in\nalways | madvise | never) ;;\n*)\n\techo \"the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never\" >&2\n\texit 2\n\t;;\nesac\n[ -f \"$sys/enabled\" ] || {\n\techo \"this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set\" >&2\n\texit 1\n}\n\n# The mode in force is the word the kernel brackets: \"always [madvise] never\". A file holding one bare\n# word (the stand-in tree of the tests) is read as that word.\nmode() { sed -n -e 's/.*\\[\\([a-z]*\\)\\].*/\\1/p' -e t -e 's/^ *\\([a-z]*\\) *$/\\1/p' \"$sys/enabled\"; }\n\n[ $# -ge 1 ] || set -- usage\ncase \"$1\" in\napply)\n\tnow=$(mode)\n\t# What the machine had before the mesh first set it, kept once, so `restore` gives it back.\n\t[ -s \"$state/found\" ] || printf '%s\\n' \"$now\" >\"$state/found\"\n\tif [ \"$now\" = \"$want\" ]; then\n\t\techo \"transparent huge pages: $want, as set\"\n\t\texit 0\n\tfi\n\tprintf '%s\\n' \"$want\" >\"$sys/enabled\"\n\tgot=$(mode)\n\t[ \"$got\" = \"$want\" ] || {\n\t\techo \"wrote $want to $sys/enabled and the kernel says $got\" >&2\n\t\texit 1\n\t}\n\techo \"transparent huge pages: $now -> $want\"\n\t# A file folio already in the page cache keeps its size: the active journal files, mapped by journald\n\t# and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new\n\t# files that are mapped under the new mode. A failed rotation is said and does not fail the setting.\n\t# At boot nothing is in huge folios yet, and the persistent journal opens only after this unit: no rotation\n\t# while the service manager is still bringing the machine up.\n\tbooting=$(systemctl is-system-running 2>/dev/null || true)\n\tcase \"$booting\" in\n\tinitializing | starting) echo \"the machine is still starting: nothing to rotate yet\" ;;\n\t*)\n\t\tif [ \"$want\" != always ] && command -v journalctl >/dev/null 2>&1; then\n\t\t\tjournalctl --rotate || echo \"the journal could not be rotated: its active files keep their huge pages until they are archived\" >&2\n\t\tfi\n\t\t;;\n\tesac\n\t;;\nrestore)\n\tfound=$(cat \"$state/found\" 2>/dev/null || true)\n\tcase \"$found\" in\n\talways | madvise | never)\n\t\tprintf '%s\\n' \"$found\" >\"$sys/enabled\"\n\t\techo \"transparent huge pages: back to $found, as the machine had them\"\n\t\t;;\n\t*) echo \"transparent huge pages: nothing recorded as found, left at $(mode)\" ;;\n\tesac\n\t# Unassigned or shut down: nothing of the module is kept; the next start records what it finds again.\n\trm -f \"$state/found\"\n\t;;\n*)\n\techo \"usage: huge-pages apply|restore [<transparent_hugepage directory> [<state directory>]]\" >&2\n\texit 2\n\t;;\nesac\n"
"template": "files/huge-pages.sh"
},
{
"id": "unit",
"type": "file",
"path": "/etc/systemd/system/mesh-kernel-huge-pages.service",
"mode": "0644",
"content": "[Unit]\nDescription=Transparent huge pages as the mesh sets them on this machine\n# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting\n# transparent-huge-pages; reloading it (the setting changed) sets the new mode; stopping it gives back the\n# mode the machine had before. It runs early in boot, before the services that fill the page cache.\nDefaultDependencies=no\nAfter=local-fs.target\nBefore=sysinit.target shutdown.target\nConflicts=shutdown.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=${dir:state}/huge-pages apply\nExecReload=${dir:state}/huge-pages apply\nExecStop=${dir:state}/huge-pages restore\n# It writes one file under /sys and its own state, and asks the journal daemon to rotate over its socket.\nProtectSystem=strict\nReadWritePaths=${dir:state}\nProtectHome=yes\nPrivateNetwork=yes\nNoNewPrivileges=yes\n\n[Install]\nWantedBy=sysinit.target\n"
"template": "files/mesh-kernel-huge-pages.service"
},
{
"id": "apply",
@@ -65,6 +66,11 @@
"loads": [
"kernel-tools"
]
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
}