Merge pull request 'The kernel module loads tun at boot and keeps the running kernel's modules across an upgrade; mesh-build-image takes no ask where it cannot build (group fix/ace-tun)' (#543) from fix/ace-tun into main

modules/kernel/README.md

@@ -53,6 +53,22 @@ built for the fallback kernel too) after its boot-entries service, so the entry
 | file | `/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook` | runs the script after a kernel image is installed, upgraded or removed |
 | packages | `linux-lts`, `linux-lts-headers` | declared after the service and the hook |
 
+## Kernel modules at boot, and the running kernel's modules kept
+
+A machine that runs for weeks meets a kernel upgrade that removes `/usr/lib/modules/<the running kernel>`: from then on
+no module that was not loaded yet can be loaded until the next boot. On 2026-10-28 a build node, up 24 days on
+`linux-lts` 6.18.54 with only 6.18.58's modules on disk, had `/dev/net/tun` and no tun driver, and every image build
+there failed in buildah's network setup ("Failed to open() /dev/net/tun: No such device").
+
+| | what | notes |
+|---|---|---|
+| file | `/etc/modules-load.d/mesh-kernel.conf` | the kernel modules loaded at every boot by systemd-modules-load: `tun` (pasta, which buildah uses for a RUN line's network, and every user-space tunnel open `/dev/net/tun`). Read at boot only: nothing is loaded by an apply |
+| package | `kernel-modules-hook` | keeps the running kernel's modules across an upgrade (saved before the transaction, put back after), so a module first needed later still loads |
+| service | `linux-modules-cleanup.service` | the package's oneshot, enabled at boot and stopped otherwise: it moves the modules of a kernel that is neither running nor owned by a package to `/usr/lib/modules/.old`, which the package's tmpfiles rule empties after four weeks. An apply never starts it |
+
+A machine whose running kernel's modules are already gone loads nothing new until it is started again; the image build
+module says so in its health and takes no image build there meanwhile.
+
 ## Tools
 
 | tool | | answers |
@@ -63,7 +79,7 @@ built for the fallback kernel too) after its boot-entries service, so the entry
 ## Tests
 
 `go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the
-manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); the boot-entries script run with `sh` over stand-in roots (a systemd-boot entries directory, a GRUB menu, a hook that copies kernels to one fixed name): the entry from the booted command line, a menu generated again only when the kernels changed, no loader, no room, the taken-over hook; `kernel_boot` over the same roots; and the huge-pages script as the mesh
+manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); the kernel modules loaded at boot (`tun`) and the package and boot service that keep the running kernel's modules; the boot-entries script run with `sh` over stand-in roots (a systemd-boot entries directory, a GRUB menu, a hook that copies kernels to one fixed name): the entry from the booted command line, a menu generated again only when the kernels changed, no loader, no room, the taken-over hook; `kernel_boot` over the same roots; and the huge-pages script as the mesh
 writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once,
 rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a
 reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel

modules/kernel/cmd/kernel-tools/manifest_test.go

@@ -24,7 +24,7 @@ func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) {
 	for _, r := range m.Resources {
 		ids = append(ids, r["id"].(string))
 	}
-	if strings.Join(ids, " ") != "state huge-pages unit apply boot-entries boot-unit boot-hook boot-apply linux-lts linux-lts-headers" {
+	if strings.Join(ids, " ") != "state huge-pages unit apply boot-entries boot-unit boot-hook boot-apply linux-lts linux-lts-headers modules-load modules-keep modules-cleanup" {
 		t.Fatalf("order: %v", ids)
 	}
 	script := m.resource(t, "huge-pages")
@@ -57,6 +57,31 @@ func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) {
 	if h := apply["health"].(map[string]any); h["kind"] != "unit" {
 		t.Fatalf("health: %v", h)
 	}
+	// A build node, 2026-10-28: up 24 days on linux-lts 6.18.54 after an upgrade to 6.18.58 removed the running kernel's modules,
+	// so tun could not be loaded when a build first asked for it. tun is loaded at every boot, before an upgrade can take
+	// its file away, and the running kernel's modules are kept across an upgrade (kernel-modules-hook) and cleaned up at
+	// the next boot.
+	load := m.resource(t, "modules-load")
+	if load["path"] != "/etc/modules-load.d/mesh-kernel.conf" || load["mode"] != "0644" {
+		t.Fatalf("modules-load: %v", load)
+	}
+	var names []string
+	for _, l := range strings.Split(load.text(t), "\n") {
+		if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") {
+			names = append(names, l)
+		}
+	}
+	if strings.Join(names, " ") != "tun" {
+		t.Fatalf("the modules loaded at boot: %v", names)
+	}
+	if keep := m.resource(t, "modules-keep"); keep["type"] != "package" || keep["package"] != "kernel-modules-hook" {
+		t.Fatalf("modules-keep: %v", keep)
+	}
+	// A oneshot at boot: enabled, and stopped once it ran, never started by an apply (it moves the modules of kernels no
+	// package owns away).
+	if c := m.resource(t, "modules-cleanup"); c["unit"] != "linux-modules-cleanup.service" || c["state"] != "stopped" || c["boot"] != "enabled" {
+		t.Fatalf("modules-cleanup: %v", c)
+	}
 }
 
 // bed is the script as the mesh writes it for one value of the setting, a stand-in transparent_hugepage

modules/kernel/files/modules-load.conf

@@ -0,0 +1,4 @@
+# Written by the mesh's kernel module (systemd-modules-load reads it at boot).
+# tun: buildah's network for a RUN line (pasta) and every other user-space tunnel open /dev/net/tun.
+# Loaded at boot, so an upgrade that removes the running kernel's modules cannot keep it out later.
+tun

modules/kernel/module.json

@@ -100,6 +100,25 @@
       "id": "linux-lts-headers",
       "type": "package",
       "package": "linux-lts-headers"
+    },
+    {
+      "id": "modules-load",
+      "type": "file",
+      "path": "/etc/modules-load.d/mesh-kernel.conf",
+      "mode": "0644",
+      "template": "files/modules-load.conf"
+    },
+    {
+      "id": "modules-keep",
+      "type": "package",
+      "package": "kernel-modules-hook"
+    },
+    {
+      "id": "modules-cleanup",
+      "type": "service",
+      "unit": "linux-modules-cleanup.service",
+      "state": "stopped",
+      "boot": "enabled"
     }
   ],
   "build": {

modules/mesh-build-image/README.md

@@ -16,7 +16,12 @@ credential and an environment built from scratch.
 
 The account's subordinate user and group ranges are not declared anywhere: the node-engine makes every `user` account
 with `useradd`, which allocates them. The module's health reads `/etc/subuid` and `/etc/subgid` and is unhealthy
-without a range for the account, without buildah, or when a user namespace cannot be made.
+without a range for the account, without buildah, when a user namespace cannot be made, or when `/dev/net/tun`
+cannot be opened (buildah makes a RUN line's network with pasta, which needs the tun driver; on 2026-10-28 a build node had the
+device node and no driver, so every image build there failed with "Failed to open() /dev/net/tun: No such device"). The
+kernel module loads `tun` at boot. Whatever the health refuses, the node is **not eligible**: the seat's puller judges
+by a capacity of 0 with that reason (`capacity.<node>`, which `build_usage` shows), so the asks go to a node that can
+build them; a look that passes again wakes the puller.
 
 The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can read holds a key.
 

modules/mesh-build-image/cmd/mesh-build-image/asks.go

@@ -59,7 +59,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
 		load = func(key string) ([]byte, error) { return m.ReadPauses.Get(key) }
 	}
 	return &buildstep.Puller{Source: source, Language: delivery.AskImage, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load,
-		Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Logf: m.Say,
+		Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Eligible: m.Eligible, Logf: m.Say,
 		Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
 			if err := m.Check(ctx); err != nil {
 				return nil, err

modules/mesh-build-image/cmd/mesh-build-image/asks_test.go

@@ -71,4 +71,17 @@ func TestAPulledImageAskCopiesThenBuilds(t *testing.T) {
 			t.Errorf("%s.%s is not served", Seat, verb)
 		}
 	}
+	// A node without /dev/net/tun (a build node, 2026-10-28) takes no ask: its capacity is 0 and says why, so the ask goes to a node
+	// that can build it instead of failing in buildah's network setup; once it is there again, the capacity is back.
+	tun := m.Tun
+	m.Tun = t.TempDir() + "/none"
+	_ = m.Check(context.Background())
+	if c := p.CapacityDetail(); c.Effective != 0 || !strings.Contains(c.Why, "not eligible: "+m.Tun+" cannot be opened") {
+		t.Errorf("without /dev/net/tun the capacity = %+v", c)
+	}
+	m.Tun = tun
+	_ = m.Check(context.Background())
+	if c := p.CapacityDetail(); c.Effective != 1 || c.Ineligible != "" {
+		t.Errorf("with /dev/net/tun again the capacity = %+v", c)
+	}
 }

modules/mesh-build-image/cmd/mesh-build-image/helpers_test.go

@@ -190,6 +190,7 @@ func moduleWith(t *testing.T, f *fake, states *keptStates) *Module {
 		SubUIDFile:    write("subuid", "someone:100000:65536\nmesh-build-image:624288:65536\n"),
 		SubGIDFile:    write("subgid", "mesh-build-image:624288:65536\n"),
 		RegistryFile:  write("registry", f.address()+"\n"),
+		Tun:           write("tun", ""),
 		RegistryWait:  2 * time.Second,
 		RegistryPause: 5 * time.Millisecond,
 	}

modules/mesh-build-image/cmd/mesh-build-image/module.go

@@ -38,6 +38,8 @@ type Module struct {
 	Work, Store string
 	// SubUIDFile and SubGIDFile are the subordinate id files; /etc/subuid and /etc/subgid when empty.
 	SubUIDFile, SubGIDFile string
+	// Tun is the tun device buildah's network for a RUN line (pasta) opens; TunDevice when empty.
+	Tun string
 	// RegistryFile holds the registry's address, host:port, which the manifest binds from the artifact-store provision.
 	RegistryFile string
 	// RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of
@@ -62,6 +64,10 @@ type Module struct {
 
 	spacesOnce sync.Once
 	workspaces *buildstep.Workspaces
+
+	// lastMu guards last: the refusal of the last look ("" when it could build), which Eligible answers.
+	lastMu sync.Mutex
+	last   string
 }
 
 // Look is what the module found out about its machine.
@@ -90,15 +96,44 @@ func (m *Module) look(ctx context.Context) (Look, error) {
 	if err := m.userNamespace(ctx); err != nil {
 		return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
 	}
+	if err := m.tunDevice(); err != nil {
+		return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
+	}
 	return l, nil
 }
 
-// Check is the guard: nil when the machine can build, else the refusal.
+// Check is the guard: nil when the machine can build, else the refusal. Every look is kept for Eligible, and a change
+// wakes the puller, so a node that can build again takes asks at once.
 func (m *Module) Check(ctx context.Context) error {
 	_, err := m.look(ctx)
+	m.keep(err)
 	return err
 }
 
+// keep records a look's outcome for Eligible and wakes the puller when it changed.
+func (m *Module) keep(err error) {
+	why := ""
+	if err != nil {
+		why = err.Error()
+	}
+	m.lastMu.Lock()
+	changed := m.last != why
+	m.last = why
+	m.lastMu.Unlock()
+	if changed && m.Held != nil {
+		m.Held.Wake()
+	}
+}
+
+// Eligible is the puller's eligibility (buildstep.Puller.Eligible): the refusal of the module's last look, "" when it
+// could build. While the machine cannot build, the node's capacity is 0 and says why, so its asks go to a node that can
+// build them instead of failing here (a build node without /dev/net/tun, 2026-10-28).
+func (m *Module) Eligible() string {
+	m.lastMu.Lock()
+	defer m.lastMu.Unlock()
+	return m.last
+}
+
 // Start looks once, as the module starts, and then every CheckRound until ctx ends; it also removes old workspaces.
 func (m *Module) Start(ctx context.Context) {
 	for {
@@ -119,13 +154,14 @@ func (m *Module) Start(ctx context.Context) {
 // healthy it names the buildah installed and the account's ranges (ADR 0487 §2: the ranges are proven here, not declared).
 func (m *Module) Health(ctx context.Context) health.Answer {
 	l, err := m.look(ctx)
+	m.keep(err)
 	if err != nil {
 		return health.Unhealthy(err.Error())
 	}
 	if err := m.room(); err != nil {
 		return health.Unhealthy(err.Error())
 	}
-	return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made", l.Buildah, l.Account, l.UIDs, l.GIDs))
+	return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made; %s opens", l.Buildah, l.Account, l.UIDs, l.GIDs, m.tunPath()))
 }
 
 // Status is what the module reports of itself.

modules/mesh-build-image/cmd/mesh-build-image/module_test.go

@@ -43,20 +43,33 @@ func TestHealthIsUnhealthyWithoutBuildah(t *testing.T) {
 }
 
 // The ranges are read from the two files, one each: without one in /etc/subuid or in /etc/subgid the account has no ids to
-// map, and the health says which file.
-func TestHealthIsUnhealthyWithoutARangeInEitherFile(t *testing.T) {
-	for file, set := range map[string]func(*Module, string){
-		"subuid": func(m *Module, p string) { m.SubUIDFile = p },
-		"subgid": func(m *Module, p string) { m.SubGIDFile = p },
+// map, and the health says which file. And without /dev/net/tun (a build node on 2026-10-28: its running kernel's modules were
+// gone, so tun could not be loaded) buildah's network for a RUN line cannot be made: the health says the node is not
+// eligible, and the puller judges by 0 so its asks go to a node that can build them.
+func TestHealthIsUnhealthyWithoutWhatABuildNeeds(t *testing.T) {
+	for _, c := range []struct {
+		name string
+		set  func(*Module, string)
+		says []string
+	}{
+		{"subuid", func(m *Module, p string) { m.SubUIDFile = p }, []string{"/subuid", "mesh-build-image"}},
+		{"subgid", func(m *Module, p string) { m.SubGIDFile = p }, []string{"/subgid", "mesh-build-image"}},
+		{"tun", func(m *Module, p string) { m.Tun = p + "/dev/net/tun" }, []string{"not eligible: ", "/dev/net/tun cannot be opened", "tun"}},
 	} {
 		f := newFake(t)
 		m := moduleWith(t, f, newStates())
-		p := t.TempDir() + "/" + file
+		p := t.TempDir() + "/" + c.name
 		os.WriteFile(p, []byte("someone-else:100000:65536\n"), 0o644)
-		set(m, p)
+		c.set(m, p)
 		a := m.Health(context.Background())
-		if a.Healthy || !strings.Contains(a.Why, p) || !strings.Contains(a.Why, "mesh-build-image") {
-			t.Errorf("%s: health = %+v, want unhealthy naming %s and the account", file, a, p)
+		for _, want := range c.says {
+			if a.Healthy || !strings.Contains(a.Why, want) {
+				t.Errorf("%s: health = %+v, want unhealthy saying %q", c.name, a, want)
+			}
+		}
+		// Whatever the look refuses, the node is not eligible, with the same words.
+		if got := m.Eligible(); got != a.Why {
+			t.Errorf("%s: eligible says %q, the health %q", c.name, got, a.Why)
 		}
 	}
 }

modules/mesh-build-image/cmd/mesh-build-image/probe.go

@@ -2,11 +2,13 @@ package main
 
 // What the module proves about its own machine before it builds (novox/hq ADR 0487 §2, issue 1703a): buildah runs,
 // the account holds subordinate user and group ranges (the node-engine's useradd allocates them for every `user`
-// account, so nothing declares them; this reads them), and a user namespace can be made.
+// account, so nothing declares them; this reads them), a user namespace can be made, and the tun device opens (buildah
+// makes a RUN line's network with pasta, which needs it; a build node on 2026-10-28 had the device node and no tun driver).
 
 import (
 	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -18,6 +20,9 @@ import (
 // ProbeTimeout bounds each probe: they read the machine and fetch nothing.
 const ProbeTimeout = 20 * time.Second
 
+// TunDevice is the tun device pasta opens for a RUN line's network.
+const TunDevice = "/dev/net/tun"
+
 // Default subordinate id files.
 const (
 	SubUIDFile = "/etc/subuid"
@@ -133,3 +138,23 @@ func (m *Module) ensureRuntimeDir() error {
 	}
 	return nil
 }
+
+func (m *Module) tunPath() string {
+	if m.Tun != "" {
+		return m.Tun
+	}
+	return TunDevice
+}
+
+// tunDevice opens the tun device as the account does for a build, and says the node is not eligible when it cannot: a
+// device node without its driver answers "no such device" (the tun kernel module is not loaded and cannot be, as when the
+// running kernel's modules were removed by an upgrade), and every RUN line would fail in buildah's network setup.
+func (m *Module) tunDevice() error {
+	f, err := os.OpenFile(m.tunPath(), os.O_RDWR, 0)
+	if err != nil {
+		return fmt.Errorf("not eligible: %s cannot be opened (%v), so buildah cannot make a RUN line's network (pasta): "+
+			"the tun kernel module is not loaded here (the kernel module loads it at boot; a machine whose running kernel's "+
+			"modules are gone needs a reboot)", m.tunPath(), errors.Unwrap(err))
+	}
+	return f.Close()
+}

modules/mesh-build-image/go.mod

@@ -3,7 +3,7 @@ module meshbuildimage
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027002948-db1007ffdda1
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028115016-1618837ea9fe
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 )

modules/mesh-build-image/go.sum

@@ -1,5 +1,5 @@
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027002948-db1007ffdda1 h1:9qyUcrvlklxDRBUCBqGH1L3oximOk554IfaPrTMgaTA=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027002948-db1007ffdda1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028115016-1618837ea9fe h1:QDWi8k7zxaRPnJ+V/izoy6A19hZt1LTLl3uksZSlqOg=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028115016-1618837ea9fe/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=