diff --git a/modules/kernel/README.md b/modules/kernel/README.md index 266ef7f2..032fd905 100644 --- a/modules/kernel/README.md +++ b/modules/kernel/README.md @@ -53,6 +53,22 @@ built for the fallback kernel too) after its boot-entries service, so the entry | file | `/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook` | runs the script after a kernel image is installed, upgraded or removed | | packages | `linux-lts`, `linux-lts-headers` | declared after the service and the hook | +## Kernel modules at boot, and the running kernel's modules kept + +A machine that runs for weeks meets a kernel upgrade that removes `/usr/lib/modules/`: from then on +no module that was not loaded yet can be loaded until the next boot. On 2026-10-28 a build node, up 24 days on +`linux-lts` 6.18.54 with only 6.18.58's modules on disk, had `/dev/net/tun` and no tun driver, and every image build +there failed in buildah's network setup ("Failed to open() /dev/net/tun: No such device"). + +| | what | notes | +|---|---|---| +| file | `/etc/modules-load.d/mesh-kernel.conf` | the kernel modules loaded at every boot by systemd-modules-load: `tun` (pasta, which buildah uses for a RUN line's network, and every user-space tunnel open `/dev/net/tun`). Read at boot only: nothing is loaded by an apply | +| package | `kernel-modules-hook` | keeps the running kernel's modules across an upgrade (saved before the transaction, put back after), so a module first needed later still loads | +| service | `linux-modules-cleanup.service` | the package's oneshot, enabled at boot and stopped otherwise: it moves the modules of a kernel that is neither running nor owned by a package to `/usr/lib/modules/.old`, which the package's tmpfiles rule empties after four weeks. An apply never starts it | + +A machine whose running kernel's modules are already gone loads nothing new until it is started again; the image build +module says so in its health and takes no image build there meanwhile. + ## Tools | tool | | answers | @@ -63,7 +79,7 @@ built for the fallback kernel too) after its boot-entries service, so the entry ## Tests `go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the -manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); the boot-entries script run with `sh` over stand-in roots (a systemd-boot entries directory, a GRUB menu, a hook that copies kernels to one fixed name): the entry from the booted command line, a menu generated again only when the kernels changed, no loader, no room, the taken-over hook; `kernel_boot` over the same roots; and the huge-pages script as the mesh +manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); the kernel modules loaded at boot (`tun`) and the package and boot service that keep the running kernel's modules; the boot-entries script run with `sh` over stand-in roots (a systemd-boot entries directory, a GRUB menu, a hook that copies kernels to one fixed name): the entry from the booted command line, a menu generated again only when the kernels changed, no loader, no room, the taken-over hook; `kernel_boot` over the same roots; and the huge-pages script as the mesh writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once, rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel diff --git a/modules/kernel/cmd/kernel-tools/manifest_test.go b/modules/kernel/cmd/kernel-tools/manifest_test.go index aad91fc6..21195061 100644 --- a/modules/kernel/cmd/kernel-tools/manifest_test.go +++ b/modules/kernel/cmd/kernel-tools/manifest_test.go @@ -24,7 +24,7 @@ func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) { for _, r := range m.Resources { ids = append(ids, r["id"].(string)) } - if strings.Join(ids, " ") != "state huge-pages unit apply boot-entries boot-unit boot-hook boot-apply linux-lts linux-lts-headers" { + if strings.Join(ids, " ") != "state huge-pages unit apply boot-entries boot-unit boot-hook boot-apply linux-lts linux-lts-headers modules-load modules-keep modules-cleanup" { t.Fatalf("order: %v", ids) } script := m.resource(t, "huge-pages") @@ -57,6 +57,31 @@ func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) { if h := apply["health"].(map[string]any); h["kind"] != "unit" { t.Fatalf("health: %v", h) } + // A build node, 2026-10-28: up 24 days on linux-lts 6.18.54 after an upgrade to 6.18.58 removed the running kernel's modules, + // so tun could not be loaded when a build first asked for it. tun is loaded at every boot, before an upgrade can take + // its file away, and the running kernel's modules are kept across an upgrade (kernel-modules-hook) and cleaned up at + // the next boot. + load := m.resource(t, "modules-load") + if load["path"] != "/etc/modules-load.d/mesh-kernel.conf" || load["mode"] != "0644" { + t.Fatalf("modules-load: %v", load) + } + var names []string + for _, l := range strings.Split(load.text(t), "\n") { + if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") { + names = append(names, l) + } + } + if strings.Join(names, " ") != "tun" { + t.Fatalf("the modules loaded at boot: %v", names) + } + if keep := m.resource(t, "modules-keep"); keep["type"] != "package" || keep["package"] != "kernel-modules-hook" { + t.Fatalf("modules-keep: %v", keep) + } + // A oneshot at boot: enabled, and stopped once it ran, never started by an apply (it moves the modules of kernels no + // package owns away). + if c := m.resource(t, "modules-cleanup"); c["unit"] != "linux-modules-cleanup.service" || c["state"] != "stopped" || c["boot"] != "enabled" { + t.Fatalf("modules-cleanup: %v", c) + } } // bed is the script as the mesh writes it for one value of the setting, a stand-in transparent_hugepage diff --git a/modules/kernel/files/modules-load.conf b/modules/kernel/files/modules-load.conf new file mode 100644 index 00000000..eccad6ce --- /dev/null +++ b/modules/kernel/files/modules-load.conf @@ -0,0 +1,4 @@ +# Written by the mesh's kernel module (systemd-modules-load reads it at boot). +# tun: buildah's network for a RUN line (pasta) and every other user-space tunnel open /dev/net/tun. +# Loaded at boot, so an upgrade that removes the running kernel's modules cannot keep it out later. +tun diff --git a/modules/kernel/module.json b/modules/kernel/module.json index 2a63563e..8fc90d09 100644 --- a/modules/kernel/module.json +++ b/modules/kernel/module.json @@ -100,6 +100,25 @@ "id": "linux-lts-headers", "type": "package", "package": "linux-lts-headers" + }, + { + "id": "modules-load", + "type": "file", + "path": "/etc/modules-load.d/mesh-kernel.conf", + "mode": "0644", + "template": "files/modules-load.conf" + }, + { + "id": "modules-keep", + "type": "package", + "package": "kernel-modules-hook" + }, + { + "id": "modules-cleanup", + "type": "service", + "unit": "linux-modules-cleanup.service", + "state": "stopped", + "boot": "enabled" } ], "build": { diff --git a/modules/mesh-build-image/README.md b/modules/mesh-build-image/README.md index e27eaa30..82b29c38 100644 --- a/modules/mesh-build-image/README.md +++ b/modules/mesh-build-image/README.md @@ -16,7 +16,12 @@ credential and an environment built from scratch. The account's subordinate user and group ranges are not declared anywhere: the node-engine makes every `user` account with `useradd`, which allocates them. The module's health reads `/etc/subuid` and `/etc/subgid` and is unhealthy -without a range for the account, without buildah, or when a user namespace cannot be made. +without a range for the account, without buildah, when a user namespace cannot be made, or when `/dev/net/tun` +cannot be opened (buildah makes a RUN line's network with pasta, which needs the tun driver; on 2026-10-28 a build node had the +device node and no driver, so every image build there failed with "Failed to open() /dev/net/tun: No such device"). The +kernel module loads `tun` at boot. Whatever the health refuses, the node is **not eligible**: the seat's puller judges +by a capacity of 0 with that reason (`capacity.`, which `build_usage` shows), so the asks go to a node that can +build them; a look that passes again wakes the puller. The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can read holds a key. diff --git a/modules/mesh-build-image/cmd/mesh-build-image/asks.go b/modules/mesh-build-image/cmd/mesh-build-image/asks.go index 71d8644a..ee4edf75 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/asks.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/asks.go @@ -59,7 +59,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep load = func(key string) ([]byte, error) { return m.ReadPauses.Get(key) } } return &buildstep.Puller{Source: source, Language: delivery.AskImage, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, - Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Logf: m.Say, + Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Eligible: m.Eligible, Logf: m.Say, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { if err := m.Check(ctx); err != nil { return nil, err diff --git a/modules/mesh-build-image/cmd/mesh-build-image/asks_test.go b/modules/mesh-build-image/cmd/mesh-build-image/asks_test.go index 46105474..1938cabd 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/asks_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/asks_test.go @@ -71,4 +71,17 @@ func TestAPulledImageAskCopiesThenBuilds(t *testing.T) { t.Errorf("%s.%s is not served", Seat, verb) } } + // A node without /dev/net/tun (a build node, 2026-10-28) takes no ask: its capacity is 0 and says why, so the ask goes to a node + // that can build it instead of failing in buildah's network setup; once it is there again, the capacity is back. + tun := m.Tun + m.Tun = t.TempDir() + "/none" + _ = m.Check(context.Background()) + if c := p.CapacityDetail(); c.Effective != 0 || !strings.Contains(c.Why, "not eligible: "+m.Tun+" cannot be opened") { + t.Errorf("without /dev/net/tun the capacity = %+v", c) + } + m.Tun = tun + _ = m.Check(context.Background()) + if c := p.CapacityDetail(); c.Effective != 1 || c.Ineligible != "" { + t.Errorf("with /dev/net/tun again the capacity = %+v", c) + } } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/helpers_test.go b/modules/mesh-build-image/cmd/mesh-build-image/helpers_test.go index ecaefd90..b8f1357c 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/helpers_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/helpers_test.go @@ -190,6 +190,7 @@ func moduleWith(t *testing.T, f *fake, states *keptStates) *Module { SubUIDFile: write("subuid", "someone:100000:65536\nmesh-build-image:624288:65536\n"), SubGIDFile: write("subgid", "mesh-build-image:624288:65536\n"), RegistryFile: write("registry", f.address()+"\n"), + Tun: write("tun", ""), RegistryWait: 2 * time.Second, RegistryPause: 5 * time.Millisecond, } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/module.go b/modules/mesh-build-image/cmd/mesh-build-image/module.go index 701f519f..819654c1 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/module.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/module.go @@ -38,6 +38,8 @@ type Module struct { Work, Store string // SubUIDFile and SubGIDFile are the subordinate id files; /etc/subuid and /etc/subgid when empty. SubUIDFile, SubGIDFile string + // Tun is the tun device buildah's network for a RUN line (pasta) opens; TunDevice when empty. + Tun string // RegistryFile holds the registry's address, host:port, which the manifest binds from the artifact-store provision. RegistryFile string // RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of @@ -62,6 +64,10 @@ type Module struct { spacesOnce sync.Once workspaces *buildstep.Workspaces + + // lastMu guards last: the refusal of the last look ("" when it could build), which Eligible answers. + lastMu sync.Mutex + last string } // Look is what the module found out about its machine. @@ -90,15 +96,44 @@ func (m *Module) look(ctx context.Context) (Look, error) { if err := m.userNamespace(ctx); err != nil { return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah} } + if err := m.tunDevice(); err != nil { + return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah} + } return l, nil } -// Check is the guard: nil when the machine can build, else the refusal. +// Check is the guard: nil when the machine can build, else the refusal. Every look is kept for Eligible, and a change +// wakes the puller, so a node that can build again takes asks at once. func (m *Module) Check(ctx context.Context) error { _, err := m.look(ctx) + m.keep(err) return err } +// keep records a look's outcome for Eligible and wakes the puller when it changed. +func (m *Module) keep(err error) { + why := "" + if err != nil { + why = err.Error() + } + m.lastMu.Lock() + changed := m.last != why + m.last = why + m.lastMu.Unlock() + if changed && m.Held != nil { + m.Held.Wake() + } +} + +// Eligible is the puller's eligibility (buildstep.Puller.Eligible): the refusal of the module's last look, "" when it +// could build. While the machine cannot build, the node's capacity is 0 and says why, so its asks go to a node that can +// build them instead of failing here (a build node without /dev/net/tun, 2026-10-28). +func (m *Module) Eligible() string { + m.lastMu.Lock() + defer m.lastMu.Unlock() + return m.last +} + // Start looks once, as the module starts, and then every CheckRound until ctx ends; it also removes old workspaces. func (m *Module) Start(ctx context.Context) { for { @@ -119,13 +154,14 @@ func (m *Module) Start(ctx context.Context) { // healthy it names the buildah installed and the account's ranges (ADR 0487 §2: the ranges are proven here, not declared). func (m *Module) Health(ctx context.Context) health.Answer { l, err := m.look(ctx) + m.keep(err) if err != nil { return health.Unhealthy(err.Error()) } if err := m.room(); err != nil { return health.Unhealthy(err.Error()) } - return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made", l.Buildah, l.Account, l.UIDs, l.GIDs)) + return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made; %s opens", l.Buildah, l.Account, l.UIDs, l.GIDs, m.tunPath())) } // Status is what the module reports of itself. diff --git a/modules/mesh-build-image/cmd/mesh-build-image/module_test.go b/modules/mesh-build-image/cmd/mesh-build-image/module_test.go index 4c7af329..cd26685b 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/module_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/module_test.go @@ -43,20 +43,33 @@ func TestHealthIsUnhealthyWithoutBuildah(t *testing.T) { } // The ranges are read from the two files, one each: without one in /etc/subuid or in /etc/subgid the account has no ids to -// map, and the health says which file. -func TestHealthIsUnhealthyWithoutARangeInEitherFile(t *testing.T) { - for file, set := range map[string]func(*Module, string){ - "subuid": func(m *Module, p string) { m.SubUIDFile = p }, - "subgid": func(m *Module, p string) { m.SubGIDFile = p }, +// map, and the health says which file. And without /dev/net/tun (a build node on 2026-10-28: its running kernel's modules were +// gone, so tun could not be loaded) buildah's network for a RUN line cannot be made: the health says the node is not +// eligible, and the puller judges by 0 so its asks go to a node that can build them. +func TestHealthIsUnhealthyWithoutWhatABuildNeeds(t *testing.T) { + for _, c := range []struct { + name string + set func(*Module, string) + says []string + }{ + {"subuid", func(m *Module, p string) { m.SubUIDFile = p }, []string{"/subuid", "mesh-build-image"}}, + {"subgid", func(m *Module, p string) { m.SubGIDFile = p }, []string{"/subgid", "mesh-build-image"}}, + {"tun", func(m *Module, p string) { m.Tun = p + "/dev/net/tun" }, []string{"not eligible: ", "/dev/net/tun cannot be opened", "tun"}}, } { f := newFake(t) m := moduleWith(t, f, newStates()) - p := t.TempDir() + "/" + file + p := t.TempDir() + "/" + c.name os.WriteFile(p, []byte("someone-else:100000:65536\n"), 0o644) - set(m, p) + c.set(m, p) a := m.Health(context.Background()) - if a.Healthy || !strings.Contains(a.Why, p) || !strings.Contains(a.Why, "mesh-build-image") { - t.Errorf("%s: health = %+v, want unhealthy naming %s and the account", file, a, p) + for _, want := range c.says { + if a.Healthy || !strings.Contains(a.Why, want) { + t.Errorf("%s: health = %+v, want unhealthy saying %q", c.name, a, want) + } + } + // Whatever the look refuses, the node is not eligible, with the same words. + if got := m.Eligible(); got != a.Why { + t.Errorf("%s: eligible says %q, the health %q", c.name, got, a.Why) } } } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/probe.go b/modules/mesh-build-image/cmd/mesh-build-image/probe.go index ed3b96f5..88e798ef 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/probe.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/probe.go @@ -2,11 +2,13 @@ package main // What the module proves about its own machine before it builds (novox/hq ADR 0487 §2, issue 1703a): buildah runs, // the account holds subordinate user and group ranges (the node-engine's useradd allocates them for every `user` -// account, so nothing declares them; this reads them), and a user namespace can be made. +// account, so nothing declares them; this reads them), a user namespace can be made, and the tun device opens (buildah +// makes a RUN line's network with pasta, which needs it; a build node on 2026-10-28 had the device node and no tun driver). import ( "bufio" "context" + "errors" "fmt" "os" "path/filepath" @@ -18,6 +20,9 @@ import ( // ProbeTimeout bounds each probe: they read the machine and fetch nothing. const ProbeTimeout = 20 * time.Second +// TunDevice is the tun device pasta opens for a RUN line's network. +const TunDevice = "/dev/net/tun" + // Default subordinate id files. const ( SubUIDFile = "/etc/subuid" @@ -133,3 +138,23 @@ func (m *Module) ensureRuntimeDir() error { } return nil } + +func (m *Module) tunPath() string { + if m.Tun != "" { + return m.Tun + } + return TunDevice +} + +// tunDevice opens the tun device as the account does for a build, and says the node is not eligible when it cannot: a +// device node without its driver answers "no such device" (the tun kernel module is not loaded and cannot be, as when the +// running kernel's modules were removed by an upgrade), and every RUN line would fail in buildah's network setup. +func (m *Module) tunDevice() error { + f, err := os.OpenFile(m.tunPath(), os.O_RDWR, 0) + if err != nil { + return fmt.Errorf("not eligible: %s cannot be opened (%v), so buildah cannot make a RUN line's network (pasta): "+ + "the tun kernel module is not loaded here (the kernel module loads it at boot; a machine whose running kernel's "+ + "modules are gone needs a reboot)", m.tunPath(), errors.Unwrap(err)) + } + return f.Close() +}