novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

A node that cannot open /dev/net/tun takes no image build, and every node loads tun at boot and keeps its running kernel's modules

Browse Source 
An image build node up 24 days had lost its running kernel's modules to a linux-lts upgrade, so tun
could not load and every RUN line failed in pasta. mesh-build-image now proves the tun device and,
whatever its look refuses, judges its node not eligible (capacity 0 with the reason), so the asks go
to a node that can build them. The kernel module loads tun at boot and declares kernel-modules-hook,
which keeps the running kernel's modules across an upgrade. Needs mesh-sdk fix/ace-tun (Puller.Eligible).
This commit is contained in:
jschoubben
2026-10-28 12:44:58 +01:00
parent 25f6154564
commit d13f74df40
11 changed files with 173 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-build-image/README.md
+6 -1
View File
@@ -16,7 +16,12 @@ credential and an environment built from scratch.
The account's subordinate user and group ranges are not declared anywhere: the node-engine makes every `user` account
with `useradd`, which allocates them. The module's health reads `/etc/subuid` and `/etc/subgid` and is unhealthy
without a range for the account, without buildah, or when a user namespace cannot be made.
without a range for the account, without buildah, when a user namespace cannot be made, or when `/dev/net/tun`
cannot be opened (buildah makes a RUN line's network with pasta, which needs the tun driver; on 2026-10-28 a build node had the
device node and no driver, so every image build there failed with "Failed to open() /dev/net/tun: No such device"). The
kernel module loads `tun` at boot. Whatever the health refuses, the node is **not eligible**: the seat's puller judges
by a capacity of 0 with that reason (`capacity.<node>`, which `build_usage` shows), so the asks go to a node that can
build them; a look that passes again wakes the puller.
The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can read holds a key.
modules/mesh-build-image/cmd/mesh-build-image/asks.go
+1 -1
View File
@@ -59,7 +59,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
load = func(key string) ([]byte, error) { return m.ReadPauses.Get(key) }
}
return &buildstep.Puller{Source: source, Language: delivery.AskImage, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load,
Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Logf: m.Say,
Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Eligible: m.Eligible, Logf: m.Say,
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
if err := m.Check(ctx); err != nil {
return nil, err
modules/mesh-build-image/cmd/mesh-build-image/asks_test.go
+13
View File
@@ -71,4 +71,17 @@ func TestAPulledImageAskCopiesThenBuilds(t *testing.T) {
t.Errorf("%s.%s is not served", Seat, verb)
}
}
// A node without /dev/net/tun (a build node, 2026-10-28) takes no ask: its capacity is 0 and says why, so the ask goes to a node
// that can build it instead of failing in buildah's network setup; once it is there again, the capacity is back.
tun := m.Tun
m.Tun = t.TempDir() + "/none"
_ = m.Check(context.Background())
if c := p.CapacityDetail(); c.Effective != 0 || !strings.Contains(c.Why, "not eligible: "+m.Tun+" cannot be opened") {
t.Errorf("without /dev/net/tun the capacity = %+v", c)
}
m.Tun = tun
_ = m.Check(context.Background())
if c := p.CapacityDetail(); c.Effective != 1 || c.Ineligible != "" {
t.Errorf("with /dev/net/tun again the capacity = %+v", c)
}
}
modules/mesh-build-image/cmd/mesh-build-image/helpers_test.go
+1
View File
@@ -190,6 +190,7 @@ func moduleWith(t *testing.T, f *fake, states *keptStates) *Module {
SubUIDFile: write("subuid", "someone:100000:65536\nmesh-build-image:624288:65536\n"),
SubGIDFile: write("subgid", "mesh-build-image:624288:65536\n"),
RegistryFile: write("registry", f.address()+"\n"),
Tun: write("tun", ""),
RegistryWait: 2 * time.Second,
RegistryPause: 5 * time.Millisecond,
}
modules/mesh-build-image/cmd/mesh-build-image/module.go
+38 -2
View File
@@ -38,6 +38,8 @@ type Module struct {
Work, Store string
// SubUIDFile and SubGIDFile are the subordinate id files; /etc/subuid and /etc/subgid when empty.
SubUIDFile, SubGIDFile string
// Tun is the tun device buildah's network for a RUN line (pasta) opens; TunDevice when empty.
Tun string
// RegistryFile holds the registry's address, host:port, which the manifest binds from the artifact-store provision.
RegistryFile string
// RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of
@@ -62,6 +64,10 @@ type Module struct {
spacesOnce sync.Once
workspaces *buildstep.Workspaces
// lastMu guards last: the refusal of the last look ("" when it could build), which Eligible answers.
lastMu sync.Mutex
last string
}
// Look is what the module found out about its machine.
@@ -90,15 +96,44 @@ func (m *Module) look(ctx context.Context) (Look, error) {
if err := m.userNamespace(ctx); err != nil {
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
}
if err := m.tunDevice(); err != nil {
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
}
return l, nil
}
// Check is the guard: nil when the machine can build, else the refusal.
// Check is the guard: nil when the machine can build, else the refusal. Every look is kept for Eligible, and a change
// wakes the puller, so a node that can build again takes asks at once.
func (m *Module) Check(ctx context.Context) error {
_, err := m.look(ctx)
m.keep(err)
return err
}
// keep records a look's outcome for Eligible and wakes the puller when it changed.
func (m *Module) keep(err error) {
why := ""
if err != nil {
why = err.Error()
}
m.lastMu.Lock()
changed := m.last != why
m.last = why
m.lastMu.Unlock()
if changed && m.Held != nil {
m.Held.Wake()
}
}
// Eligible is the puller's eligibility (buildstep.Puller.Eligible): the refusal of the module's last look, "" when it
// could build. While the machine cannot build, the node's capacity is 0 and says why, so its asks go to a node that can
// build them instead of failing here (a build node without /dev/net/tun, 2026-10-28).
func (m *Module) Eligible() string {
m.lastMu.Lock()
defer m.lastMu.Unlock()
return m.last
}
// Start looks once, as the module starts, and then every CheckRound until ctx ends; it also removes old workspaces.
func (m *Module) Start(ctx context.Context) {
for {
@@ -119,13 +154,14 @@ func (m *Module) Start(ctx context.Context) {
// healthy it names the buildah installed and the account's ranges (ADR 0487 §2: the ranges are proven here, not declared).
func (m *Module) Health(ctx context.Context) health.Answer {
l, err := m.look(ctx)
m.keep(err)
if err != nil {
return health.Unhealthy(err.Error())
}
if err := m.room(); err != nil {
return health.Unhealthy(err.Error())
}
return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made", l.Buildah, l.Account, l.UIDs, l.GIDs))
return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made; %s opens", l.Buildah, l.Account, l.UIDs, l.GIDs, m.tunPath()))
}
// Status is what the module reports of itself.
modules/mesh-build-image/cmd/mesh-build-image/module_test.go
+22 -9
View File
@@ -43,20 +43,33 @@ func TestHealthIsUnhealthyWithoutBuildah(t *testing.T) {
}
// The ranges are read from the two files, one each: without one in /etc/subuid or in /etc/subgid the account has no ids to
// map, and the health says which file.
func TestHealthIsUnhealthyWithoutARangeInEitherFile(t *testing.T) {
for file, set := range map[string]func(*Module, string){
"subuid": func(m *Module, p string) { m.SubUIDFile = p },
"subgid": func(m *Module, p string) { m.SubGIDFile = p },
// map, and the health says which file. And without /dev/net/tun (a build node on 2026-10-28: its running kernel's modules were
// gone, so tun could not be loaded) buildah's network for a RUN line cannot be made: the health says the node is not
// eligible, and the puller judges by 0 so its asks go to a node that can build them.
func TestHealthIsUnhealthyWithoutWhatABuildNeeds(t *testing.T) {
for _, c := range []struct {
name string
set func(*Module, string)
says []string
}{
{"subuid", func(m *Module, p string) { m.SubUIDFile = p }, []string{"/subuid", "mesh-build-image"}},
{"subgid", func(m *Module, p string) { m.SubGIDFile = p }, []string{"/subgid", "mesh-build-image"}},
{"tun", func(m *Module, p string) { m.Tun = p + "/dev/net/tun" }, []string{"not eligible: ", "/dev/net/tun cannot be opened", "tun"}},
} {
f := newFake(t)
m := moduleWith(t, f, newStates())
p := t.TempDir() + "/" + file
p := t.TempDir() + "/" + c.name
os.WriteFile(p, []byte("someone-else:100000:65536\n"), 0o644)
set(m, p)
c.set(m, p)
a := m.Health(context.Background())
if a.Healthy || !strings.Contains(a.Why, p) || !strings.Contains(a.Why, "mesh-build-image") {
t.Errorf("%s: health = %+v, want unhealthy naming %s and the account", file, a, p)
for _, want := range c.says {
if a.Healthy || !strings.Contains(a.Why, want) {
t.Errorf("%s: health = %+v, want unhealthy saying %q", c.name, a, want)
}
}
// Whatever the look refuses, the node is not eligible, with the same words.
if got := m.Eligible(); got != a.Why {
t.Errorf("%s: eligible says %q, the health %q", c.name, got, a.Why)
}
}
}
modules/mesh-build-image/cmd/mesh-build-image/probe.go
+26 -1
View File
@@ -2,11 +2,13 @@ package main
// What the module proves about its own machine before it builds (novox/hq ADR 0487 §2, issue 1703a): buildah runs,
// the account holds subordinate user and group ranges (the node-engine's useradd allocates them for every `user`
// account, so nothing declares them; this reads them), and a user namespace can be made.
// account, so nothing declares them; this reads them), a user namespace can be made, and the tun device opens (buildah
// makes a RUN line's network with pasta, which needs it; a build node on 2026-10-28 had the device node and no tun driver).
import (
"bufio"
"context"
"errors"
"fmt"
"os"
"path/filepath"
@@ -18,6 +20,9 @@ import (
// ProbeTimeout bounds each probe: they read the machine and fetch nothing.
const ProbeTimeout = 20 * time.Second
// TunDevice is the tun device pasta opens for a RUN line's network.
const TunDevice = "/dev/net/tun"
// Default subordinate id files.
const (
SubUIDFile = "/etc/subuid"
@@ -133,3 +138,23 @@ func (m *Module) ensureRuntimeDir() error {
}
return nil
}
func (m *Module) tunPath() string {
if m.Tun != "" {
return m.Tun
}
return TunDevice
}
// tunDevice opens the tun device as the account does for a build, and says the node is not eligible when it cannot: a
// device node without its driver answers "no such device" (the tun kernel module is not loaded and cannot be, as when the
// running kernel's modules were removed by an upgrade), and every RUN line would fail in buildah's network setup.
func (m *Module) tunDevice() error {
f, err := os.OpenFile(m.tunPath(), os.O_RDWR, 0)
if err != nil {
return fmt.Errorf("not eligible: %s cannot be opened (%v), so buildah cannot make a RUN line's network (pasta): "+
"the tun kernel module is not loaded here (the kernel module loads it at boot; a machine whose running kernel's "+
"modules are gone needs a reboot)", m.tunPath(), errors.Unwrap(err))
}
return f.Close()
}