jschoubben
d13f74df40
An image build node up 24 days had lost its running kernel's modules to a linux-lts upgrade, so tun could not load and every RUN line failed in pasta. mesh-build-image now proves the tun device and, whatever its look refuses, judges its node not eligible (capacity 0 with the reason), so the asks go to a node that can build them. The kernel module loads tun at boot and declares kernel-modules-hook, which keeps the running kernel's modules across an upgrade. Needs mesh-sdk fix/ace-tun (Puller.Eligible).
214 lines
8.0 KiB
Go
214 lines
8.0 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
|
"git.novox.be/novox/mesh-sdk/go/health"
|
|
)
|
|
|
|
// CheckRound is how often the module looks at itself again, beside the look before every step and at every health look.
|
|
const CheckRound = 10 * time.Minute
|
|
|
|
// Module is the image build module's one running state.
|
|
type Module struct {
|
|
Node string
|
|
// Account and UID are the account the module runs as; Home is its home.
|
|
Account, UID, Home string
|
|
// Steps is the Put of the steps state; ReadSteps reads it back for the read tools.
|
|
Steps buildstep.Put
|
|
// Held is the puller whose running asks a step-end tool will not end (issue 1890); nil in a test.
|
|
Held *buildstep.Puller
|
|
ReadSteps StateRead
|
|
// Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back.
|
|
Pauses buildstep.Put
|
|
ReadPauses StateRead
|
|
// Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not).
|
|
Ctx context.Context
|
|
Say func(format string, a ...any)
|
|
// Run runs the commands of a step (StepRunner); Work holds the workspaces and Store buildah's image store, both
|
|
// outside the account's home (ADR 0466 §1).
|
|
Run forgekey.Runner
|
|
Work, Store string
|
|
// SubUIDFile and SubGIDFile are the subordinate id files; /etc/subuid and /etc/subgid when empty.
|
|
SubUIDFile, SubGIDFile string
|
|
// Tun is the tun device buildah's network for a RUN line (pasta) opens; TunDevice when empty.
|
|
Tun string
|
|
// RegistryFile holds the registry's address, host:port, which the manifest binds from the artifact-store provision.
|
|
RegistryFile string
|
|
// RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of
|
|
// the SDK's fetch and merge. A test points them at a local repository (buildstep.AllowFileRemote).
|
|
RemoteBase string
|
|
FetchOptions []buildstep.Option
|
|
// HTTP talks to the registry; a client of its own when nil. RegistryWait and RegistryPause bound the wait for a
|
|
// registry held still (RegistryWait and RegistryFirstPause when zero).
|
|
HTTP *http.Client
|
|
RegistryWait time.Duration
|
|
RegistryPause time.Duration
|
|
// Now is the clock; time.Now when nil.
|
|
Now func() time.Time
|
|
// Floor refuses every fetch and build while the work or the image store's filesystem has less free than it (issue
|
|
// 1839); main sets floorOf(node), and a zero Floor checks nothing.
|
|
Floor buildstep.Floor
|
|
|
|
// Settings is the module's settings file, which holds the node's build capacity (buildstep.CapacitySetting); Capacity,
|
|
// when set, answers it instead (a test).
|
|
Settings string
|
|
Capacity func() int
|
|
|
|
spacesOnce sync.Once
|
|
workspaces *buildstep.Workspaces
|
|
|
|
// lastMu guards last: the refusal of the last look ("" when it could build), which Eligible answers.
|
|
lastMu sync.Mutex
|
|
last string
|
|
}
|
|
|
|
// Look is what the module found out about its machine.
|
|
type Look struct {
|
|
Buildah string
|
|
UIDs, GIDs int
|
|
Account string
|
|
}
|
|
|
|
// look proves the machine: buildah runs, the account holds ranges, a user namespace can be made. The first thing
|
|
// missing is a *buildstep.Refusal naming it, so a step ends refused and the health says the same.
|
|
func (m *Module) look(ctx context.Context) (Look, error) {
|
|
l := Look{Account: m.Account}
|
|
// The runtime directory comes first: buildah refuses before anything else when its XDG_RUNTIME_DIR is not there
|
|
// (issue 1884), so a look that made it only after `buildah --version` never made it at all.
|
|
if err := m.ensureRuntimeDir(); err != nil {
|
|
return l, &buildstep.Refusal{Text: err.Error()}
|
|
}
|
|
var err error
|
|
if l.Buildah, err = m.buildahVersion(ctx); err != nil {
|
|
return l, &buildstep.Refusal{Text: err.Error()}
|
|
}
|
|
if l.UIDs, l.GIDs, err = m.ranges(); err != nil {
|
|
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
|
|
}
|
|
if err := m.userNamespace(ctx); err != nil {
|
|
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
|
|
}
|
|
if err := m.tunDevice(); err != nil {
|
|
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
|
|
}
|
|
return l, nil
|
|
}
|
|
|
|
// Check is the guard: nil when the machine can build, else the refusal. Every look is kept for Eligible, and a change
|
|
// wakes the puller, so a node that can build again takes asks at once.
|
|
func (m *Module) Check(ctx context.Context) error {
|
|
_, err := m.look(ctx)
|
|
m.keep(err)
|
|
return err
|
|
}
|
|
|
|
// keep records a look's outcome for Eligible and wakes the puller when it changed.
|
|
func (m *Module) keep(err error) {
|
|
why := ""
|
|
if err != nil {
|
|
why = err.Error()
|
|
}
|
|
m.lastMu.Lock()
|
|
changed := m.last != why
|
|
m.last = why
|
|
m.lastMu.Unlock()
|
|
if changed && m.Held != nil {
|
|
m.Held.Wake()
|
|
}
|
|
}
|
|
|
|
// Eligible is the puller's eligibility (buildstep.Puller.Eligible): the refusal of the module's last look, "" when it
|
|
// could build. While the machine cannot build, the node's capacity is 0 and says why, so its asks go to a node that can
|
|
// build them instead of failing here (a build node without /dev/net/tun, 2026-10-28).
|
|
func (m *Module) Eligible() string {
|
|
m.lastMu.Lock()
|
|
defer m.lastMu.Unlock()
|
|
return m.last
|
|
}
|
|
|
|
// Start looks once, as the module starts, and then every CheckRound until ctx ends; it also removes old workspaces.
|
|
func (m *Module) Start(ctx context.Context) {
|
|
for {
|
|
if err := m.Check(ctx); err != nil && m.Say != nil {
|
|
m.Say("this machine cannot build images: %v", err)
|
|
}
|
|
m.CleanWorkspaces(m.clock())
|
|
buildstep.EndStaleRound(m.stepScan(), m.Node, m.Say) // asks past the step bound, on the module's own round (issue 1890)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(CheckRound):
|
|
}
|
|
}
|
|
}
|
|
|
|
// Health is the module's health (the runtime-health tool, ADR 0366): unhealthy while it cannot build, saying why,
|
|
// healthy it names the buildah installed and the account's ranges (ADR 0487 §2: the ranges are proven here, not declared).
|
|
func (m *Module) Health(ctx context.Context) health.Answer {
|
|
l, err := m.look(ctx)
|
|
m.keep(err)
|
|
if err != nil {
|
|
return health.Unhealthy(err.Error())
|
|
}
|
|
if err := m.room(); err != nil {
|
|
return health.Unhealthy(err.Error())
|
|
}
|
|
return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made; %s opens", l.Buildah, l.Account, l.UIDs, l.GIDs, m.tunPath()))
|
|
}
|
|
|
|
// Status is what the module reports of itself.
|
|
func (m *Module) Status(ctx context.Context) map[string]any {
|
|
l, err := m.look(ctx)
|
|
out := map[string]any{"node": m.Node, "account": m.Account, "buildah": l.Buildah, "subuids": l.UIDs, "subgids": l.GIDs, "refusing": err != nil,
|
|
"cpu": m.cpuStatus()}
|
|
if err != nil {
|
|
out["why"] = err.Error()
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Long is a tool that runs a long step (ADR 0469 §2): it answers at once with the step's id, and the step ends on its
|
|
// key in the steps state. The guard runs inside the step, before run, so a machine that cannot build ends every step
|
|
// refused with the cause named.
|
|
func (m *Module) Long(tool string, run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
|
|
return func(args map[string]any) (any, error) {
|
|
id, err := buildstep.Start(m.Ctx, m.Steps, m.Node, tool, func(ctx context.Context) (any, error) {
|
|
if err := m.Check(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := m.room(); err != nil {
|
|
return nil, err
|
|
}
|
|
defer m.holdArg(args)()
|
|
return run(ctx, args)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
key, _ := delivery.StepKey(m.Node, id)
|
|
return map[string]any{"step": id, "key": key, "state": delivery.StepRunning}, nil
|
|
}
|
|
}
|
|
|
|
// Short is a tool that answers in its own call; the guard runs before it, and a refusal is the call's error.
|
|
func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
|
|
return func(args map[string]any) (any, error) {
|
|
if err := m.Check(m.Ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := m.room(); err != nil {
|
|
return nil, err
|
|
}
|
|
defer m.holdArg(args)()
|
|
return run(m.Ctx, args)
|
|
}
|
|
}
|