mesh/merge-gate head 60010aa1fa12 merged into trunk 89e15c7bcda6 — pass: builds mesh-build-go, mesh-build-image, mesh-build-node → g14, novox, shanks; no b…
mesh/repo-check head 60010aa1fa12 merged into trunk 89e15c7bcda6 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery-group group fix/1839-build-floor rejected: a member's own check failed
mesh/delivery superseded: a newer head of the same pull request
mesh/check failed: unit: logrotate/cmd/logrotate-tools.TestTheBaseParses: manifest_test.go:52: logrotate -d: exit status 1 | 60010aa1fa12 on 89e15c7bc…
mesh-build-go, mesh-build-node and mesh-build-image refuse every fetch, test and build, and say unhealthy, while their work or cache filesystem has less than 20 GiB free (build-agent's floor), so the builder chooses another node instead of filling this one. Each serves build_<x>_release, removing a workspace once its caller's last step answered; a failed fetch removes its own; a workspace never released goes two hours after its last use (ADR 0509) in place of a day, never while a step holds it. Pins mesh-sdk's fix/1839-build-floor (buildstep.Floor, buildstep.Workspaces).
166 lines
6.1 KiB
Go
166 lines
6.1 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
|
"git.novox.be/novox/mesh-sdk/go/health"
|
|
)
|
|
|
|
// CheckRound is how often the module looks at itself again, beside the look before every step and at every health look.
|
|
const CheckRound = 10 * time.Minute
|
|
|
|
// Module is the image build module's one running state.
|
|
type Module struct {
|
|
Node string
|
|
// Account and UID are the account the module runs as; Home is its home.
|
|
Account, UID, Home string
|
|
// Steps is the Put of the steps state; ReadSteps reads it back for the read tools.
|
|
Steps buildstep.Put
|
|
ReadSteps StateRead
|
|
// Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not).
|
|
Ctx context.Context
|
|
Say func(format string, a ...any)
|
|
// Run runs the commands of a step (StepRunner); Work holds the workspaces and Store buildah's image store, both
|
|
// outside the account's home (ADR 0466 §1).
|
|
Run forgekey.Runner
|
|
Work, Store string
|
|
// SubUIDFile and SubGIDFile are the subordinate id files; /etc/subuid and /etc/subgid when empty.
|
|
SubUIDFile, SubGIDFile string
|
|
// RegistryFile holds the registry's address, host:port, which the manifest binds from the artifact-store provision.
|
|
RegistryFile string
|
|
// RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of
|
|
// the SDK's fetch and merge. A test points them at a local repository (buildstep.AllowFileRemote).
|
|
RemoteBase string
|
|
FetchOptions []buildstep.Option
|
|
// HTTP talks to the registry; a client of its own when nil. RegistryWait and RegistryPause bound the wait for a
|
|
// registry held still (RegistryWait and RegistryFirstPause when zero).
|
|
HTTP *http.Client
|
|
RegistryWait time.Duration
|
|
RegistryPause time.Duration
|
|
// Now is the clock; time.Now when nil.
|
|
Now func() time.Time
|
|
// Floor refuses every fetch and build while the work or the image store's filesystem has less free than it (issue
|
|
// 1839); main sets floorOf(node), and a zero Floor checks nothing.
|
|
Floor buildstep.Floor
|
|
|
|
spacesOnce sync.Once
|
|
workspaces *buildstep.Workspaces
|
|
}
|
|
|
|
// Look is what the module found out about its machine.
|
|
type Look struct {
|
|
Buildah string
|
|
UIDs, GIDs int
|
|
Account string
|
|
}
|
|
|
|
// look proves the machine: buildah runs, the account holds ranges, a user namespace can be made. The first thing
|
|
// missing is a *buildstep.Refusal naming it, so a step ends refused and the health says the same.
|
|
func (m *Module) look(ctx context.Context) (Look, error) {
|
|
l := Look{Account: m.Account}
|
|
// The runtime directory comes first: buildah refuses before anything else when its XDG_RUNTIME_DIR is not there
|
|
// (issue 1884), so a look that made it only after `buildah --version` never made it at all.
|
|
if err := m.ensureRuntimeDir(); err != nil {
|
|
return l, &buildstep.Refusal{Text: err.Error()}
|
|
}
|
|
var err error
|
|
if l.Buildah, err = m.buildahVersion(ctx); err != nil {
|
|
return l, &buildstep.Refusal{Text: err.Error()}
|
|
}
|
|
if l.UIDs, l.GIDs, err = m.ranges(); err != nil {
|
|
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
|
|
}
|
|
if err := m.userNamespace(ctx); err != nil {
|
|
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
|
|
}
|
|
return l, nil
|
|
}
|
|
|
|
// Check is the guard: nil when the machine can build, else the refusal.
|
|
func (m *Module) Check(ctx context.Context) error {
|
|
_, err := m.look(ctx)
|
|
return err
|
|
}
|
|
|
|
// Start looks once, as the module starts, and then every CheckRound until ctx ends; it also removes old workspaces.
|
|
func (m *Module) Start(ctx context.Context) {
|
|
for {
|
|
if err := m.Check(ctx); err != nil && m.Say != nil {
|
|
m.Say("this machine cannot build images: %v", err)
|
|
}
|
|
m.CleanWorkspaces(m.clock())
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(CheckRound):
|
|
}
|
|
}
|
|
}
|
|
|
|
// Health is the module's health (the runtime-health tool, ADR 0366): unhealthy while it cannot build, saying why,
|
|
// healthy it names the buildah installed and the account's ranges (ADR 0487 §2: the ranges are proven here, not declared).
|
|
func (m *Module) Health(ctx context.Context) health.Answer {
|
|
l, err := m.look(ctx)
|
|
if err != nil {
|
|
return health.Unhealthy(err.Error())
|
|
}
|
|
if err := m.room(); err != nil {
|
|
return health.Unhealthy(err.Error())
|
|
}
|
|
return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made", l.Buildah, l.Account, l.UIDs, l.GIDs))
|
|
}
|
|
|
|
// Status is what the module reports of itself.
|
|
func (m *Module) Status(ctx context.Context) map[string]any {
|
|
l, err := m.look(ctx)
|
|
out := map[string]any{"node": m.Node, "account": m.Account, "buildah": l.Buildah, "subuids": l.UIDs, "subgids": l.GIDs, "refusing": err != nil}
|
|
if err != nil {
|
|
out["why"] = err.Error()
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Long is a tool that runs a long step (ADR 0469 §2): it answers at once with the step's id, and the step ends on its
|
|
// key in the steps state. The guard runs inside the step, before run, so a machine that cannot build ends every step
|
|
// refused with the cause named.
|
|
func (m *Module) Long(tool string, run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
|
|
return func(args map[string]any) (any, error) {
|
|
id, err := buildstep.Start(m.Ctx, m.Steps, m.Node, tool, func(ctx context.Context) (any, error) {
|
|
if err := m.Check(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := m.room(); err != nil {
|
|
return nil, err
|
|
}
|
|
defer m.holdArg(args)()
|
|
return run(ctx, args)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
key, _ := delivery.StepKey(m.Node, id)
|
|
return map[string]any{"step": id, "key": key, "state": delivery.StepRunning}, nil
|
|
}
|
|
}
|
|
|
|
// Short is a tool that answers in its own call; the guard runs before it, and a refusal is the call's error.
|
|
func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
|
|
return func(args map[string]any) (any, error) {
|
|
if err := m.Check(m.Ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := m.room(); err != nil {
|
|
return nil, err
|
|
}
|
|
defer m.holdArg(args)()
|
|
return run(m.Ctx, args)
|
|
}
|
|
}
|