Files
mesh-catalog/modules/mesh-build-go/cmd/mesh-build-go/module.go
T

292 lines
14 KiB
Go

package main
import (
"context"
"encoding/json"
"fmt"
stdio "git.novox.be/novox/mesh-sdk/go"
"net/http"
"os/exec"
"sync"
"time"
buildrun "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildrun"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
"git.novox.be/novox/mesh-sdk/go/forgekey"
"git.novox.be/novox/mesh-sdk/go/health"
)
// CheckRound is how often the module reads the installed toolchain again, beside the check before every step and
// the one every health look makes.
const CheckRound = 10 * time.Minute
// Module is the Go build module's one running state: the node it runs on, its recipe, the toolchain guard and the
// states it writes (toolchains, steps).
type Module struct {
// SayEvent publishes an event of the go build seat on a seat subject (stdio.SeatPublish; nil says none, a test's): the
// puller's ask-taken, step-progress and step-ended (design 47 G §496, §602).
SayEvent func(subject string, body any, id string) error
Node string
Recipe Recipe
// Probe reads the installed Go version (ProbeGo).
Probe func(ctx context.Context) (string, error)
// Toolchains and Steps are the Put of the states of those names.
Toolchains, Steps steprun.Put
// ReadSteps and ReadToolchainsState read those two states back for the read tools (issue 1562).
// Held is the puller whose running asks a step-end tool will not end (issue 1890); nil in a test.
Held *steprun.Puller
ReadSteps, ReadToolchainsState StateRead
// Detected and ReadDetected are the Put and the read of the state detected-steps (design 47 G §326, §414): the
// answer of the verb steps, one key per artifact.
Detected steprun.Put
ReadDetected func(key string) (json.RawMessage, error)
// ForgetDetected deletes one key of detected-steps; RecordAct writes an act in the hand-act log of the seat
// mesh-hand-acts (§463, §480): a drop writes it first.
ForgetDetected func(key string) error
RecordAct func(what, why string) error
// Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back.
Pauses steprun.Put
ReadPauses StateRead
// ForgetPause deletes one key of the pauses state: a dropped ask's, once it is refused (the seat's verb drop).
ForgetPause func(key string) error
// WatchPauses follows the drops other holders write in the shared pauses state (stdio.State(...).Watch).
WatchPauses func(key string, handle func(stdio.StateChange) error) error
// Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not).
Ctx context.Context
Say func(format string, a ...any)
// Run runs the commands of a step (StepRunner); Work holds the workspaces and Cache the module and build caches,
// both outside the account's home (ADR 0466 §1); Home is the account's home.
Run forgekey.Runner
Home, Work, Cache string
// RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of
// the SDK's fetch and merge. A test points them at a local repository (steprun.AllowFileRemote).
RemoteBase string
FetchOptions []steprun.Option
// HTTP fetches the files an archive artifact is pinned to (url and sha256); a client of its own with a fifteen-minute
// timeout when nil. A test serves its own.
HTTP *http.Client
// UnitBound is the unit level's bound; zero is UnitBound. A test lowers it to see a level over its bound.
UnitBound time.Duration
// IntegrationBounds are the integration level's bounds; zero values are IntegrationSetup and IntegrationRun.
IntegrationBounds Bounds
// Now is the clock the levels' times are read from; time.Now when nil.
Now func() time.Time
// Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is
// unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing.
Bubblewrap func(ctx context.Context) (string, error)
// Parallel is how many go.mod directories a level runs at once; zero is half the processors, and at least one
// (ADR 0536 §3: a node runs two checks at once).
Parallel int
// Inside is the integration level's own process, started shut off in bubblewrap by a test step (level.go): its
// commands run as they are, never shut off a second time.
Inside bool
// GoSeatFile is the binding of the provision go-package-registry, the Go seat's read side (design 51 *Go
// packages*): bound, the mesh's Go modules are fetched through it first, Go's public proxy second, and the forge's
// git over the account's ssh last (Go's direct, kept for this milestone only); unbound, as before, directly.
GoSeatFile string
// ArtifactStoreFile is the binding of the provision artifact-store, read for a group run's package members (§465).
ArtifactStoreFile string
// Call asks a seat's verb over the bus: the publish of a done archive (publish.go). stdio.Ask when nil; a test's
// fake holder.
Call callFunc
// Executable is this program's path, which the integration level starts again shut off; os.Executable when empty.
Executable string
// NodeToolchains is where the Node build module keeps its installed toolchains, bound read-only into the
// integration level (ADR 0533 §1); NodeToolchains when empty. A test points it elsewhere.
NodeToolchains string
// HasNeed answers Has in place of the node's own reading; a test's.
HasNeed func(need string) bool
// StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own
// program, StartBus in bus.go, when nil).
StartBus func(ctx context.Context, dir string) (url string, stop func(), err error)
// Floor refuses every step while the work or cache filesystem has less free than it (issue 1839); main sets
// floorOf(node), and a zero Floor checks nothing.
Floor steprun.Floor
// Settings is the module's settings file, which holds the node's build capacity (sdkstep.CapacitySetting); Capacity,
// when set, answers it instead (a test).
Settings string
Capacity func() int
// SayStep says a pipeline step's end (steprun.AskSteps.SayStep); the seat's publish on this node when nil.
SayStep func(sdkstep.StepEnded) error
// ReadManifest asks the delivery indexer's read tool (ManifestsTool) for one manifest by its key, which a compose ask's
// request names its manifests by (compose.go); it reads none of the indexer's state (ADR 0596); GoModCache, when set, is the Go module cache in place of Cache's (a test).
ReadManifest func(key string) (json.RawMessage, error)
// AskInput asks a seat verb the module invokes (module.json) as the runtime calls it (stdio.Ask): the verb a run ask's
// input names, answering the input by its digest (meshdeliverycheckerclient.ReadInput).
AskInput func(key string, body any) (json.RawMessage, error)
// PlaceHead, when set, lays a run's head at a directory instead of fetching it (a test).
PlaceHead func(ctx context.Context, h sdkstep.RunHead, dir string) error
GoModCache string
// KeptBuilds is the object store of the builds kept for the artifact store's holder (keptbuilds.go); nil keeps none.
// Self is this module's name as the runtime launched it (MESH_MODULE), the first part of the store's name on the bus.
KeptBuilds KeptObjects
Self string
// SayToolchain says an event on a seat's subject (stdio.SeatPublish; nil says none, a test's); ToolchainProgram finds the go
// program whose digest the toolchain's is (exec.LookPath when nil).
SayToolchain func(subject string, body any, id string) error
ToolchainProgram func() (string, error)
reporterOnce sync.Once
reporter *toolchainReporter
digestMu sync.Mutex
digestFor, progDigest string
spacesOnce sync.Once
workspaces *steprun.Workspaces
}
// guard is the SDK guard for the version the probe just read: the module names no Go version (ADR 0477 §2), so the
// version wanted is the one installed and the guard refuses only when go cannot be run at all, writing the installed
// version to <node>.go of the toolchains state every time.
func (m *Module) guard(installed string, probeErr error) steprun.Guard {
return steprun.Guard{Language: "go", Want: installed, Recipe: m.Recipe.Version(), Node: m.Node, Put: m.reportingPut(),
Probe: func(context.Context) (string, error) { return installed, probeErr }}
}
// Check reads the installed toolchain, writes <node>.go of the toolchains state and answers nil, or the refusal when
// go cannot be run (steprun.Guard.Check). A version other than another node's is not a refusal: the indexer waits
// while build nodes disagree (ADR 0477 §2). It runs at start, before every step, at every health look and every
// CheckRound.
func (m *Module) Check(ctx context.Context) error {
installed, err := m.Probe(ctx)
if err := m.guard(installed, err).Check(ctx); err != nil {
return err
}
return m.checkBubblewrap(ctx)
}
// Start checks once, as the module starts, and then every CheckRound until ctx ends.
func (m *Module) Start(ctx context.Context) {
for {
if err := m.Check(ctx); err != nil && m.Say != nil {
m.Say("the toolchain: %v", err)
}
m.CleanWorkspaces(time.Now())
buildrun.EndStaleRound(m.stepScan(), m.Node, m.Say) // asks past the step bound, on the module's own round (issue 1890)
select {
case <-ctx.Done():
return
case <-time.After(CheckRound):
}
}
}
// Health is the module's health (the runtime-health tool, ADR 0366): unhealthy only while go cannot be run, saying why, so the
// controller raises the node's condition; healthy it names the installed version (ADR 0477 §2). It looks again each time it is
// asked.
func (m *Module) Health(ctx context.Context) health.Answer {
installed, perr := m.Probe(ctx)
if err := m.guard(installed, perr).Check(ctx); err != nil {
return health.Unhealthy(err.Error())
}
if err := m.checkBubblewrap(ctx); err != nil {
return health.Unhealthy(err.Error())
}
if err := m.room(); err != nil {
return health.Unhealthy(err.Error())
}
note := ""
if m.Held != nil {
why, fault := m.Held.CapacityNote()
if fault {
return health.Unhealthy(why)
}
if why != "" {
note = "; " + why // a capacity set to 0 is a choice, said so a starved builder is seen
}
}
return health.Healthy(fmt.Sprintf("%s installed, the version the module builds with (recipe %s)", installed, m.Recipe.Version()) + note)
}
// Long is a tool that runs a long step (ADR 0469 §2): it answers at once with the step's id, and the step ends on
// its key in the steps state. The guard runs inside the step, before run, so a node whose go cannot be run ends every step
// refused with the cause named.
func (m *Module) Long(tool string, run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
return func(args map[string]any) (any, error) {
id, err := steprun.Start(m.Ctx, m.Steps, m.Node, tool, func(ctx context.Context) (any, error) {
if err := m.Check(ctx); err != nil {
return nil, err
}
if err := m.room(); err != nil {
return nil, err
}
defer m.holdArg(args)()
return run(ctx, args)
})
if err != nil {
return nil, err
}
key, _ := sdkstep.StepKey(m.Node, id)
return map[string]any{"step": id, "key": key, "state": sdkstep.StepRunning}, nil
}
}
// Short is a tool that answers in its own call; the guard runs before it, and a refusal is the call's error.
func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
return func(args map[string]any) (any, error) {
if err := m.Check(m.Ctx); err != nil {
return nil, err
}
if err := m.room(); err != nil {
return nil, err
}
defer m.holdArg(args)()
return run(m.Ctx, args)
}
}
// Status is what the module reports of itself: the node, the recipe, the toolchain found and why it refuses, if it
// does, and the processors its steps may take (issue 1806).
func (m *Module) Status(ctx context.Context) map[string]any {
err := m.Check(ctx)
installed, _ := m.Probe(ctx)
out := map[string]any{
"node": m.Node, "recipe": m.Recipe.Version(), "installed": installed,
"postgres-major": m.Recipe.PostgresMajor, "nats-server": m.Recipe.NatsServer, "refusing": err != nil,
"cpu": m.cpuStatus(),
}
if err != nil {
out["why"] = err.Error()
}
return out
}
// reportingPut is the toolchains Put that then says toolchain-reported on the Go build seat's subject of this node when
// the toolchain changed (design 47 G §492): at the module's start and on each change of the key. A write that failed
// says nothing; an event that failed is said and does not refuse the step.
func (m *Module) reportingPut() steprun.Put {
m.reporterOnce.Do(func() {
m.reporter = &toolchainReporter{AskLanguage: sdkstep.AskGo, Node: m.Node, Publish: m.SayToolchain}
})
return putReporting(m.Toolchains, m.reporter, m.programDigest, m.Say)
}
// programDigest is the digest of the Go program this node's go-toolchain provides, the `go` found on the path: read again
// only when the installed version changed (design 47 G §472; the lead's answer A-152).
func (m *Module) programDigest(version string) string {
m.digestMu.Lock()
defer m.digestMu.Unlock()
if m.digestFor == version && m.progDigest != "" {
return m.progDigest
}
find := m.ToolchainProgram
if find == nil {
find = func() (string, error) { return exec.LookPath("go") }
}
path, err := find()
if err == nil {
var d string
if d, err = programFileDigest(path); err == nil {
m.digestFor, m.progDigest = version, d
return d
}
}
if m.Say != nil {
m.Say("the digest of the go toolchain program could not be read: %v", err)
}
return ""
}