211 lines
7.6 KiB
Go
211 lines
7.6 KiB
Go
package main
|
|
|
|
// **A compose ask runs mesh-sdk's composer at a commit** (novox/hq ADR 0482 clause 4, issues 1621 and 1622): the
|
|
// assigner asks it, to compose a group with its mesh-sdk member's head, and to read the group's manifests with the
|
|
// mesh-sdk the running controller and node-engine pin (ADR 0482 clause 5, issue 1623). The module fetches the commit
|
|
// (its fetch step: the forge key and the network, none of the change's code), builds the composer package in the Go
|
|
// module the ask names, in a run step shut off with nothing fetched, and keeps the program under its key
|
|
// (delivery.ComposerKey), so each commit is built once on a node. It fills the request's manifests from the delivery
|
|
// indexer's manifests state, by digest, and runs the program shut off: no network, none of the account, the request a
|
|
// file of a workspace of its own. The answer is the composer's verdict whole (delivery.ComposeAnswer); the module
|
|
// judges nothing of it.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/composer"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
// ManifestsState is the delivery indexer's manifests by digest, which a compose ask's request names (the module reads
|
|
// it, module.json reads).
|
|
const ManifestsState = delivery.IndexerModule + ".manifests"
|
|
|
|
// composerLocks hold one mutex per composer key, so a composer two asks need at once is built once.
|
|
var composerLocks sync.Map
|
|
|
|
// composerPath is where the composer of key is kept: the cache directory, beside Go's own caches.
|
|
func (m *Module) composerPath(key string) string {
|
|
return filepath.Join(m.Cache, "composers", key, "mesh-compose")
|
|
}
|
|
|
|
// compose runs a compose ask: the composer of its commit, built once, over its request with the manifests filled.
|
|
func (m *Module) compose(ctx context.Context, fetch buildstep.StepFunc, a delivery.StepAsk) (any, error) {
|
|
job := *a.Compose
|
|
repository, _ := job.Fetch["repository"].(string)
|
|
commit, _ := job.Fetch["commit"].(string)
|
|
installed, err := m.Probe(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
key := delivery.ComposerKey(repository, commit, job.Dir, job.Package, delivery.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()})
|
|
exe, built, err := m.composerOf(ctx, fetch, job, key)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req, err := composer.Decode(job.Request)
|
|
if err != nil {
|
|
return nil, buildstep.Refuse("the compose ask %s: %v", a.ID, err)
|
|
}
|
|
if err := composer.Fill(&req, m.manifestOf); err != nil {
|
|
return nil, err
|
|
}
|
|
verdict, err := m.runComposer(ctx, exe, req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return delivery.ComposeAnswer{Built: built, Key: key, Verdict: verdict}, nil
|
|
}
|
|
|
|
// manifestOf is the manifest the indexer keeps under digest, as its repository holds it.
|
|
func (m *Module) manifestOf(digest string) ([]byte, error) {
|
|
if m.ReadManifest == nil {
|
|
return nil, errors.New("this module reads no manifests state")
|
|
}
|
|
key, err := delivery.ManifestsKey(digest)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
raw, err := m.ReadManifest(key)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if raw == nil {
|
|
return nil, fmt.Errorf("the indexer keeps no manifest of digest %s", digest)
|
|
}
|
|
man, err := delivery.DecodeManifest(raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return []byte(man.JSON), nil
|
|
}
|
|
|
|
// composerOf is the composer kept under key, or, when none is, the one built now from the fetched commit (built true).
|
|
func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job delivery.ComposeJob, key string) (string, bool, error) {
|
|
exe := m.composerPath(key)
|
|
mu, _ := composerLocks.LoadOrStore(exe, &sync.Mutex{})
|
|
mu.(*sync.Mutex).Lock()
|
|
defer mu.(*sync.Mutex).Unlock()
|
|
if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() {
|
|
return exe, false, nil
|
|
}
|
|
if fetch == nil {
|
|
return "", false, buildstep.Refuse("this build module fetches nothing")
|
|
}
|
|
out, err := fetch(ctx, job.Fetch)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
raw, err := json.Marshal(out)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
var f struct {
|
|
Workspace string `json:"workspace"`
|
|
}
|
|
if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" {
|
|
return "", false, fmt.Errorf("the fetch answered no workspace: %s", raw)
|
|
}
|
|
defer m.releaseSaid(f.Workspace)
|
|
done := m.spaces().Hold(f.Workspace)
|
|
defer done()
|
|
dir := filepath.Join(m.Work, f.Workspace, "tree", job.Dir)
|
|
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil {
|
|
return "", false, fmt.Errorf("%s at %s holds no go.mod in %s, so no composer can be built there", job.Fetch["repository"], job.Fetch["commit"], job.Dir)
|
|
}
|
|
into := filepath.Join(m.Work, f.Workspace, "composer", "mesh-compose")
|
|
if err := os.MkdirAll(filepath.Dir(into), 0o700); err != nil {
|
|
return "", false, err
|
|
}
|
|
run, err := m.stepRunner(f.Workspace, false)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
env := append(m.goEnvFor(dir), "GOPROXY=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0")
|
|
r := run(ctx, env, "go", "-C", dir, "build", "-trimpath", "-o", into, job.Package)
|
|
if r.Status != 0 || r.Err != "" {
|
|
// The words Go says when the module holds no such package (a mesh-sdk from before the composer) are kept whole,
|
|
// so the asker tells that apart from a fault.
|
|
return "", false, fmt.Errorf("the composer %s does not build at %s: %s", job.Package, job.Fetch["commit"], compileError(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
|
}
|
|
if err := copyExecutable(into, exe); err != nil {
|
|
return "", false, fmt.Errorf("the composer could not be kept: %w", err)
|
|
}
|
|
return exe, true, nil
|
|
}
|
|
|
|
// copyExecutable copies a built program into place through a file beside it, so a half-written program is never kept.
|
|
func copyExecutable(from, to string) error {
|
|
if err := os.MkdirAll(filepath.Dir(to), 0o700); err != nil {
|
|
return err
|
|
}
|
|
src, err := os.Open(from)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer src.Close()
|
|
tmp := to + ".new"
|
|
dst, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := io.Copy(dst, src); err != nil {
|
|
dst.Close()
|
|
return err
|
|
}
|
|
if err := dst.Close(); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(tmp, to)
|
|
}
|
|
|
|
// releaseSaid removes a workspace once its steps ended, saying a failure: the sweep removes it later.
|
|
func (m *Module) releaseSaid(workspace string) {
|
|
if err := m.spaces().Release(workspace); err != nil && m.Say != nil {
|
|
m.Say("the compose ask's workspace %s: %v", workspace, err)
|
|
}
|
|
}
|
|
|
|
// runComposer runs the composer over req in a workspace of its own, shut off, and answers its verdict: the last line of
|
|
// its standard output, a composer.Verdict of the version this module reads.
|
|
func (m *Module) runComposer(ctx context.Context, exe string, req composer.Request) (json.RawMessage, error) {
|
|
id := buildstep.NewID()
|
|
done := m.spaces().Hold(id)
|
|
defer m.releaseSaid(id)
|
|
defer done()
|
|
ws := filepath.Join(m.Work, id)
|
|
if err := os.MkdirAll(ws, 0o700); err != nil {
|
|
return nil, err
|
|
}
|
|
raw, err := json.Marshal(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
file := filepath.Join(ws, "request.json")
|
|
if err := os.WriteFile(file, raw, 0o600); err != nil {
|
|
return nil, err
|
|
}
|
|
run := m.Run
|
|
if !m.Inside {
|
|
iso := m.isolation(id, false)
|
|
iso.ReadOnly = append(iso.ReadOnly, exe)
|
|
run = buildstep.Isolated(m.Run, iso)
|
|
}
|
|
r := run(ctx, nil, exe, "-request", file)
|
|
line := strings.TrimSpace(lastLine(r.Stdout))
|
|
var v composer.Verdict
|
|
if r.Status != 0 || r.Err != "" || json.Unmarshal([]byte(line), &v) != nil || v.V != composer.Version {
|
|
return nil, fmt.Errorf("the composer answered no verdict (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout))
|
|
}
|
|
return json.RawMessage(line), nil
|
|
}
|