mesh-catalog/modules/mesh-build-go/cmd/mesh-build-go/compose.go

package main

// **A compose ask runs a composer command at a commit** (novox/hq ADR 0482 clause 4, issues 1621 and 1622): the command is
// the node-engine's, mesh-compose in its repository's Go module at the root, since the node-engine alone composes
// (ADR 0593 §7); the ask names the repository, the directory and the package, and this module builds whatever it names. A
// group that changes the node-engine's composition is composed by the command built from its head, as the running
// engine cannot judge its own next code. The module fetches the commit
// (its fetch step: the forge key and the network, none of the change's code), builds the composer package in the Go
// module the ask names, in a run step shut off with nothing fetched, and keeps the program under its key
// (sdkstep.ComposerKey), so each commit is built once on a node. It fills the request's manifests from the delivery
// indexer's manifests state, by digest, and runs the program shut off: no network, none of the account, the request a
// file of a workspace of its own. The answer is the composer's verdict whole (sdkstep.ComposeAnswer); the module
// judges nothing of it.

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
	"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
	catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
	sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
	"git.novox.be/novox/mesh-delivery/sdk/composer"
	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/views"
)

// ManifestsTool is the delivery indexer's read tool of its manifests by digest, which a compose ask's request names (the
// module invokes it, module.json invokes). A manifest by its digest never changes and a compose ask is rare, so the
// module asks for the one key it needs when it needs it, and keeps no picture of the indexer's whole state (ADR 0615).
// indexerModule is the module that holds the seat mesh-catalog, whose manifests state a compose ask reads.
const indexerModule = "mesh-delivery-indexer"

const ManifestsTool = indexerModule + ".indexer_read_manifests"

// composerLocks hold one mutex per composer key, so a composer two asks need at once is built once.
var composerLocks sync.Map

// composerPath is where the composer of key is kept: the cache directory, beside Go's own caches.
func (m *Module) composerPath(key string) string {
	return m.programPath("composers", key)
}

// compose runs a compose ask: the composer of its commit, built once, over its request with the manifests filled.
func (m *Module) compose(ctx context.Context, fetch buildstep.StepFunc, a sdkstep.StepAsk) (any, error) {
	job := *a.Compose
	repository, commit := job.Fetch.Repository, job.Fetch.Commit
	installed, err := m.Probe(ctx)
	if err != nil {
		return nil, err
	}
	key := sdkstep.ComposerKey(repository, commit, job.Dir, job.Package, sdkstep.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()}, job.Fetch.Packages)
	exe, built, err := m.composerOf(ctx, fetch, job, key)
	if err != nil {
		return nil, err
	}
	body, err := job.RequestBytes()
	if err != nil {
		return nil, buildstep.Refuse("the compose ask %s: %v", a.ID, err)
	}
	req, err := composer.Decode(body)
	if err != nil {
		return nil, buildstep.Refuse("the compose ask %s: %v", a.ID, err)
	}
	if err := composer.Fill(&req, m.manifestOf); err != nil {
		return nil, err
	}
	verdict, err := m.runComposer(ctx, exe, req)
	if err != nil {
		return nil, err
	}
	return sdkstep.ComposeAnswer{Built: built, Key: key, Verdict: verdict}, nil
}

// askManifest asks the indexer's read tool for one manifest by its key: nil when it keeps none. A call the indexer does not
// answer is a finding of this module's own, which the condition keeper raises (ADR 0615 §6) and the next answer clears.
func askManifest(key string) (json.RawMessage, error) {
	return askManifestVia(stdio.Ask, meshconditionsclient.FindingWriter{State: stdio.State("findings")}, time.Now, key)
}

func askManifestVia(ask func(string, any) (json.RawMessage, error), findings views.FindingWriter, now func() time.Time, key string) (json.RawMessage, error) {
	st := views.Status{Owner: indexerModule, Tool: ManifestsTool, State: "manifests", Asked: now().UTC()}
	raw, err := ask(ManifestsTool, map[string]any{"key": key})
	if err != nil {
		st.Error = err.Error()
		_ = findings.Put(views.UnansweredFinding("mesh-build-go", st, now()))
		return nil, err
	}
	_ = findings.Delete(views.UnansweredFinding("mesh-build-go", st, now()))
	var a struct {
		Found bool            `json:"found"`
		Value json.RawMessage `json:"value"`
	}
	if err := json.Unmarshal(raw, &a); err != nil {
		return nil, fmt.Errorf("%s answered something that is not a key's value: %w", ManifestsTool, err)
	}
	if !a.Found {
		return nil, nil
	}
	return a.Value, nil
}

// manifestOf is the manifest the indexer keeps under digest, as its repository holds it.
func (m *Module) manifestOf(digest string) ([]byte, error) {
	if m.ReadManifest == nil {
		return nil, errors.New("this module reads no manifests state")
	}
	key, err := catalogclient.ManifestsKey(digest)
	if err != nil {
		return nil, err
	}
	raw, err := m.ReadManifest(key)
	if err != nil {
		return nil, err
	}
	if raw == nil {
		return nil, fmt.Errorf("the indexer keeps no manifest of digest %s", digest)
	}
	man, err := catalogclient.DecodeManifest(raw)
	if err != nil {
		return nil, err
	}
	return []byte(man.Manifest), nil
}

// composerOf is the composer kept under key, or, when none is, the one built now from the fetched commit (built true).
func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job sdkstep.ComposeJob, key string) (string, bool, error) {
	args, err := buildstep.FetchArgs(job.Fetch)
	if err != nil {
		return "", false, err
	}
	return m.programOf(ctx, fetch, "composers", "composer", key, args, job.Dir, job.Package)
}

// programOf is the program kept under key in the cache directory kind, or, when none is, the one built now: the commit
// fetch names is fetched (the fetch step: the forge key and the network, none of the change's code), and package in the
// Go module of dir is built in a run step shut off with nothing fetched. what names it in a failure.
func (m *Module) programOf(ctx context.Context, fetch buildstep.StepFunc, kind, what, key string, fetchArgs map[string]any, dir, pkg string) (string, bool, error) {
	exe := m.programPath(kind, key)
	mu := lockOf(exe)
	mu.Lock()
	defer mu.Unlock()
	if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() {
		touch(exe, m.clock())
		return exe, false, nil
	}
	if fetch == nil {
		return "", false, buildstep.Refuse("this build module fetches nothing")
	}
	out, err := fetch(ctx, fetchArgs)
	if err != nil {
		return "", false, err
	}
	raw, err := json.Marshal(out)
	if err != nil {
		return "", false, err
	}
	var f struct {
		Workspace string `json:"workspace"`
	}
	if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" {
		return "", false, fmt.Errorf("the fetch answered no workspace: %s", raw)
	}
	defer m.releaseSaid(f.Workspace)
	done := m.spaces().Hold(f.Workspace)
	defer done()
	tree := filepath.Join(m.Work, f.Workspace, "tree", dir)
	if _, err := os.Stat(filepath.Join(tree, "go.mod")); err != nil {
		return "", false, fmt.Errorf("%s at %s holds no go.mod in %s, so no %s can be built there", fetchArgs["repository"], fetchArgs["commit"], dir, what)
	}
	into := filepath.Join(m.Work, f.Workspace, what, "built")
	if err := os.MkdirAll(filepath.Dir(into), 0o700); err != nil {
		return "", false, err
	}
	run, err := m.stepRunner(f.Workspace, false)
	if err != nil {
		return "", false, err
	}
	env := append(m.goEnvFor(tree), "GOPROXY=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0")
	r := run(ctx, env, "go", "-C", tree, "build", "-trimpath", "-o", into, pkg)
	if r.Status != 0 || r.Err != "" {
		// The words Go says when the module holds no such package (a mesh-sdk from before the composer) are kept whole,
		// so the asker tells that apart from a fault.
		return "", false, fmt.Errorf("the %s %s does not build at %s: %s", what, pkg, fetchArgs["commit"], compileError(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
	}
	if err := copyExecutable(into, exe); err != nil {
		return "", false, fmt.Errorf("the %s could not be kept: %w", what, err)
	}
	return exe, true, nil
}

// copyExecutable copies a built program into place through a file beside it, so a half-written program is never kept.
func copyExecutable(from, to string) error {
	if err := os.MkdirAll(filepath.Dir(to), 0o700); err != nil {
		return err
	}
	src, err := os.Open(from)
	if err != nil {
		return err
	}
	defer src.Close()
	tmp := to + ".new"
	dst, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
	if err != nil {
		return err
	}
	if _, err := io.Copy(dst, src); err != nil {
		dst.Close()
		return err
	}
	if err := dst.Close(); err != nil {
		return err
	}
	return os.Rename(tmp, to)
}

// releaseSaid removes a workspace once its steps ended, saying a failure: the sweep removes it later.
func (m *Module) releaseSaid(workspace string) {
	if err := m.spaces().Release(workspace); err != nil && m.Say != nil {
		m.Say("the compose ask's workspace %s: %v", workspace, err)
	}
}

// runComposer runs the composer over req in a workspace of its own, shut off, and answers its verdict: the last line of
// its standard output, a composer.Verdict of the version this module reads.
func (m *Module) runComposer(ctx context.Context, exe string, req composer.Request) (json.RawMessage, error) {
	id := sdkstep.NewID()
	done := m.spaces().Hold(id)
	defer m.releaseSaid(id)
	defer done()
	ws := filepath.Join(m.Work, id)
	if err := os.MkdirAll(ws, 0o700); err != nil {
		return nil, err
	}
	raw, err := json.Marshal(req)
	if err != nil {
		return nil, err
	}
	file := filepath.Join(ws, "request.json")
	if err := os.WriteFile(file, raw, 0o600); err != nil {
		return nil, err
	}
	run := m.Run
	if !m.Inside {
		iso := m.isolation(id, false)
		iso.ReadOnly = append(iso.ReadOnly, exe)
		run = buildstep.Isolated(m.Run, iso)
	}
	r := run(ctx, nil, exe, "-request", file)
	line := strings.TrimSpace(lastLine(r.Stdout))
	var v composer.Verdict
	if r.Status != 0 || r.Err != "" || json.Unmarshal([]byte(line), &v) != nil || v.V != composer.Version {
		return nil, fmt.Errorf("the composer answered no verdict (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout))
	}
	return json.RawMessage(line), nil
}