mesh-catalog/modules/mesh-build-image/cmd/mesh-build-image/fetch.go

package main

// The fetch step (novox/hq ADR 0487 §1, ADR 0466 §4 §5, ADR 0469 §2; issue 1703b): a commit of a repository of the
// mesh's forge, fetched over SSH as the module's own account into a workspace of its own. Nothing is downloaded
// beyond it: an image's Dockerfile fetches what it needs itself, inside the build.

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
	sdkstep "git.novox.be/novox/mesh-sdk/go/buildstep"
)

var (
	commitOK     = regexp.MustCompile(`^[0-9a-f]{40}$`)
	repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
	workspaceOK  = regexp.MustCompile(`^[a-z0-9]+$`)
)

// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
const ForgeAlias = "mesh-forge"

// FetchResult is what a passed fetch step says: the workspace it made and the commit in it, and, only when the call
// gave a base, the merge preview of ADR 0484 clause 3 (the trunk commit, the merge commit of the head onto it, the
// files in conflict, the paths the merge changes and the tree of each module directory). A head that does not
// merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is left at the trunk.
type FetchResult struct {
	Workspace string   `json:"workspace"`
	Commit    string   `json:"commit"`
	Base      string   `json:"base,omitempty"`
	Merge     string   `json:"merge,omitempty"`
	Conflicts []string `json:"conflicts,omitempty"`
	Changed   []string `json:"changed,omitempty"`
	// Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge.
	Programs *[]buildstep.Program          `json:"programs,omitempty"`
	Trees    map[string]sdkstep.ModuleTree `json:"trees,omitempty"`
}

func (m *Module) fetchTool() StepTool {
	return StepTool{
		Name: FetchTool,
		Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
			"workspace of its own. Answers a step id at once; the step ends on its key in the steps state with the " +
			"workspace's id.",
		Input: map[string]any{
			"type": "object",
			"properties": map[string]any{
				"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
				"commit":     map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
				"ref":        map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
				"base":       map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
			},
			"required": []string{"repository", "commit"},
		},
		Long: true,
		Run:  m.fetch,
	}
}

// fetch makes a workspace and fills it. The workspace is held while the fetch runs, and removed when the fetch fails,
// since nobody learns its id to release it (issue 1839).
func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
	repository, _ := args["repository"].(string)
	commit, _ := args["commit"].(string)
	ref, _ := args["ref"].(string)
	base, _ := args["base"].(string)
	if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
		return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
	}
	if !commitOK.MatchString(commit) {
		return nil, fmt.Errorf("%q is not a full commit (40 hex digits)", commit)
	}
	if base != "" && !commitOK.MatchString(base) {
		return nil, fmt.Errorf("%q is not a trunk commit (40 hex digits)", base)
	}
	m.CleanWorkspaces(m.clock())
	id := sdkstep.NewID()
	done := m.spaces().Hold(id)
	defer func() {
		done()
		if err != nil {
			if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
				m.Say("the failed fetch's workspace %s: %v", id, rerr)
			}
		}
	}()
	return m.fetchInto(ctx, repository, commit, ref, base, id)
}

func (m *Module) fetchInto(ctx context.Context, repository, commit, ref, base, id string) (any, error) {
	tree := filepath.Join(m.Work, id, "tree")
	if err := os.MkdirAll(tree, 0o700); err != nil {
		return nil, err
	}
	remote := m.remote(repository)
	if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.FetchOptions...); err != nil {
		return nil, err
	}
	res := FetchResult{Workspace: id, Commit: commit}
	if base != "" {
		if err := m.merge(ctx, &res, tree, remote, base, commit); err != nil {
			return nil, err
		}
	}
	return res, nil
}

// remote is the address a repository is fetched from: the forge over ssh as the account, or RemoteBase when a test set it.
func (m *Module) remote(repository string) string {
	base := m.RemoteBase
	if base == "" {
		base = "ssh://" + ForgeAlias + "/"
	}
	return base + repository + ".git"
}

// merge makes the merge preview of the head onto base (the SDK's buildstep.MergeInto, shared by every build module)
// and writes the answer into res.
func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base, commit string) error {
	mr, err := buildstep.MergeInto(ctx, m.Run, tree, remote, base, commit, m.FetchOptions...)
	if err != nil {
		return err
	}
	res.Base = mr.Base
	res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed
	if mr.Merge != "" {
		res.Programs = &mr.Programs
	}
	if res.Merge == "" {
		return nil
	}
	res.Trees, err = buildstep.ModuleTrees(ctx, m.Run, nil, tree, res.Merge)
	return err
}

// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
// net for a caller that never released one (issue 1839, ADR 0509).
func (m *Module) CleanWorkspaces(now time.Time) {
	m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
}

// clock is the time now: Now when a test set it, else the wall clock.
func (m *Module) clock() time.Time {
	if m.Now != nil {
		return m.Now()
	}
	return time.Now()
}

func firstLine(s string) string {
	for _, l := range strings.Split(s, "\n") {
		if l = strings.TrimSpace(l); l != "" {
			return l
		}
	}
	return ""
}