Take a module in from the builder's done key of its trunk fingerprint: one record of the key's artifacts, its manifest judged against the catalogue trunk's predefined keys (issue 1851, A3, A4b, A4c)
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Option changes FetchCommit.
|
||||
type Option func(*fetchOptions)
|
||||
|
||||
type fetchOptions struct{ allowFile bool }
|
||||
|
||||
// AllowFileRemote lets FetchCommit take a file:// remote. It is for a test's local repository; production
|
||||
// code does not pass it, and FetchCommit then refuses anything but ssh://.
|
||||
func AllowFileRemote() Option { return func(o *fetchOptions) { o.allowFile = true } }
|
||||
|
||||
var (
|
||||
commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
refOK = regexp.MustCompile(`^refs/[A-Za-z0-9._/-]+$`)
|
||||
)
|
||||
|
||||
// FetchCommit fetches one commit of a forge repository into dir, an empty directory, and checks it out detached
|
||||
// (novox/hq ADR 0466 §4, §5; issue 1474). remote is ssh://<ssh name>/<owner>/<repo>.git, the ssh name being the
|
||||
// one the build module's templates set up (ADR 0409 §1); commit is a full 40-hex sha; ref is optional, and names
|
||||
// the pull request's head (refs/pull/<n>/head) when the commit is only there. Every git command runs through run
|
||||
// as `git -C dir ...` with GIT_TERMINAL_PROMPT=0 and no credential helper, so nothing is asked for and no
|
||||
// credential is used. A remote that is not ssh://, or one with user information, and a commit that is not 40
|
||||
// hex digits are refused before anything runs.
|
||||
func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, ref string, opts ...Option) error {
|
||||
var o fetchOptions
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
if err := checkFetch(o, dir, remote, commit, ref); err != nil {
|
||||
return err
|
||||
}
|
||||
env := []string{"GIT_TERMINAL_PROMPT=0"}
|
||||
git := func(args ...string) (string, error) {
|
||||
full := append([]string{"-c", "credential.helper=", "-C", dir}, args...)
|
||||
r := run(ctx, env, "git", full...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
return r.Stdout, nil
|
||||
}
|
||||
if _, err := git("init", "-q"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := git("remote", "add", "origin", remote); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := git("fetch", "--no-tags", "--depth", "1", "origin", commit); err != nil {
|
||||
if ref == "" {
|
||||
return err
|
||||
}
|
||||
if _, rerr := git("fetch", "--no-tags", "origin", ref); rerr != nil {
|
||||
return fmt.Errorf("%v; then the ref %s: %w", err, ref, rerr)
|
||||
}
|
||||
if _, verr := git("cat-file", "-e", commit+"^{commit}"); verr != nil {
|
||||
return fmt.Errorf("the commit %s is not in %s: %w", commit, ref, verr)
|
||||
}
|
||||
}
|
||||
if _, err := git("checkout", "-q", "--detach", commit); err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if head := strings.TrimSpace(out); head != commit {
|
||||
return fmt.Errorf("the checkout is at %s, not at the commit %s", head, commit)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkFetch(o fetchOptions, dir, remote, commit, ref string) error {
|
||||
if dir == "" {
|
||||
return fmt.Errorf("buildstep: no workspace directory")
|
||||
}
|
||||
if !commitOK.MatchString(commit) {
|
||||
return fmt.Errorf("buildstep: %q is not a full commit (40 hex digits)", commit)
|
||||
}
|
||||
if ref != "" && !refOK.MatchString(ref) {
|
||||
return fmt.Errorf("buildstep: %q is not a ref (refs/...)", ref)
|
||||
}
|
||||
u, err := url.Parse(remote)
|
||||
if err != nil {
|
||||
return fmt.Errorf("buildstep: the remote is not a URL: %w", err)
|
||||
}
|
||||
if u.User != nil {
|
||||
return fmt.Errorf("buildstep: the remote carries user information; the key is the credential")
|
||||
}
|
||||
switch {
|
||||
case u.Scheme == "ssh" && u.Host != "":
|
||||
case u.Scheme == "file" && o.allowFile:
|
||||
default:
|
||||
return fmt.Errorf("buildstep: the remote %q is not ssh://", remote)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Guard is a build module's check that the toolchain installed on its node is the one its recipe builds with
|
||||
// (novox/hq ADR 0469 §1; issue 1473). The same guard serves ADR 0470 §4's postgres major for the integration
|
||||
// level, with its own Probe and Want.
|
||||
type Guard struct {
|
||||
// Language is the toolchain's key part: go or typescript (delivery.ToolchainKey).
|
||||
Language string
|
||||
// Want is the toolchain version the recipe builds with, and Recipe the recipe's own version. The indexer
|
||||
// compares both texts as they are written here.
|
||||
Want, Recipe string
|
||||
// Probe reads the installed version (go version, node --version).
|
||||
Probe func(ctx context.Context) (installed string, err error)
|
||||
Node string
|
||||
// Put is the toolchains state's Put (stdio.State(delivery.ToolchainsState).Put).
|
||||
Put Put
|
||||
}
|
||||
|
||||
// Check probes the installed version, writes it to <node>.<language> of the toolchains state every time it runs,
|
||||
// and answers nil when it is the recipe's. Otherwise it answers a *Refusal naming both versions; the installed
|
||||
// version is written even then, so the indexer sees that builders disagree. A probe that fails is a *Refusal
|
||||
// naming its failure, and nothing is written. A write that fails is an error that is not a Refusal.
|
||||
func (g Guard) Check(ctx context.Context) error {
|
||||
installed, err := g.Probe(ctx)
|
||||
if err != nil {
|
||||
return &Refusal{Text: fmt.Sprintf("the installed %s version could not be read: %v", g.Language, err), Want: g.Want}
|
||||
}
|
||||
key, err := delivery.ToolchainKey(g.Node, g.Language)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := delivery.EncodeToolchain(delivery.Toolchain{Language: g.Language, Version: installed, Recipe: g.Recipe, At: now().UTC()})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := g.Put(key, json.RawMessage(raw)); err != nil {
|
||||
return fmt.Errorf("buildstep: writing the toolchain %s: %w", key, err)
|
||||
}
|
||||
if installed != g.Want {
|
||||
return &Refusal{Text: fmt.Sprintf("%s %s installed, the recipe builds with %s %s", g.Language, installed, g.Language, g.Want),
|
||||
Want: g.Want, Have: installed}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package buildstep
|
||||
|
||||
import "strings"
|
||||
|
||||
// LauncherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is started as
|
||||
// index.serve.mjs (novox/hq ADR 0193), an ES module by its own extension whatever the bundle's package.json says.
|
||||
const LauncherSuffix = ".serve.mjs"
|
||||
|
||||
// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root: for TypeScript
|
||||
// every entrypoint `x.js` is started by `x.serve.mjs` beside it; an entrypoint that is not JavaScript and every other
|
||||
// language, whose build is already executable, get none. The one rule mesh-build-node, build-agent and the controller
|
||||
// share (R4e), so the name a build writes is the name a reader expects.
|
||||
func Launchers(language string, entrypoints []string) map[string]string {
|
||||
out := map[string]string{}
|
||||
if language != "typescript" {
|
||||
return out
|
||||
}
|
||||
for _, entry := range entrypoints {
|
||||
if strings.HasSuffix(entry, ".js") {
|
||||
out[entry] = strings.TrimSuffix(entry, ".js") + LauncherSuffix
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// MergeResult is what MergeInto made of a head and a trunk commit: the trunk commit (Base) and the head, whole, and the
|
||||
// merge commit and the paths the merge changes against the trunk commit; or, when the head does not merge cleanly, the
|
||||
// files in conflict, with Merge and Changed empty.
|
||||
type MergeResult struct {
|
||||
Base, Head, Merge string
|
||||
Conflicts []string
|
||||
Changed []string
|
||||
}
|
||||
|
||||
// MergeInto makes, in the workspace dir that FetchCommit left at commit, the merge of commit into the trunk commit
|
||||
// base, as the forge's default merge would (a merge commit, never a fast-forward), and leaves the workspace at the merge
|
||||
// (novox/hq ADR 0484 clause 3, ADR 0323). It is the one merge the build modules' fetch steps make, with the identity
|
||||
// and flags of the controller's check merge (mesh-controller internal/builder/check_merge.go): a name nobody owns, no
|
||||
// signature, no hook, nothing of the machine's git configuration that changes a merge. Nothing is pushed.
|
||||
//
|
||||
// A merge needs both histories, so it makes the fetch whole (the shallow clone FetchCommit made is deepened) and
|
||||
// fetches base by its sha. A head that does not merge cleanly is an answer, not an error: the files in conflict, the
|
||||
// merge undone and the workspace left at base. A head already in base answers a Merge equal to Base and no changed
|
||||
// paths. An error is that the merge could not be tried: a base or commit that is not a full sha, no base, a workspace
|
||||
// not at commit, or a fetch or git command that failed. remote and the options are those of FetchCommit; the runner's
|
||||
// bound must allow a whole fetch.
|
||||
func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, commit string, opts ...Option) (MergeResult, error) {
|
||||
var o fetchOptions
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
res := MergeResult{Base: base, Head: commit}
|
||||
if base == "" {
|
||||
return res, fmt.Errorf("buildstep: no base: a merge needs the trunk commit it is made onto")
|
||||
}
|
||||
if err := checkFetch(o, dir, remote, base, ""); err != nil {
|
||||
return res, err
|
||||
}
|
||||
if err := checkFetch(o, dir, remote, commit, ""); err != nil {
|
||||
return res, err
|
||||
}
|
||||
env := []string{"GIT_TERMINAL_PROMPT=0"}
|
||||
git := func(args ...string) (string, error) {
|
||||
full := append([]string{"-c", "credential.helper=", "-C", dir}, args...)
|
||||
r := run(ctx, env, "git", full...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
return r.Stdout, nil
|
||||
}
|
||||
at, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
if head := strings.TrimSpace(at); head != commit {
|
||||
return res, fmt.Errorf("buildstep: the workspace is at %s, not at the commit %s", head, commit)
|
||||
}
|
||||
shallow, err := git("rev-parse", "--is-shallow-repository")
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
fetch := []string{"fetch", "--no-tags", "origin", base}
|
||||
if strings.TrimSpace(shallow) == "true" {
|
||||
fetch = []string{"fetch", "--no-tags", "--unshallow", "origin", base}
|
||||
}
|
||||
if _, err := git(fetch...); err != nil {
|
||||
return res, fmt.Errorf("the trunk commit %s: %w", base, err)
|
||||
}
|
||||
if _, err := git("checkout", "-q", "--detach", base); err != nil {
|
||||
return res, fmt.Errorf("the trunk commit %s cannot be checked out: %w", base, err)
|
||||
}
|
||||
message := fmt.Sprintf("Check the merge of %s into %s (made for a check, never pushed)", short(commit), short(base))
|
||||
args := append(append([]string{}, mergeIdentity...), "merge", "--no-ff", "--no-edit", "--no-verify", "--quiet", "-m", message, commit)
|
||||
if _, err := git(args...); err != nil {
|
||||
out, _ := git("diff", "--name-only", "--diff-filter=U")
|
||||
res.Conflicts = strings.Fields(out)
|
||||
_, _ = git("merge", "--abort")
|
||||
if len(res.Conflicts) == 0 {
|
||||
return res, fmt.Errorf("the head %s could not be merged into the trunk commit %s: %w", short(commit), short(base), err)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
merge, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("the merge cannot be read: %w", err)
|
||||
}
|
||||
res.Merge = strings.TrimSpace(merge)
|
||||
changed, err := git("diff", "--name-only", "--no-renames", base, res.Merge)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("the paths the merge changes cannot be read: %w", err)
|
||||
}
|
||||
res.Changed = strings.Fields(changed)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// mergeIdentity is who a check's merge is made as: it is never pushed, so it names nobody's account. No hook of the
|
||||
// clone's or the build machine's git configuration runs, and no setting there changes the merge (the controller's
|
||||
// check merge, internal/builder/check_merge.go).
|
||||
var mergeIdentity = []string{"-c", "user.name=mesh merge check", "-c", "user.email=merge-check@mesh.invalid",
|
||||
"-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "-c", "merge.verifySignatures=false"}
|
||||
|
||||
func short(commit string) string {
|
||||
if len(commit) > 12 {
|
||||
return commit[:12]
|
||||
}
|
||||
return commit
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Pack tars and gzips a directory the way the build agent does, byte for byte, and answers the archive and its
|
||||
// digest, "sha256:" and the hex of the archive's bytes (issue 1479; novox/hq ADR 0405 §7, ADR 0453 §7). It is a
|
||||
// copy of mesh-controller's internal/builder/builder.go pack (origin/main 073ccc29, :1351-1425), which stays
|
||||
// where it is until the build agent goes: a build module's digest is compared with the build agent's for one
|
||||
// fingerprint, so the bytes must be equal.
|
||||
//
|
||||
// Deterministically: regular files only (a symlink or a device is refused), sorted relative paths with
|
||||
// slashes, each header carrying its name, permission bits and size and nothing else, the gzip default writer.
|
||||
func Pack(root string) (archive []byte, digest string, err error) {
|
||||
raw, err := pack(root)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return raw, "sha256:" + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func pack(root string) ([]byte, error) {
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("%s is not a directory", root)
|
||||
}
|
||||
|
||||
var paths []string
|
||||
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() {
|
||||
// Only files. A symlink or a device in an archive is refused by the host that unpacks
|
||||
// it, so putting one in would build something that cannot be applied.
|
||||
if !info.IsDir() && !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("%s is neither a file nor a directory, and an archive carries "+
|
||||
"only those", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
paths = append(paths, path)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// filepath.Walk walks in lexical order; the sort is kept as the build agent keeps it, since an archive
|
||||
// whose digest changes because the traversal did is a silent failure.
|
||||
sort.Strings(paths)
|
||||
|
||||
var out bytes.Buffer
|
||||
zipped := gzip.NewWriter(&out)
|
||||
writer := tar.NewWriter(zipped)
|
||||
for _, path := range paths {
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
relative, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mode := int64(info.Mode().Perm())
|
||||
if err := writer.WriteHeader(&tar.Header{
|
||||
Name: filepath.ToSlash(relative), Mode: mode, Size: int64(len(body)),
|
||||
Typeflag: tar.TypeReg,
|
||||
// Everything else left at its zero value on purpose: no times, owners or names.
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := writer.Write(body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := zipped.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package buildstep
|
||||
|
||||
// Refusal is a step that is not run on purpose, as against one that failed: the toolchain installed is not the
|
||||
// recipe's, the integration level's postgres major is not the one it is built for (novox/hq ADR 0469 §1, ADR 0470
|
||||
// §4; issue 1473). A step's run returns one and the runner ends the step refused with its text.
|
||||
type Refusal struct {
|
||||
// Text is the reason, in words the operator reads.
|
||||
Text string
|
||||
// Want is the version the recipe asks for and Have the one found; either may be empty.
|
||||
Want, Have string
|
||||
}
|
||||
|
||||
func (r *Refusal) Error() string { return r.Text }
|
||||
@@ -0,0 +1,99 @@
|
||||
// Package buildstep is what both build modules share: the runner of a long step that answers a tool call at once
|
||||
// and ends on its key in the module's `steps` state (novox/hq ADR 0469 §2; issue 1471), the toolchain guard
|
||||
// (issue 1473) and the fetch of one commit over SSH (issue 1474).
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Put writes one key of a state: stdio.State(delivery.StepsState).Put's shape (go/state.go).
|
||||
type Put func(key string, value any) (uint64, error)
|
||||
|
||||
// EndAttempts and EndRetry bound how a step's end is written: a write that fails is tried again, since the end
|
||||
// is the only way the step's reader learns it is over.
|
||||
var (
|
||||
EndAttempts = 5
|
||||
EndRetry = 2 * time.Second
|
||||
)
|
||||
|
||||
var now = time.Now
|
||||
|
||||
// NewID is a step id: the time in milliseconds in base 36 and six random hex digits, lower-case letters and
|
||||
// digits only, so it is one key token and differs between two steps of one node even in one millisecond.
|
||||
func NewID() string {
|
||||
var b [3]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return strconv.FormatInt(now().UnixMilli(), 36) + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// Start begins a step of tool on node and answers its id at once: the key <node>.<id> holds state running when
|
||||
// Start returns. run is then called in a goroutine of its own, under ctx (the process's, not the tool call's: the
|
||||
// call is cut at 30 s, the step is not). When it returns, the key is written passed with run's result, failed with
|
||||
// the error's text, or refused when the error is a *Refusal; a panic in run ends the step failed, naming it, and
|
||||
// the process goes on. If the running write fails, nothing was started and Start answers the error.
|
||||
func Start(ctx context.Context, put Put, node, tool string, run func(ctx context.Context) (result any, err error)) (string, error) {
|
||||
step := delivery.Step{ID: NewID(), Tool: tool, Node: node, State: delivery.StepRunning, Started: now().UTC()}
|
||||
key, err := delivery.StepKey(node, step.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := write(put, key, step); err != nil {
|
||||
return "", fmt.Errorf("buildstep: writing the running step %s: %w", key, err)
|
||||
}
|
||||
go func() {
|
||||
result, err := protect(ctx, run)
|
||||
step.Ended = now().UTC()
|
||||
switch ref := (*Refusal)(nil); {
|
||||
case err == nil:
|
||||
step.State = delivery.StepPassed
|
||||
if result != nil {
|
||||
raw, merr := json.Marshal(result)
|
||||
if merr != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, "the result is not JSON: "+merr.Error()
|
||||
} else {
|
||||
step.Result = raw
|
||||
}
|
||||
}
|
||||
case errors.As(err, &ref):
|
||||
step.State, step.Reason = delivery.StepRefused, err.Error()
|
||||
default:
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
}
|
||||
for attempt := 1; ; attempt++ {
|
||||
if werr := write(put, key, step); werr == nil || attempt >= EndAttempts {
|
||||
return
|
||||
}
|
||||
time.Sleep(EndRetry)
|
||||
}
|
||||
}()
|
||||
return step.ID, nil
|
||||
}
|
||||
|
||||
// protect is run with a panic turned into the error that ends the step.
|
||||
func protect(ctx context.Context, run func(ctx context.Context) (any, error)) (result any, err error) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
result, err = nil, fmt.Errorf("the step panicked: %v", p)
|
||||
}
|
||||
}()
|
||||
return run(ctx)
|
||||
}
|
||||
|
||||
func write(put Put, key string, step delivery.Step) error {
|
||||
raw, err := delivery.EncodeStep(step)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = put(key, json.RawMessage(raw))
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// ModuleTree is what a merge holds of one module directory: its tree id, the sha256 of its module.json and the sha256
|
||||
// of each lock file it has (go.sum, package-lock.json), from which the checker and the indexer take the module's inputs
|
||||
// (novox/hq ADR 0484 clause 3).
|
||||
type ModuleTree struct {
|
||||
Tree string `json:"tree"`
|
||||
Manifest string `json:"manifest"`
|
||||
Locks map[string]string `json:"locks,omitempty"`
|
||||
}
|
||||
|
||||
var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||
|
||||
// ModuleTrees are the trees, in commit, of every directory of the workspace dir that holds a module.json, by directory
|
||||
// (the repository's root is "."). The workspace is at commit, so the manifests and the lock files are read from it. env is
|
||||
// the environment git runs with (nil for none). Both build modules' fetch steps answer it for a merge.
|
||||
func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, commit string) (map[string]ModuleTree, error) {
|
||||
r := run(ctx, env, "git", "-C", dir, "ls-files", "--", "module.json", "*/module.json")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
trees := map[string]ModuleTree{}
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if l = strings.TrimSpace(l); l == "" {
|
||||
continue
|
||||
}
|
||||
mdir := filepath.ToSlash(filepath.Dir(l))
|
||||
spec := commit + ":" + mdir
|
||||
if mdir == "." {
|
||||
spec = commit + "^{tree}"
|
||||
}
|
||||
t := run(ctx, env, "git", "-C", dir, "rev-parse", spec)
|
||||
if t.Status != 0 || t.Err != "" {
|
||||
return nil, fmt.Errorf("git rev-parse %s: %s", spec, firstLine(t.Err+" "+t.Stderr))
|
||||
}
|
||||
mt := ModuleTree{Tree: strings.TrimSpace(t.Stdout)}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(l)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mt.Manifest = sha256Hex(raw)
|
||||
for _, lock := range lockFiles {
|
||||
if raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(mdir), lock)); err == nil {
|
||||
if mt.Locks == nil {
|
||||
mt.Locks = map[string]string{}
|
||||
}
|
||||
mt.Locks[lock] = sha256Hex(raw)
|
||||
}
|
||||
}
|
||||
trees[mdir] = mt
|
||||
}
|
||||
return trees, nil
|
||||
}
|
||||
|
||||
func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
|
||||
@@ -0,0 +1,127 @@
|
||||
// Package forgekey is the key keeper every module that reaches the forge over SSH shares: ssh-client for a
|
||||
// machine's accounts, a build module for its own (design 47 R4c-3, ADR 0466 §2; issue 1469).
|
||||
package forgekey
|
||||
|
||||
// The key this machine's account uses to reach the forge over SSH (novox/hq ADR 0399, ADR 0400; issue 955,
|
||||
// area S1). It is made here, on the machine, once: ssh-keygen writes ~/.ssh/mesh_ed25519 only when it is
|
||||
// missing, so the private half never leaves the machine and an existing key is never replaced. The public
|
||||
// half and its fingerprint go into this module's `keys` state, one entry per account under <node>.<account>,
|
||||
// for the forge's provider to register. The tools bundle is the one process of this module, so the keeper
|
||||
// runs inside it beside the tools (main.go), the way claude-code reports what it holds.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// KeyFile is the key's name under ~/.ssh: never one of the operator's own id_* files.
|
||||
const KeyFile = "mesh_ed25519"
|
||||
|
||||
// The kinds of key (ADR 0400 §1, ADR 0466 §2): a machine's account's own, and a build module's.
|
||||
const (
|
||||
KindMachine = "machine"
|
||||
KindBuilder = "builder"
|
||||
)
|
||||
|
||||
// KeyRetry is how soon it looks again after a failure.
|
||||
const KeyRetry = 30 * time.Second
|
||||
|
||||
// KeyRound is how often the keeper looks again.
|
||||
const KeyRound = 10 * time.Minute
|
||||
|
||||
// KeyEntry is one value of the keys state. It carries the public half only.
|
||||
type KeyEntry struct {
|
||||
Node string `json:"node"`
|
||||
Account string `json:"account"`
|
||||
Kind string `json:"kind"`
|
||||
Public string `json:"public"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
At string `json:"at"`
|
||||
}
|
||||
|
||||
// KeyPutter is the keys state, as far as the keeper needs it.
|
||||
type KeyPutter interface {
|
||||
Put(key string, value any) (uint64, error)
|
||||
}
|
||||
|
||||
// KeyKeeper keeps one account's key and its entry.
|
||||
type KeyKeeper struct {
|
||||
Home, Node, Account string
|
||||
Kind string // KindMachine or KindBuilder
|
||||
Run Runner
|
||||
State KeyPutter
|
||||
Now func() time.Time
|
||||
// After waits between rounds; time.After when nil. A test sets it to see which pause was chosen.
|
||||
After func(time.Duration) <-chan time.Time
|
||||
|
||||
reported string // the public half and fingerprint last put
|
||||
}
|
||||
|
||||
func (k *KeyKeeper) path() string { return filepath.Join(k.Home, ".ssh", KeyFile) }
|
||||
|
||||
// Once makes the key when it is missing and puts the entry when it differs from the one last put.
|
||||
func (k *KeyKeeper) Once(ctx context.Context) error {
|
||||
if err := k.ensure(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := os.ReadFile(k.path() + ".pub")
|
||||
if err != nil {
|
||||
return fmt.Errorf("the public half: %w", err)
|
||||
}
|
||||
pk, err := ParseKeyLine(string(raw))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s.pub: %w", k.path(), err)
|
||||
}
|
||||
public := strings.TrimSpace(string(raw))
|
||||
if sig := public + "|" + pk.Fingerprint; sig == k.reported {
|
||||
return nil
|
||||
}
|
||||
entry := KeyEntry{Node: k.Node, Account: k.Account, Kind: k.Kind, Public: public,
|
||||
Fingerprint: pk.Fingerprint, At: k.Now().UTC().Format(time.RFC3339)}
|
||||
if _, err := k.State.Put(k.Node+"."+k.Account, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
k.reported = public + "|" + pk.Fingerprint
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensure makes the pair when the private half is missing. Never over an existing key.
|
||||
func (k *KeyKeeper) ensure(ctx context.Context) error {
|
||||
if _, err := os.Lstat(k.path()); err == nil {
|
||||
return nil
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(k.path()), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
r := k.Run(ctx, nil, "ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", k.Account+"@"+k.Node+" (mesh)", "-f", k.path())
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("ssh-keygen: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Keep runs Once now and every KeyRound, reporting a failure through say and trying again at the next round.
|
||||
func (k *KeyKeeper) Keep(ctx context.Context, say func(string, ...any)) {
|
||||
for {
|
||||
pause := KeyRound
|
||||
if err := k.Once(ctx); err != nil {
|
||||
say("the forge key: %v", err)
|
||||
pause = KeyRetry // the state or the bus's grant may arrive after the process
|
||||
}
|
||||
after := k.After
|
||||
if after == nil {
|
||||
after = time.After
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-after(pause):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package forgekey
|
||||
|
||||
// Keys, by their public half only. A fingerprint is computed here from the public key's bytes, as
|
||||
// ssh-keygen -l does (SHA256 of the key blob, unpadded base64); a private key is never read by this
|
||||
// process — its first line, the PEM header, says it is one, and ssh-keygen is asked about it.
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// KeyTypes are the public key algorithms an authorized_keys or known_hosts line can carry.
|
||||
var KeyTypes = map[string]bool{
|
||||
"ssh-ed25519": true, "ssh-rsa": true, "ssh-dss": true,
|
||||
"ecdsa-sha2-nistp256": true, "ecdsa-sha2-nistp384": true, "ecdsa-sha2-nistp521": true,
|
||||
"sk-ssh-ed25519@openssh.com": true, "sk-ecdsa-sha2-nistp256@openssh.com": true,
|
||||
}
|
||||
|
||||
// PublicKey is one public key as a line carries it.
|
||||
type PublicKey struct {
|
||||
Type string `json:"type"`
|
||||
Bits int `json:"bits"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Comment string `json:"comment"`
|
||||
Options string `json:"options,omitempty"`
|
||||
Weak string `json:"weak,omitempty"`
|
||||
}
|
||||
|
||||
// ParseKeyLine reads one authorized_keys line (options, type, key, comment) or a public key file's.
|
||||
func ParseKeyLine(line string) (PublicKey, error) {
|
||||
fields := fieldsQuoted(strings.TrimSpace(line))
|
||||
for i, f := range fields {
|
||||
if !KeyTypes[f] || i+1 >= len(fields) {
|
||||
continue
|
||||
}
|
||||
k := PublicKey{Type: f, Options: strings.Join(fields[:i], " "), Comment: strings.Join(fields[i+2:], " ")}
|
||||
blob, err := base64.StdEncoding.DecodeString(fields[i+1])
|
||||
if err != nil {
|
||||
return k, fmt.Errorf("the key after %s is not base64", f)
|
||||
}
|
||||
sum := sha256.Sum256(blob)
|
||||
k.Fingerprint = "SHA256:" + base64.RawStdEncoding.EncodeToString(sum[:])
|
||||
k.Bits = bitsOf(f, blob)
|
||||
switch {
|
||||
case f == "ssh-dss":
|
||||
k.Weak = "DSA: refused by current OpenSSH"
|
||||
case f == "ssh-rsa" && k.Bits > 0 && k.Bits < 3072:
|
||||
k.Weak = fmt.Sprintf("RSA of %d bits: below 3072", k.Bits)
|
||||
}
|
||||
return k, nil
|
||||
}
|
||||
return PublicKey{}, fmt.Errorf("no public key on this line")
|
||||
}
|
||||
|
||||
// fieldsQuoted splits on spaces outside double quotes, as sshd reads an options field.
|
||||
func fieldsQuoted(s string) []string {
|
||||
var out []string
|
||||
var cur strings.Builder
|
||||
quoted := false
|
||||
for _, ch := range s {
|
||||
switch {
|
||||
case ch == '"':
|
||||
quoted = !quoted
|
||||
cur.WriteRune(ch)
|
||||
case (ch == ' ' || ch == '\t') && !quoted:
|
||||
if cur.Len() > 0 {
|
||||
out = append(out, cur.String())
|
||||
cur.Reset()
|
||||
}
|
||||
default:
|
||||
cur.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
if cur.Len() > 0 {
|
||||
out = append(out, cur.String())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// bitsOf reads a key's size from its blob: the RSA modulus, the curve, or ed25519's fixed 256.
|
||||
func bitsOf(kind string, blob []byte) int {
|
||||
strs := [][]byte{}
|
||||
for len(blob) >= 4 {
|
||||
n := binary.BigEndian.Uint32(blob)
|
||||
if int(n) > len(blob)-4 {
|
||||
break
|
||||
}
|
||||
strs = append(strs, blob[4:4+n])
|
||||
blob = blob[4+n:]
|
||||
}
|
||||
switch {
|
||||
case strings.Contains(kind, "ed25519"):
|
||||
return 256
|
||||
case kind == "ssh-rsa" && len(strs) >= 3:
|
||||
return new(big.Int).SetBytes(strs[2]).BitLen()
|
||||
case kind == "ssh-dss" && len(strs) >= 2:
|
||||
return new(big.Int).SetBytes(strs[1]).BitLen()
|
||||
case strings.Contains(kind, "nistp256"):
|
||||
return 256
|
||||
case strings.Contains(kind, "nistp384"):
|
||||
return 384
|
||||
case strings.Contains(kind, "nistp521"):
|
||||
return 521
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package forgekey
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Ran is what a command did: its output, its exit status, and why it never ran to an answer.
|
||||
type Ran struct {
|
||||
Stdout string
|
||||
Stderr string
|
||||
Status int
|
||||
// Err is "ENOENT" when the program is not installed, or says it was ended for taking too long.
|
||||
Err string
|
||||
}
|
||||
|
||||
// Runner runs one command with extra environment words, so every tool can be tested without ssh.
|
||||
type Runner func(ctx context.Context, env []string, name string, args ...string) Ran
|
||||
|
||||
// CallTimeout bounds one command of ExecRunner: below the runtime's thirty-second call limit. A variable so a
|
||||
// test can lower it.
|
||||
var CallTimeout = 20 * time.Second
|
||||
|
||||
// ExecRunner runs a command on this machine, with no terminal and nothing on its stdin, bounded by
|
||||
// CallTimeout.
|
||||
func ExecRunner(ctx context.Context, env []string, name string, args ...string) Ran {
|
||||
return ExecRunnerFor(CallTimeout)(ctx, env, name, args...)
|
||||
}
|
||||
|
||||
// ExecRunnerFor is ExecRunner with its own bound (issue 1497): a build module's long step passes 0, so only the
|
||||
// step's own context ends a command, and a fetch may outlast a tool call's thirty seconds.
|
||||
func ExecRunnerFor(bound time.Duration) Runner {
|
||||
return func(ctx context.Context, env []string, name string, args ...string) Ran {
|
||||
return execRun(ctx, bound, false, env, name, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// ExecRunnerClean is ExecRunnerFor whose commands get exactly the env argument as their environment, and nothing of
|
||||
// this process's (issue 1497; novox/hq ADR 0466 §4: no registry password in a fetch's environment). The caller
|
||||
// gives PATH and HOME itself. A directory is the command's own business (`git -C`, `go -C`).
|
||||
func ExecRunnerClean(bound time.Duration) Runner {
|
||||
return func(ctx context.Context, env []string, name string, args ...string) Ran {
|
||||
return execRun(ctx, bound, true, env, name, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func execRun(ctx context.Context, bound time.Duration, clean bool, env []string, name string, args ...string) Ran {
|
||||
if bound > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, bound)
|
||||
defer cancel()
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
if clean {
|
||||
cmd.Env = append([]string{}, env...) // non-nil, so an empty list is an empty environment
|
||||
} else {
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
}
|
||||
var out, errb bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &out, &errb
|
||||
err := cmd.Run()
|
||||
r := Ran{Stdout: out.String(), Stderr: errb.String()}
|
||||
var exitErr *exec.ExitError
|
||||
switch {
|
||||
case errors.Is(ctx.Err(), context.DeadlineExceeded):
|
||||
r.Status, r.Err = 124, "ended by its step's deadline"
|
||||
if bound > 0 {
|
||||
r.Err = fmt.Sprintf("no answer within %d s", int(bound/time.Second))
|
||||
}
|
||||
case errors.Is(err, exec.ErrNotFound):
|
||||
r.Status, r.Err = 127, "ENOENT"
|
||||
case errors.As(err, &exitErr):
|
||||
r.Status = exitErr.ExitCode()
|
||||
case err != nil:
|
||||
r.Status, r.Err = 1, err.Error()
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Vendored
+2
@@ -2,11 +2,13 @@
|
||||
## explicit; go 1.26.0
|
||||
git.novox.be/novox/mesh-sdk/go
|
||||
git.novox.be/novox/mesh-sdk/go/asks
|
||||
git.novox.be/novox/mesh-sdk/go/buildstep
|
||||
git.novox.be/novox/mesh-sdk/go/catalogue
|
||||
git.novox.be/novox/mesh-sdk/go/compose
|
||||
git.novox.be/novox/mesh-sdk/go/declaration
|
||||
git.novox.be/novox/mesh-sdk/go/delivery
|
||||
git.novox.be/novox/mesh-sdk/go/facts
|
||||
git.novox.be/novox/mesh-sdk/go/forgekey
|
||||
git.novox.be/novox/mesh-sdk/go/membership
|
||||
git.novox.be/novox/mesh-sdk/go/overlay
|
||||
git.novox.be/novox/mesh-sdk/go/process
|
||||
|
||||
Reference in New Issue
Block a user