From 927852fda8f3198e51ad532c1db86a21231504cf Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 20:05:21 +0000 Subject: [PATCH] Lay out a build module's long steps, guard its toolchain and fetch one commit over SSH (issues 1471, 1473, 1474), part 4 of 7 --- go/buildstep/fetch.go | 114 ++++++++++++++++++++++++++++++++++++++++ go/buildstep/guard.go | 52 ++++++++++++++++++ go/buildstep/refusal.go | 13 +++++ go/buildstep/step.go | 99 ++++++++++++++++++++++++++++++++++ 4 files changed, 278 insertions(+) create mode 100644 go/buildstep/fetch.go create mode 100644 go/buildstep/guard.go create mode 100644 go/buildstep/refusal.go create mode 100644 go/buildstep/step.go diff --git a/go/buildstep/fetch.go b/go/buildstep/fetch.go new file mode 100644 index 0000000..df6a1b0 --- /dev/null +++ b/go/buildstep/fetch.go @@ -0,0 +1,114 @@ +package buildstep + +import ( + "context" + "fmt" + "net/url" + "regexp" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Option changes FetchCommit. +type Option func(*fetchOptions) + +type fetchOptions struct{ allowFile bool } + +// AllowFileRemote lets FetchCommit take a file:// remote. It is for a test's local repository; production +// code does not pass it, and FetchCommit then refuses anything but ssh://. +func AllowFileRemote() Option { return func(o *fetchOptions) { o.allowFile = true } } + +var ( + commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`) + refOK = regexp.MustCompile(`^refs/[A-Za-z0-9._/-]+$`) +) + +// FetchCommit fetches one commit of a forge repository into dir, an empty directory, and checks it out detached +// (novox/hq ADR 0466 §4, §5; issue 1474). remote is ssh:////.git, the ssh name being the +// one the build module's templates set up (ADR 0409 §1); commit is a full 40-hex sha; ref is optional, and names +// the pull request's head (refs/pull//head) when the commit is only there. Every git command runs through run +// as `git -C dir ...` with GIT_TERMINAL_PROMPT=0 and no credential helper, so nothing is asked for and no +// credential is used. A remote that is not ssh://, or one with user information, and a commit that is not 40 +// hex digits are refused before anything runs. +func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, ref string, opts ...Option) error { + var o fetchOptions + for _, opt := range opts { + opt(&o) + } + if err := checkFetch(o, dir, remote, commit, ref); err != nil { + return err + } + env := []string{"GIT_TERMINAL_PROMPT=0"} + git := func(args ...string) (string, error) { + full := append([]string{"-c", "credential.helper=", "-C", dir}, args...) + r := run(ctx, env, "git", full...) + if r.Status != 0 || r.Err != "" { + return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr)) + } + return r.Stdout, nil + } + if _, err := git("init", "-q"); err != nil { + return err + } + if _, err := git("remote", "add", "origin", remote); err != nil { + return err + } + if _, err := git("fetch", "--no-tags", "--depth", "1", "origin", commit); err != nil { + if ref == "" { + return err + } + if _, rerr := git("fetch", "--no-tags", "origin", ref); rerr != nil { + return fmt.Errorf("%v; then the ref %s: %w", err, ref, rerr) + } + if _, verr := git("cat-file", "-e", commit+"^{commit}"); verr != nil { + return fmt.Errorf("the commit %s is not in %s: %w", commit, ref, verr) + } + } + if _, err := git("checkout", "-q", "--detach", commit); err != nil { + return err + } + out, err := git("rev-parse", "HEAD") + if err != nil { + return err + } + if head := strings.TrimSpace(out); head != commit { + return fmt.Errorf("the checkout is at %s, not at the commit %s", head, commit) + } + return nil +} + +func checkFetch(o fetchOptions, dir, remote, commit, ref string) error { + if dir == "" { + return fmt.Errorf("buildstep: no workspace directory") + } + if !commitOK.MatchString(commit) { + return fmt.Errorf("buildstep: %q is not a full commit (40 hex digits)", commit) + } + if ref != "" && !refOK.MatchString(ref) { + return fmt.Errorf("buildstep: %q is not a ref (refs/...)", ref) + } + u, err := url.Parse(remote) + if err != nil { + return fmt.Errorf("buildstep: the remote is not a URL: %w", err) + } + if u.User != nil { + return fmt.Errorf("buildstep: the remote carries user information; the key is the credential") + } + switch { + case u.Scheme == "ssh" && u.Host != "": + case u.Scheme == "file" && o.allowFile: + default: + return fmt.Errorf("buildstep: the remote %q is not ssh://", remote) + } + return nil +} + +func firstLine(s string) string { + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" { + return l + } + } + return "" +} diff --git a/go/buildstep/guard.go b/go/buildstep/guard.go new file mode 100644 index 0000000..a9dc49c --- /dev/null +++ b/go/buildstep/guard.go @@ -0,0 +1,52 @@ +package buildstep + +import ( + "context" + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Guard is a build module's check that the toolchain installed on its node is the one its recipe builds with +// (novox/hq ADR 0469 §1; issue 1473). The same guard serves ADR 0470 §4's postgres major for the integration +// level, with its own Probe and Want. +type Guard struct { + // Language is the toolchain's key part: go or typescript (delivery.ToolchainKey). + Language string + // Want is the toolchain version the recipe builds with, and Recipe the recipe's own version. The indexer + // compares both texts as they are written here. + Want, Recipe string + // Probe reads the installed version (go version, node --version). + Probe func(ctx context.Context) (installed string, err error) + Node string + // Put is the toolchains state's Put (stdio.State(delivery.ToolchainsState).Put). + Put Put +} + +// Check probes the installed version, writes it to . of the toolchains state every time it runs, +// and answers nil when it is the recipe's. Otherwise it answers a *Refusal naming both versions; the installed +// version is written even then, so the indexer sees that builders disagree. A probe that fails is a *Refusal +// naming its failure, and nothing is written. A write that fails is an error that is not a Refusal. +func (g Guard) Check(ctx context.Context) error { + installed, err := g.Probe(ctx) + if err != nil { + return &Refusal{Text: fmt.Sprintf("the installed %s version could not be read: %v", g.Language, err), Want: g.Want} + } + key, err := delivery.ToolchainKey(g.Node, g.Language) + if err != nil { + return err + } + raw, err := delivery.EncodeToolchain(delivery.Toolchain{Language: g.Language, Version: installed, Recipe: g.Recipe, At: now().UTC()}) + if err != nil { + return err + } + if _, err := g.Put(key, json.RawMessage(raw)); err != nil { + return fmt.Errorf("buildstep: writing the toolchain %s: %w", key, err) + } + if installed != g.Want { + return &Refusal{Text: fmt.Sprintf("%s %s installed, the recipe builds with %s %s", g.Language, installed, g.Language, g.Want), + Want: g.Want, Have: installed} + } + return nil +} diff --git a/go/buildstep/refusal.go b/go/buildstep/refusal.go new file mode 100644 index 0000000..0f88529 --- /dev/null +++ b/go/buildstep/refusal.go @@ -0,0 +1,13 @@ +package buildstep + +// Refusal is a step that is not run on purpose, as against one that failed: the toolchain installed is not the +// recipe's, the integration level's postgres major is not the one it is built for (novox/hq ADR 0469 §1, ADR 0470 +// §4; issue 1473). A step's run returns one and the runner ends the step refused with its text. +type Refusal struct { + // Text is the reason, in words the operator reads. + Text string + // Want is the version the recipe asks for and Have the one found; either may be empty. + Want, Have string +} + +func (r *Refusal) Error() string { return r.Text } diff --git a/go/buildstep/step.go b/go/buildstep/step.go new file mode 100644 index 0000000..c5ae0d9 --- /dev/null +++ b/go/buildstep/step.go @@ -0,0 +1,99 @@ +// Package buildstep is what both build modules share: the runner of a long step that answers a tool call at once +// and ends on its key in the module's `steps` state (novox/hq ADR 0469 §2; issue 1471), the toolchain guard +// (issue 1473) and the fetch of one commit over SSH (issue 1474). +package buildstep + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Put writes one key of a state: stdio.State(delivery.StepsState).Put's shape (go/state.go). +type Put func(key string, value any) (uint64, error) + +// EndAttempts and EndRetry bound how a step's end is written: a write that fails is tried again, since the end +// is the only way the step's reader learns it is over. +var ( + EndAttempts = 5 + EndRetry = 2 * time.Second +) + +var now = time.Now + +// NewID is a step id: the time in milliseconds in base 36 and six random hex digits, lower-case letters and +// digits only, so it is one key token and differs between two steps of one node even in one millisecond. +func NewID() string { + var b [3]byte + _, _ = rand.Read(b[:]) + return strconv.FormatInt(now().UnixMilli(), 36) + hex.EncodeToString(b[:]) +} + +// Start begins a step of tool on node and answers its id at once: the key . holds state running when +// Start returns. run is then called in a goroutine of its own, under ctx (the process's, not the tool call's: the +// call is cut at 30 s, the step is not). When it returns, the key is written passed with run's result, failed with +// the error's text, or refused when the error is a *Refusal; a panic in run ends the step failed, naming it, and +// the process goes on. If the running write fails, nothing was started and Start answers the error. +func Start(ctx context.Context, put Put, node, tool string, run func(ctx context.Context) (result any, err error)) (string, error) { + step := delivery.Step{ID: NewID(), Tool: tool, Node: node, State: delivery.StepRunning, Started: now().UTC()} + key, err := delivery.StepKey(node, step.ID) + if err != nil { + return "", err + } + if err := write(put, key, step); err != nil { + return "", fmt.Errorf("buildstep: writing the running step %s: %w", key, err) + } + go func() { + result, err := protect(ctx, run) + step.Ended = now().UTC() + switch ref := (*Refusal)(nil); { + case err == nil: + step.State = delivery.StepPassed + if result != nil { + raw, merr := json.Marshal(result) + if merr != nil { + step.State, step.Reason = delivery.StepFailed, "the result is not JSON: "+merr.Error() + } else { + step.Result = raw + } + } + case errors.As(err, &ref): + step.State, step.Reason = delivery.StepRefused, err.Error() + default: + step.State, step.Reason = delivery.StepFailed, err.Error() + } + for attempt := 1; ; attempt++ { + if werr := write(put, key, step); werr == nil || attempt >= EndAttempts { + return + } + time.Sleep(EndRetry) + } + }() + return step.ID, nil +} + +// protect is run with a panic turned into the error that ends the step. +func protect(ctx context.Context, run func(ctx context.Context) (any, error)) (result any, err error) { + defer func() { + if p := recover(); p != nil { + result, err = nil, fmt.Errorf("the step panicked: %v", p) + } + }() + return run(ctx) +} + +func write(put Put, key string, step delivery.Step) error { + raw, err := delivery.EncodeStep(step) + if err != nil { + return err + } + _, err = put(key, json.RawMessage(raw)) + return err +}