package buildstep import ( "context" "encoding/json" "fmt" "path" "sort" "strings" "git.novox.be/novox/mesh-sdk/go/delivery" ) // **One ask, one whole job** (novox/hq ADR 0525 §3). A build module runs a pulled ask with its own step tools' bodies, // the same ones its request and reply tools run (ADR 0525 §8), in the order the job names: a build ask fetches, then // builds or copies each artifact; a check ask fetches, digests the level, takes the kept verdicts and tests the rest, // the packages whose tests the change touches first. The answers are the tools' own, whole, so the asker judges them as // it judged the tools' answers. // StepFunc is one step tool's body: its arguments in, its answer out. type StepFunc func(ctx context.Context, args map[string]any) (any, error) // AskSteps are the step tools a build module runs an ask with. Copy is the image module's alone; Digest the Go module's. type AskSteps struct { Fetch, Build, Copy, Digest, Test StepFunc // Toolchain is the toolchain the module builds with now; a build ask naming another is refused. Nil checks nothing // (an image or an archive names none). Toolchain func(ctx context.Context) (delivery.ToolchainUsed, error) // Kept looks up a kept verdict by level and digest (the checker's kept bucket, delivery.KeptKey): whether one is // kept and how many tests it counted. Nil takes none. Kept func(ctx context.Context, level, digest string) (tests int, found bool, err error) // Release removes a workspace once the ask's steps on it ended (the module's release tool's body); nil leaves it to // the module's sweep. Release func(workspace string) error Logf func(format string, args ...any) } // RunAsk runs ask's job with steps and answers a delivery.BuildAnswer or a delivery.CheckAnswer. func RunAsk(ctx context.Context, steps AskSteps, ask delivery.StepAsk) (any, error) { if ask.Build != nil { return steps.build(ctx, *ask.Build) } if ask.Check != nil { return steps.check(ctx, *ask.Check) } return nil, Refuse("the ask %s holds no job", ask.ID) } func raw(v any) (json.RawMessage, error) { if r, ok := v.(json.RawMessage); ok { return r, nil } return json.Marshal(v) } // fetched is what an ask reads of a fetch's answer. type fetched struct { Workspace string `json:"workspace"` Merge string `json:"merge"` Conflicts []string `json:"conflicts"` Changed []string `json:"changed"` Modules []string `json:"modules"` } func (s AskSteps) fetch(ctx context.Context, args map[string]any) (json.RawMessage, fetched, error) { if s.Fetch == nil { return nil, fetched{}, Refuse("this build module fetches nothing") } out, err := s.Fetch(ctx, args) if err != nil { return nil, fetched{}, err } r, err := raw(out) if err != nil { return nil, fetched{}, err } var f fetched if err := json.Unmarshal(r, &f); err != nil || f.Workspace == "" { return r, f, fmt.Errorf("the fetch answered no workspace: %s", r) } return r, f, nil } func (s AskSteps) build(ctx context.Context, job delivery.BuildJob) (any, error) { answer := delivery.BuildAnswer{} if job.Toolchain != nil && s.Toolchain != nil { tc, err := s.Toolchain(ctx) if err != nil { return nil, err } if tc.Version != job.Toolchain.Version || tc.Recipe != job.Toolchain.Recipe { return nil, Refuse("the ask is for %s (recipe %s) and this node builds with %s (recipe %s)", job.Toolchain.Version, job.Toolchain.Recipe, tc.Version, tc.Recipe) } answer.Toolchain = &tc } var f fetched defer func() { s.release(f.Workspace) }() for _, a := range job.Artifacts { if a.From == "" && f.Workspace == "" { r, got, err := s.fetch(ctx, job.Fetch) if err != nil { return nil, err } if _, candidate := job.Fetch["base"]; candidate { // A candidate is built from its head merged onto the trunk; a head that does not merge leaves the tree at the // trunk, and building that would store the trunk's build under the candidate's fingerprint. if len(got.Conflicts) > 0 { return nil, fmt.Errorf("the head does not merge into the trunk: files in conflict: %s", strings.Join(got.Conflicts, ", ")) } if got.Merge == "" { return nil, fmt.Errorf("the fetch with a base answered no merge: %s", r) } } f = got } var ( out any err error ) if a.From != "" { if s.Copy == nil { return nil, Refuse("this build module copies no artifact (%s)", a.Artifact) } out, err = s.Copy(ctx, map[string]any{"module": job.Module, "artifact": a.Artifact, "from": a.From}) } else { if s.Build == nil { return nil, Refuse("this build module builds no artifact (%s)", a.Artifact) } args := map[string]any{"workspace": f.Workspace, "module": job.Path, "artifact": a.Artifact} if job.Publish { args["publish"] = true } out, err = s.Build(ctx, args) } if err != nil { return nil, fmt.Errorf("%s: %w", a.Artifact, err) } r, err := raw(out) if err != nil { return nil, err } var built struct { Digest string `json:"digest"` Reference string `json:"reference"` } if err := json.Unmarshal(r, &built); err != nil || built.Digest == "" { return nil, fmt.Errorf("the build of %s answered no digest: %s", a.Artifact, r) } answer.Artifacts = append(answer.Artifacts, delivery.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference}) } return answer, nil } func (s AskSteps) check(ctx context.Context, job delivery.CheckJob) (any, error) { r, f, err := s.fetch(ctx, job.Fetch) if err != nil { return nil, err } defer s.release(f.Workspace) answer := delivery.CheckAnswer{Fetch: r} if len(f.Conflicts) > 0 { return answer, nil // the asker reads the conflict from the fetch's answer: a verdict, not an error } runner := "" if job.Runner != "" && (job.RunnerWhen == "" || changesUnder(f.Changed, job.RunnerWhen)) { runner = job.Runner } withRunner := func(args map[string]any) map[string]any { if runner != "" { args["runner"] = runner } return args } if s.Test == nil { return nil, Refuse("this build module tests nothing") } test := func(args map[string]any) (bool, error) { out, err := s.Test(ctx, args) if err != nil { return false, err } r, err := raw(out) if err != nil { return false, err } answer.Tests = append(answer.Tests, r) var t struct { Passed bool `json:"passed"` } _ = json.Unmarshal(r, &t) return t.Passed, nil } if s.Digest == nil { // A module with no digest (Node) tests each module directory of the tree, those whose tests changed first. for _, priority := range []bool{true, false} { for _, dir := range f.Modules { if changesTestIn(f.Changed, dir) != priority { continue } passed, err := test(withRunner(map[string]any{"workspace": f.Workspace, "module": dir, "level": job.Level})) if err != nil { return nil, fmt.Errorf("the %s level of %s: %w", job.Level, dir, err) } if !passed { return answer, nil } } } return answer, nil } todo, all, err := s.plan(ctx, job, f, withRunner, &answer) if err != nil { return nil, err } for _, priority := range []bool{true, false} { args := withRunner(map[string]any{"workspace": f.Workspace, "level": job.Level}) if job.Run != "" { args["run"] = job.Run } if all { if priority { continue // nothing to put first: the level runs whole after } } else { var run []string for _, p := range todo { if changesTestOf(f.Changed, p) == priority { run = append(run, p) } } if len(run) == 0 { continue } args["packages"] = run } passed, err := test(args) if err != nil { return nil, fmt.Errorf("the %s level: %w", job.Level, err) } if !passed { return answer, nil } } return answer, nil } // release removes an ask's workspace, saying a failure: the module's sweep removes it later. func (s AskSteps) release(workspace string) { if workspace == "" || s.Release == nil { return } if err := s.Release(workspace); err != nil && s.Logf != nil { s.Logf("the workspace %s is left to the sweep: %v", workspace, err) } } // plan digests the level and takes the kept verdicts: the packages left to test, or all when the level runs whole (a // try of some tests, or a digest that named nothing). func (s AskSteps) plan(ctx context.Context, job delivery.CheckJob, f fetched, withRunner func(map[string]any) map[string]any, answer *delivery.CheckAnswer) ([]string, bool, error) { if job.Run != "" { return nil, true, nil // a partial run proves no package: nothing is taken or kept } out, err := s.Digest(ctx, withRunner(map[string]any{"workspace": f.Workspace, "level": job.Level})) if err != nil { if s.Logf != nil { s.Logf("no digest of the %s level, so it runs whole: %v", job.Level, err) } return nil, true, nil } r, err := raw(out) if err != nil { return nil, true, nil } answer.Digest = r var d struct { Packages map[string]string `json:"packages"` } if err := json.Unmarshal(r, &d); err != nil || len(d.Packages) == 0 { return nil, true, nil } pkgs := make([]string, 0, len(d.Packages)) for p := range d.Packages { pkgs = append(pkgs, p) } sort.Strings(pkgs) var todo []string for _, p := range pkgs { if job.Kept && s.Kept != nil { tests, found, err := s.Kept(ctx, job.Level, d.Packages[p]) if err != nil && s.Logf != nil { s.Logf("the kept verdict of %s at %s cannot be read, so it runs: %v", p, job.Level, err) } if err == nil && found { answer.Kept = append(answer.Kept, delivery.KeptTaken{Package: p, Digest: d.Packages[p], Tests: tests}) continue } } todo = append(todo, p) } if len(todo) == 0 { return nil, false, nil } return todo, false, nil } // changesUnder says whether a path the merge changes is under prefix. func changesUnder(changed []string, prefix string) bool { for _, p := range changed { if strings.HasPrefix(p, prefix) { return true } } return false } // isTestFile says whether a repository-relative path is a test file: Go's _test.go, and a TypeScript module's .test. // and .spec. files and test directories. func isTestFile(p string) bool { switch { case strings.HasSuffix(p, "_test.go"): return true case strings.Contains(p, ".test.") || strings.Contains(p, ".spec."): return true } for _, seg := range strings.Split(path.Dir(p), "/") { if seg == "__tests__" || seg == "test" || seg == "tests" { return true } } return false } // changesTestOf says whether the merge changes a test file of the Go package with the import path pkg: the file's // directory and the import path agree on their last segments. func changesTestOf(changed []string, pkg string) bool { for _, f := range changed { if !strings.HasSuffix(f, "_test.go") { continue } dir := path.Dir(f) if dir == pkg || strings.HasSuffix(pkg, "/"+dir) || strings.HasSuffix(dir, "/"+pkg) { return true } } return false } // changesTestIn says whether the merge changes a test file inside the directory of a Node module. func changesTestIn(changed []string, dir string) bool { for _, f := range changed { if !isTestFile(f) || strings.HasSuffix(f, "_test.go") { continue } if dir == "." || dir == "" || strings.HasPrefix(f, strings.TrimSuffix(dir, "/")+"/") { return true } } return false }