package buildstep import ( "encoding/json" "fmt" "path" "slices" "sort" "strings" "time" meshkeys "git.novox.be/novox/mesh-sdk/go/keys" ) // **The build ask, once.** The three build seats (node-build-go, node-build-image, node-build-node) accept the same ask, // defined once as the part `build-step` of the Delivery contracts. The generator writes one Go type per seat, so the // seat packages hold three structurally equal StepAsks; this file is the one neutral form the shared runner and the // askers use, and AskOf is the one conversion from any seat's (a seat's value and this form are the same JSON, held // equal by the part's fixtures: the contracts' tests decode each in both). This package imports no seat's contract // (novox/hq ADR 0634 §6). // The kinds of ask: the seat's accepted kinds, in the order a holder pulls them. const ( AskDelivery = "delivery" AskCheck = "check" AskRecheck = "recheck" ) // AskKinds are the accepted kinds of a build seat, in the order a holder pulls them (ADR 0536 §4). var AskKinds = []string{AskDelivery, AskCheck, AskRecheck} // The languages of the build seats: each is one build module's (mesh-build-) and one seat's // (node-build-). const ( AskGo = "go" AskNode = "node" AskImage = "image" ) // AskLanguages are the build seats' languages. var AskLanguages = []string{AskGo, AskNode, AskImage} // NeedNodeBuildNode is the need of a check whose tests run through the Node build seat's holder on the same node // (issue 890): a node that holds no Node build module declines the ask. const NeedNodeBuildNode = "node-build-node" // AskNeeds are the needs an ask may name. var AskNeeds = []string{NeedNodeBuildNode} // CheckNeed refuses a need an ask may not name. func CheckNeed(need string) error { if !slices.Contains(AskNeeds, need) { return fmt.Errorf("buildstep: %q is not a need an ask may name (%s)", need, strings.Join(AskNeeds, ", ")) } return nil } // Lacks are the needs has does not say its node has; a nil has lacks all. func Lacks(needs []string, has func(need string) bool) []string { var out []string for _, n := range needs { if has == nil || !has(n) { out = append(out, n) } } return out } // AskRuns is how many times a holder runs one ask before it ends refused; AskMaxDeliver is the redelivery bound that // leaves one more for the terminal step (ADR 0525 §5). const ( AskRuns = 3 AskMaxDeliver = AskRuns + 2 ) // StepAskVersion is the layout version of an ask. const StepAskVersion = 1 // The levels a check ask names. const ( LevelUnit = "unit" LevelIntegration = "integration" ) // The registry seats a build publishes through (ADR 0605 §1). var RegistrySeats = []string{"npm-package-registry", "go-package-registry", "container-image-registry", "oci-artifact-registry"} // NodeNamesArg and NodeNamesEnv carry the node names a check's tests run against. const ( NodeNamesArg = "node-names" NodeNamesEnv = "MESH_NODE_NAMES" ) func checkAskKind(kind string) error { if !slices.Contains(AskKinds, kind) { return fmt.Errorf("buildstep: the ask kind %q is not delivery, check or recheck", kind) } return nil } func checkAskLanguage(language string) error { if !slices.Contains(AskLanguages, language) { return fmt.Errorf("buildstep: the ask language %q is not go, node or image", language) } return nil } // BuildSeat is the seat of a language's build module. func BuildSeat(language string) (string, error) { if err := checkAskLanguage(language); err != nil { return "", err } return "node-build-" + language, nil } // BuildModuleOf is the module of a language's build seat. func BuildModuleOf(language string) (string, error) { if err := checkAskLanguage(language); err != nil { return "", err } return "mesh-build-" + language, nil } // AskSubject is the subject a build seat accepts a kind of ask on: the seat's accept subject. func AskSubject(language, kind string) (string, error) { seat, err := BuildSeat(language) if err != nil { return "", err } if err := checkAskKind(kind); err != nil { return "", err } return "mesh.seat." + seat + ".accept." + kind, nil } // ParseAskSubject is the inverse of AskSubject. func ParseAskSubject(subject string) (language, kind string, err error) { rest, ok := strings.CutPrefix(subject, "mesh.seat.node-build-") if !ok { return "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject", subject) } language, kind, ok = strings.Cut(rest, ".accept.") if !ok { return "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject", subject) } if want, err := AskSubject(language, kind); err != nil || want != subject { return "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject: %v", subject, err) } return language, kind, nil } // AskStream is the work-queue stream of a language's build seat. func AskStream(language string) (string, error) { seat, err := BuildSeat(language) if err != nil { return "", err } return "SEAT_" + strings.ToUpper(strings.ReplaceAll(seat, "-", "_")), nil } // AskWorker is the shared pull worker of a kind on a language's stream. func AskWorker(language, kind string) (string, error) { stream, err := AskStream(language) if err != nil { return "", err } if err := checkAskKind(kind); err != nil { return "", err } return stream + "_" + strings.ToUpper(kind) + "_worker", nil } // AskStepKey is the key of the step of the ask askID on node. func AskStepKey(node, askID string) (string, error) { if err := CheckAskID(askID); err != nil { return "", err } return StepKey(node, askID) } // AskStepWatch is the key pattern of an ask's step on any node. func AskStepWatch(askID string) (string, error) { if err := CheckAskID(askID); err != nil { return "", err } return "*." + askID, nil } // StepAsk is one ask of a build module: the part's `StepAsk`. type StepAsk struct { V int `json:"v"` ID string `json:"id"` Kind string `json:"kind"` Language string `json:"language"` Asker string `json:"asker"` At time.Time `json:"at,omitzero"` Build *BuildJob `json:"build,omitempty"` Check *CheckJob `json:"check,omitempty"` Compose *ComposeJob `json:"compose,omitempty"` // Needs are what a check's tests need of the holder's node (CheckNeed); a holder lacking one declines the ask. Needs []string `json:"needs,omitempty"` // Packages pin the mesh packages a build or a check builds against. Packages PackageVersions `json:"packages,omitempty"` } // Fetch is what a build module's fetch tool is asked, inside an ask: the part's `Fetch`. type Fetch struct { V int `json:"v,omitempty"` Repository string `json:"repository"` Commit string `json:"commit"` Ref string `json:"ref,omitempty"` Base string `json:"base,omitempty"` Dir string `json:"dir,omitempty"` Paths []string `json:"paths,omitempty"` Packages PackageVersions `json:"packages,omitempty"` // Package is the TypeScript module's group package: the group's tagged version ({name, version}) or the group's package // member ({repository, commit, ref}). Package *FetchPackage `json:"package,omitempty"` Beside []FetchBeside `json:"beside,omitempty"` } // FetchPackage is the group package a TypeScript fetch takes (ADR 0607 §3, ADR 0500 §5): either the group's tagged version // (Name and Version) or the group's package member (Repository, Commit and Ref). type FetchPackage struct { Name string `json:"name,omitempty"` Version string `json:"version,omitempty"` Repository string `json:"repository,omitempty"` Commit string `json:"commit,omitempty"` Ref string `json:"ref,omitempty"` } // FetchBeside is a sibling repository placed beside the fetched tree. type FetchBeside struct { Repository string `json:"repository"` Commit string `json:"commit"` } // ToolchainUsed is the toolchain a build ran with. type ToolchainUsed struct { Language string `json:"language"` Recipe string `json:"recipe"` Version string `json:"version"` } // AskArtifact is one artifact of a build ask: built from the fetched tree, or copied from an upstream pin (From). type AskArtifact struct { Artifact string `json:"artifact"` From string `json:"from,omitempty"` Seat string `json:"seat,omitempty"` Version string `json:"version,omitempty"` } // BuildJob is a trunk fingerprint's build. type BuildJob struct { Module string `json:"module"` Path string `json:"path,omitempty"` Fp string `json:"fp,omitempty"` Fetch *Fetch `json:"fetch,omitempty"` Artifacts []AskArtifact `json:"artifacts"` Publish bool `json:"publish,omitempty"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` } // CheckJob is the check of one or both levels of a fetched tree. type CheckJob struct { Fetch Fetch `json:"fetch"` Levels []string `json:"levels"` Kept bool `json:"kept,omitempty"` Runner string `json:"runner,omitempty"` RunnerWhen string `json:"runner_when,omitempty"` Run string `json:"run,omitempty"` Packages []string `json:"packages,omitempty"` NodeNames []string `json:"node-names,omitempty"` // Toolchain is the toolchain the checker agreed for the language (ADR 0584 clause 2): a holder whose node reports // another version declines the check. Toolchain *ToolchainUsed `json:"toolchain,omitempty"` } // ComposeJob asks one compose-candidate build. type ComposeJob struct { Fetch Fetch `json:"fetch"` Dir string `json:"dir"` Package string `json:"package"` Request json.RawMessage `json:"request"` Reader string `json:"reader,omitempty"` } // AskOf is any seat's ask (or this form) as the neutral StepAsk: the one conversion, since the three seats' types // are the part's one layout. A value that does not fit the layout is refused. func AskOf[T any](seatAsk T) (StepAsk, error) { raw, err := json.Marshal(seatAsk) if err != nil { return StepAsk{}, fmt.Errorf("buildstep: an ask does not encode: %w", err) } return DecodeStepAsk(raw) } // AskAs is the neutral ask as a seat's type T (the inverse of AskOf). func AskAs[T any](a StepAsk) (T, error) { var out T raw, err := json.Marshal(a) if err != nil { return out, err } err = json.Unmarshal(raw, &out) return out, err } func (a StepAsk) check() error { if err := CheckAskID(a.ID); err != nil { return err } if err := checkAskKind(a.Kind); err != nil { return err } if err := checkAskLanguage(a.Language); err != nil { return err } if err := meshkeys.CheckName("asker", a.Asker); err != nil { return err } if a.At.IsZero() { return fmt.Errorf("buildstep: the ask %s names no time", a.ID) } jobs := 0 for _, set := range []bool{a.Build != nil, a.Check != nil, a.Compose != nil} { if set { jobs++ } } switch { case jobs != 1: return fmt.Errorf("buildstep: the ask %s holds a build, a check or a compose, exactly one", a.ID) case a.Compose != nil && (a.Kind != AskCheck || a.Language != AskGo): return fmt.Errorf("buildstep: the ask %s is a compose, which the Go seat's check stream alone carries", a.ID) case a.Build != nil && a.Kind == AskRecheck: return fmt.Errorf("buildstep: the ask %s is a build on the recheck stream, which carries checks alone", a.ID) case a.Check != nil && a.Kind == AskDelivery: return fmt.Errorf("buildstep: the ask %s is a check on the delivery stream, which carries trunk builds alone", a.ID) case a.Check != nil && a.Language == AskImage: return fmt.Errorf("buildstep: the ask %s is a check of an image, which has no tests", a.ID) case len(a.Needs) > 0 && a.Check == nil: return fmt.Errorf("buildstep: the ask %s names needs and is no check, whose tests alone need a node", a.ID) case len(a.Packages) > 0 && a.Compose != nil: return fmt.Errorf("buildstep: the ask %s is a compose and names package versions, which a compose does not take", a.ID) } if err := a.Packages.Check(); err != nil { return fmt.Errorf("the ask %s: %w", a.ID, err) } for _, n := range a.Needs { if err := CheckNeed(n); err != nil { return fmt.Errorf("the ask %s: %w", a.ID, err) } } switch { case a.Build != nil: return a.Build.check(a.ID) case a.Compose != nil: return a.Compose.check(a.ID) } return a.Check.check(a.ID) } func (c ComposeJob) check(id string) error { if err := c.Fetch.check(id); err != nil { return err } if c.Dir == "" || c.Dir != path.Clean(c.Dir) || strings.HasPrefix(c.Dir, "/") || c.Dir == ".." || strings.HasPrefix(c.Dir, "../") { return fmt.Errorf("buildstep: the compose ask %s names no clean directory inside the repository (%q)", id, c.Dir) } if c.Package == "" { return fmt.Errorf("buildstep: the compose ask %s names no package to build", id) } if !json.Valid(c.Request) || len(c.Request) == 0 || c.Request[0] != '{' { return fmt.Errorf("buildstep: the compose ask %s holds no request object", id) } return nil } func (b BuildJob) check(id string) error { if err := meshkeys.CheckName("module", b.Module); err != nil { return err } if len(b.Artifacts) == 0 { return fmt.Errorf("buildstep: the build ask %s names no artifact", id) } fetch := false for _, a := range b.Artifacts { if err := meshkeys.CheckName("artifact", a.Artifact); err != nil { return err } if a.Seat != "" && !slices.Contains(RegistrySeats, a.Seat) { return fmt.Errorf("buildstep: the build ask %s publishes %s through %q, which is no registry seat (%s)", id, a.Artifact, a.Seat, strings.Join(RegistrySeats, ", ")) } if a.Version != "" && a.Seat == "" { return fmt.Errorf("buildstep: the build ask %s gives %s the version %s and names no seat to publish it through", id, a.Artifact, a.Version) } fetch = fetch || a.From == "" } if fetch { if b.Path == "" { return fmt.Errorf("buildstep: the build ask %s names no directory (path) for the module %s", id, b.Module) } if b.Fetch == nil { return fmt.Errorf("buildstep: the ask %s fetches no repository /", id) } return b.Fetch.check(id) } return nil } func (c CheckJob) check(id string) error { if len(c.Levels) == 0 { return fmt.Errorf("buildstep: the check ask %s names no level", id) } for _, l := range c.Levels { if l != LevelUnit && l != LevelIntegration { return fmt.Errorf("buildstep: the level %q is not %s or %s", l, LevelUnit, LevelIntegration) } } for _, p := range c.Packages { if p == "" || strings.ContainsAny(p, " \t\n") { return fmt.Errorf("buildstep: the check ask %s names a package %q that is no import path", id, p) } } if t := c.Toolchain; t != nil && (t.Language == "" || t.Version == "" || t.Recipe == "") { return fmt.Errorf("buildstep: the check ask %s names a toolchain without its language, version and recipe", id) } seen := map[string]bool{} for _, n := range c.NodeNames { if err := meshkeys.CheckName("node", n); err != nil || strings.Contains(n, ",") || seen[n] { return fmt.Errorf("buildstep: the check ask %s names the node %q, which is no node name or named twice", id, n) } seen[n] = true } return c.Fetch.check(id) } func (f Fetch) check(id string) error { if f.Repository == "" || !strings.Contains(f.Repository, "/") { return fmt.Errorf("buildstep: the ask %s fetches no repository /", id) } if !askCommitOK.MatchString(f.Commit) { return fmt.Errorf("buildstep: the ask %s fetches no full commit", id) } return nil } // EncodeStepAsk is a as the bytes the asker puts on a build seat's stream: stamped with the version and UTC, checked. func EncodeStepAsk(a StepAsk) ([]byte, error) { a.V = StepAskVersion a.At = a.At.UTC() if err := a.check(); err != nil { return nil, err } return json.Marshal(a) } // DecodeStepAsk reads an ask, refusing another version, an unknown shape or a malformed ask. func DecodeStepAsk(data []byte) (StepAsk, error) { var probe struct { V *int `json:"v"` } if err := json.Unmarshal(data, &probe); err != nil { return StepAsk{}, fmt.Errorf("buildstep: an ask is not JSON: %w", err) } if probe.V == nil || *probe.V != StepAskVersion { return StepAsk{}, fmt.Errorf("%w: an ask of version %v, this reader reads %d", ErrUnknownVersion, probe.V, StepAskVersion) } var a StepAsk if err := json.Unmarshal(data, &a); err != nil { return StepAsk{}, fmt.Errorf("buildstep: an ask does not fit its shape: %w", err) } if err := a.check(); err != nil { return StepAsk{}, err } return a, nil } // PackageNeeds is what a package of a check needs of a node. type PackageNeeds struct { Package string `json:"package"` Needs []string `json:"needs"` } // StepDeclined is the result of a step a holder declined: its node lacks what the ask needs. type StepDeclined struct { Declined bool `json:"declined"` Needs []string `json:"needs"` Lacks []string `json:"lacks"` } // DeclinedOf reads a refused step's result as a decline. func DeclinedOf(step Step) (StepDeclined, bool) { if step.State != StepRefused || len(step.Result) == 0 { return StepDeclined{}, false } var d StepDeclined if err := json.Unmarshal(step.Result, &d); err != nil || !d.Declined { return StepDeclined{}, false } return d, true } // UnmetNeeds are the needs the levels' unmet packages name, sorted, once each. func UnmetNeeds(levels []LevelAnswer) []string { seen := map[string]bool{} var out []string for _, l := range levels { for _, u := range l.Unmet { for _, n := range u.Needs { if !seen[n] { seen[n] = true out = append(out, n) } } } } sort.Strings(out) return out } // BuildAnswer is what a build ask's step answers. type BuildAnswer struct { Artifacts []AskBuilt `json:"artifacts"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` Imports []string `json:"imports,omitempty"` OutOfStep []string `json:"out-of-step,omitempty"` } // AskBuilt is one artifact a build made. type AskBuilt struct { Artifact string `json:"artifact"` Digest string `json:"digest"` Reference string `json:"reference,omitempty"` } // CheckAnswer is what a check ask's step answers. type CheckAnswer struct { Fetch FetchAnswer `json:"fetch"` Levels []LevelAnswer `json:"levels,omitempty"` } // FetchAnswer is what a build module's fetch tool answers, whole: the part's `FetchAnswer`. A check keeps it as the // answer's fetch, so the checker reads a conflict (a head that does not merge is an answer, not an error), the changed // paths, the programs and the trees from it. type FetchAnswer struct { Workspace string `json:"workspace"` Commit string `json:"commit"` Modules []string `json:"modules"` // The merge preview, only when the call gave a base. Base string `json:"base,omitempty"` Merge string `json:"merge,omitempty"` Conflicts []string `json:"conflicts,omitempty"` Changed []string `json:"changed,omitempty"` Programs []Program `json:"programs,omitempty"` Trees map[string]ModuleTree `json:"trees,omitempty"` // The Go module's. Beside []PlacedBeside `json:"beside,omitempty"` Packages []TakenVersion `json:"packages,omitempty"` // The TypeScript module's. Installed []string `json:"installed,omitempty"` Refused []FetchRefusal `json:"refused,omitempty"` Group string `json:"group,omitempty"` } // Program is a changed file that is a compiled program (issue 1577). type Program struct { Path string `json:"path"` Kind string `json:"kind"` // ELF, Mach-O or PE Commit string `json:"commit,omitempty"` } // PlacedBeside is a sibling repository placed as a plain directory under the workspace's beside directory. type PlacedBeside struct { Repository string `json:"repository"` Commit string `json:"commit"` Dir string `json:"dir"` } // TakenVersion is, of a directory whose go.mod requires an asked mesh package, the version it is built against. type TakenVersion struct { Dir string `json:"dir"` Package string `json:"package"` Asked string `json:"asked"` Named string `json:"named"` Built string `json:"built"` Sum string `json:"sum,omitempty"` Said string `json:"said,omitempty"` } // FetchRefusal is a TypeScript module whose packages the fetch did not install, with why. type FetchRefusal struct { Module string `json:"module"` Reason string `json:"reason"` } // StepResult is what any step answered: one layout for the four kinds, as the seats' `StepResult` (the fields of the // kinds do not meet). A step's result bytes read into it strictly. type StepResult struct { Artifacts []AskBuilt `json:"artifacts,omitempty"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` Imports []string `json:"imports,omitempty"` OutOfStep []string `json:"out-of-step,omitempty"` Fetch *FetchAnswer `json:"fetch,omitempty"` Levels []LevelAnswer `json:"levels,omitempty"` Built bool `json:"built,omitempty"` Key string `json:"key,omitempty"` Verdict json.RawMessage `json:"verdict,omitempty"` Declined bool `json:"declined,omitempty"` Needs []string `json:"needs,omitempty"` Lacks []string `json:"lacks,omitempty"` } // ComposeAnswer is what a compose ask's step answers. type ComposeAnswer struct { Built bool `json:"built"` Key string `json:"key,omitempty"` Verdict json.RawMessage `json:"verdict,omitempty"` } // LevelAnswer is one level of a check. type LevelAnswer struct { Level string `json:"level"` Judged string `json:"judged,omitempty"` Digest *LevelDigest `json:"digest,omitempty"` Kept []KeptTaken `json:"kept,omitempty"` Tests []json.RawMessage `json:"tests,omitempty"` Unmet []PackageNeeds `json:"unmet,omitempty"` } // LevelDigest is the digest of a level's tests by package. type LevelDigest struct { Level string `json:"level,omitempty"` Packages []PackageDigest `json:"packages,omitempty"` Beside []string `json:"beside,omitempty"` WithoutTests []string `json:"without_tests,omitempty"` JudgedBy string `json:"judged_by,omitempty"` Needs []PackageNeeds `json:"needs,omitempty"` } // PackageDigest is the digest of one package's tests. type PackageDigest struct { Package string `json:"package"` Digest string `json:"digest"` } // KeptTaken is a package whose kept verdict a level took instead of running its tests. type KeptTaken struct { Package string `json:"package"` Digest string `json:"digest"` Tests int `json:"tests"` } // UnmarshalJSON reads the packages as the list or as the older map by package. func (d *LevelDigest) UnmarshalJSON(raw []byte) error { var v struct { Level string `json:"level"` Packages json.RawMessage `json:"packages"` Beside []string `json:"beside"` WithoutTests []string `json:"without_tests"` JudgedBy string `json:"judged_by"` Needs []PackageNeeds `json:"needs"` } if err := json.Unmarshal(raw, &v); err != nil { return err } var pkgs []PackageDigest if p := strings.TrimSpace(string(v.Packages)); p != "" && p != "null" { if strings.HasPrefix(p, "{") { var m map[string]string if err := json.Unmarshal(v.Packages, &m); err != nil { return fmt.Errorf("a digest's packages: %w", err) } for pkg, digest := range m { pkgs = append(pkgs, PackageDigest{Package: pkg, Digest: digest}) } } else if err := json.Unmarshal(v.Packages, &pkgs); err != nil { return fmt.Errorf("a digest's packages: %w", err) } } sort.Slice(pkgs, func(i, j int) bool { return pkgs[i].Package < pkgs[j].Package }) *d = LevelDigest{Level: v.Level, Packages: pkgs, Beside: v.Beside, WithoutTests: v.WithoutTests, JudgedBy: v.JudgedBy, Needs: v.Needs} return nil } // NeedsOf are the needs of a package in the digest. func (d *LevelDigest) NeedsOf(pkg string) []string { if d == nil { return nil } for _, n := range d.Needs { if n.Package == pkg { return n.Needs } } return nil } // Map is the digest as digest by package. func (d *LevelDigest) Map() map[string]string { if d == nil || len(d.Packages) == 0 { return nil } m := make(map[string]string, len(d.Packages)) for _, p := range d.Packages { m[p.Package] = p.Digest } return m }