Files
mesh-delivery/modules/mesh-delivery-builder/builds/builds.go
T
jschoubben 39db267630 D1 (builder): the builder names its own module and buckets from its credential's module
Its states' buckets, its finding observer, its build log entries and its step asks' asker take the credential's module (tools: the module the runtime serves them as) instead of constants of its name; builds.StateBucket/BuildsBucket replace the Bucket constants. The readers of the builds bucket (checker, view, grants) still name the builder: the next D1 subtask finds it by the seat mesh-builds.

Builds: design 47 G A-296, §428

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 22:34:08 +01:00

393 lines
18 KiB
Go

// Package builds holds what the builder module owns and no seat carries (novox/hq ADR 0634 §1): the layout of its state
// `builds` (a build per module and fingerprint) and the rules its writer keeps, the question a registry seat's holder asks
// it before it stores a publish and the entries of its build log. Other modules import it from
// the builder by its published version (ADR 0573 §1); the builder's manifest declares it as a package artifact.
package builds
import (
"encoding/json"
"fmt"
"reflect"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient/reference"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/kept"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// seats are the names of the registry seats a reference names first, from their contracts.
var seats = reference.Seats{Image: containerimageregistry.Seat, Object: ociartifactregistry.Seat}
// askLanguages are the languages a build seat is named after (node-build-<language>), which a building key's Asks are keyed by.
var askLanguages = []string{"go", "node", "image"}
func checkAskLanguage(language string) error {
if !slices.Contains(askLanguages, language) {
return fmt.Errorf("builds: the ask language %q is not go, node or image", language)
}
return nil
}
// The builder's keys (novox/hq ADR 0429 clauses 5 and 16, ADR 0437 clause 5, ADR 0405 clauses 2 and 7, ADR 0453
// clause 4): a build is `<module>.<fingerprint key part>` in the builder's bucket, one key per fingerprint, since equal
// fingerprints are equal content and the same source is never built twice. They are written only by the builder part
// from R4c and read by the checker, the assigner and the view. They are not the ended controller's build records
// (buildrecord.go: BuildKey).
//
// **The rules its writer keeps.** A builder claims a fingerprint that has no build key by creating it queued or
// building, a create that fails when the key exists, and moves it by compare-and-set; a builder reclaims only a build
// whose `by` is offline in presence. A failed key is built again as a new revision of the same key (ADR 0429 clause
// 18). Only the builder's credential writes the bucket (clause 5); every other part reads it.
const (
// BuilderModule is the module of the builder part.
BuilderModule = "mesh-delivery-builder"
// BuilderBuildsHistory is the revisions the bucket keeps per key.
BuilderBuildsHistory = 5
)
// BuildsBucket is the builder's bucket of builds: `<module>_builds`, of the module the builder runs as (its credential's), never
// a constant of its name (design 47 G A-296).
func BuildsBucket(module string) string { return StateBucket(module, "builds") }
// StateBucket is the bus bucket of a module's state: `<module>_<state>`.
func StateBucket(module, state string) string { return module + "_" + state }
// The states of a build.
const (
BuildQueued = "queued"
BuildBuilding = "building"
BuildDone = "done"
BuildFailed = "failed"
)
// BuilderBuildKey is the key, in BuilderBuildsBucket, of a module's build at a fingerprint: `<module>.<hex>`, the hex
// being the fingerprint's key part (catalogkey.KeyPart), since a bus key may not hold ':'. A module holding a dot or a
// character a key refuses, and a fingerprint not of the new form, are refused, naming them.
func BuilderBuildKey(module, fp string) (string, error) {
if err := keys.CheckName("module", module); err != nil {
return "", err
}
part, err := catalogkey.KeyPart(fp)
if err != nil {
return "", err
}
return module + "." + part, nil
}
// ParseBuilderBuildKey is the inverse of BuilderBuildKey: the module and the fingerprint (`src2:` and the hex).
func ParseBuilderBuildKey(key string) (module, fp string, err error) {
module, part, ok := strings.Cut(key, ".")
if !ok {
return "", "", fmt.Errorf("builds: %q is not a builder build key, <module>.<fingerprint hex>", key)
}
if err := keys.CheckName("module", module); err != nil {
return "", "", fmt.Errorf("builds: %q is not a builder build key: %w", key, err)
}
if fp, err = catalogkey.ParseKeyPart(part); err != nil {
return "", "", fmt.Errorf("builds: %q is not a builder build key: %w", key, err)
}
return module, fp, nil
}
// BuiltArtifact is one artifact of a done build: its name, kind, digest and the reference that names where it is, with every
// form it is named in (novox/hq ADR 0618, ADR 0624). The Delivery SDK's forms package holds the type, since the forms read it
// and forms cannot import this package.
type BuiltArtifact = forms.BuiltArtifact
const artifactStore = "artifact-store://"
// referenceProblem says what is wrong with a's reference for its kind, or "".
func referenceProblem(a BuiltArtifact) string {
r := a.Reference
if r == "" || a.Kind == "" {
return ""
}
// An upstream copy may be named by the holder's own naming, upstream/<host>/<path>@<digest>, in either form: the
// holder names it, not the module that asked for it (reference.InUpstream).
if path, kept := seats.InArtifactStore(r); kept && a.Kind == "upstream" {
if _, image, ok := reference.InUpstream(path); ok {
if _, digest, _ := strings.Cut(image, "@"); digest != a.Digest {
return "does not name its digest " + a.Digest
}
return ""
}
}
if seat, path, ok := seats.InRegistrySeat(r); ok {
want := "container-image-registry"
if a.Kind == "bundle" || a.Kind == "archive" {
want = "oci-artifact-registry"
}
module, rest, _ := strings.Cut(path, "/")
if a.Kind == "package" || seat != want || module == "" || rest != a.Name+"@"+a.Digest {
return "is not " + want + "/<module>/" + a.Name + "@" + a.Digest
}
return ""
}
switch a.Kind {
case "bundle", "archive":
path, ok := strings.CutPrefix(r, artifactStore)
module, rest, _ := strings.Cut(path, "/")
if !ok || module == "" || rest != a.Name+"/blobs/"+a.Digest {
return "is not artifact-store://<module>/" + a.Name + "/blobs/" + a.Digest
}
case "image", "upstream":
path, digest, ok := strings.Cut(r, "@")
if !ok || digest != a.Digest {
return "does not name its digest " + a.Digest
}
var host, module, name string
if rest, kept := strings.CutPrefix(path, artifactStore); kept {
module, name, _ = strings.Cut(rest, "/")
host = "kept"
} else {
parts := strings.Split(path, "/")
if len(parts) == 3 {
host, module, name = parts[0], parts[1], parts[2]
}
}
if host == "" || !(host == "kept" || host == "localhost" || strings.ContainsAny(host, ".:")) || module == "" || name != a.Name {
return "is not <registry>/<module>/" + a.Name + "@" + a.Digest
}
case "package":
at := strings.LastIndex(r, "@")
if at <= 0 || at == len(r)-1 || r[:at] == "@" {
return "is not <name>@<version>"
}
}
return ""
}
// Referenced says whether b is done and every artifact names where it is, which the controller delivers from (novox/hq
// ADR 0498 clause 8), and names the artifacts that name no reference. A done key with no artifact (a module that builds
// none, issue 1613) is referenced.
func (b BuilderBuild) Referenced() (ok bool, missing []string) {
for _, a := range b.Artifacts {
if a.Reference == "" {
missing = append(missing, a.Name)
}
}
return b.State == BuildDone && len(missing) == 0, missing
}
// The prefixes of a failed key's error that a build module answers (novox/hq ADR 0498 clauses 2, 4 and 5; ADR 0478
// clause 2), each followed by the artifact and the words of what refused it.
const (
// NotStored: the push of a bundle, an archive or an upstream copy to the artifact store failed; built again.
NotStored = "not stored: "
// NotPinned: an upstream artifact names no digest.
NotPinned = "not pinned: "
// VersionTaken: the package registry holds the package's version with another packed file.
VersionTaken = "version taken: "
// RefusedKind: no build module builds this kind of artifact in its language.
RefusedKind = "refused kind: "
// WaitsFor: a fetch found the module's lock naming a package version the registry does not hold yet; the builder
// writes the key queued with Waits (novox/hq ADR 0500 clause 6), and it is built once the package is published.
WaitsFor = "waits for: "
)
// FailedUntilChanged says whether a failed key's error is one that is not built again until the module's fingerprint
// changes (not pinned, version taken, refused kind: the same source fails the same way); every other failed key, a
// push that was not stored included, is built again as a new revision (ADR 0429 clause 18).
func FailedUntilChanged(err string) bool {
return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind)
}
// TransportFaultPrefix opens the error of a key that failed on a transport fault: `transport fault: <class>, <seat> <node>,
// <cause>` (design 47 G §568, A-76). No other failure opens it.
const TransportFaultPrefix = "transport fault:"
// FailedOnTransportFault says whether a failed key's error opens TransportFaultPrefix: a fault of the network, not of the
// source, so the builder does not retake it on its look (ADR 0664 §3, which ADR 0673 changes): a person's hand act, a builder
// that starts again, or an event that says its cause passed (design 47 G §568) builds it again.
func FailedOnTransportFault(err string) bool {
return strings.HasPrefix(err, TransportFaultPrefix)
}
// ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and
// the build module's recipe version.
type ToolchainUsed struct {
Version string `json:"version"`
Recipe string `json:"recipe"`
}
// BuilderBuild is the value of a builder build key: its State (queued, building, done or failed), By the builder
// instance that holds it, Step what it is doing or did last, Manifest the digest of the module's manifest (the key of
// the indexer's manifests bucket), Artifacts when done, Toolchains by language when it built or is building (a done
// build names them, so two builders that differ are seen: ADR 0405 clause 2), Error when failed, and Waits when queued for a
// package version.
type BuilderBuild struct {
V int `json:"v"`
State string `json:"state"`
By string `json:"by,omitempty"`
Step string `json:"step,omitempty"`
Manifest string `json:"manifest,omitempty"`
Artifacts []BuiltArtifact `json:"artifacts,omitempty"`
Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"`
Error string `json:"error,omitempty"`
// Waits is, on a queued key only, the package version `<name>@<version>` the module's lock names and the registry
// does not yet hold: the key is not claimed until that package's trunk key is done and published (novox/hq ADR 0500
// clause 6). A reader that does not know the field reads the key as queued.
Waits string `json:"waits,omitempty"`
// Asks are, on a building key, the asks the builder put for it by build seat language (AskGo, AskNode, AskImage), each
// its id (novox/hq ADR 0525 §1; issue 1887 A4b): a builder that takes the key over at its start waits for those asks'
// steps and never puts them again. A reader that does not know the field reads the key as before.
Asks map[string]string `json:"asks,omitempty"`
// Imports are, on a done key, the mesh packages the built programs import, as the build module answered them
// (BuildAnswer.Imports; novox/hq ADR 0593 §2): the builder carries them onto the build's end (BuildLogEntry.Imports).
Imports []string `json:"imports,omitempty"`
// Needs are, on a done key only, the reader capabilities (`form:<name>`) the build's reference forms need of a
// node-engine, beside the ones its manifest needs (novox/hq ADR 0618 §1). A reader that does not know the field
// reads the build as needing none.
Needs []string `json:"needs,omitempty"`
// KeptOn is, on a done key only, the build node that keeps the build instead of storing it through the seat
// oci-artifact-registry (design 47 G §671): every artifact of the build is kept there, each with its digest in
// Artifacts. A build not kept names none; the builder carries it onto the build's end (BuildLogEntry.KeptOn).
KeptOn string `json:"kept-on,omitempty"`
// EventBuilds counts, in the key's newest revision, the revisions an event started since the last one a person started
// (design 47 G §568, ADR 0673): the builder builds a key that failed on a transport fault again on an event for the third
// time at most, then asks the operator. A person's build (builder_build_again, the operator's answer) starts the count
// again. A reader that does not know the field reads none.
EventBuilds int `json:"event-builds,omitempty"`
}
func (b BuilderBuild) check() error {
if b.KeptOn != "" && b.State != BuildDone {
return fmt.Errorf("builds: a build is kept on a done key alone, this one is %s", b.State)
}
if err := trunk.CheckImports(b.Imports); err != nil {
return err
}
switch b.State {
case BuildQueued, BuildBuilding, BuildDone, BuildFailed:
default:
return fmt.Errorf("builds: the build state %q is not queued, building, done or failed", b.State)
}
if b.Manifest != "" {
if !keys.IsHex64(b.Manifest) {
return fmt.Errorf("builds: %q is not a manifest digest, 64 lowercase hex digits", b.Manifest)
}
}
for language, t := range b.Toolchains {
if t.Version == "" {
return fmt.Errorf("builds: the toolchain of %q has no version", language)
}
if t.Recipe == "" {
return fmt.Errorf("builds: the toolchain of %q has no recipe version", language)
}
}
for i, a := range b.Artifacts {
if a.Name == "" {
return fmt.Errorf("builds: artifact %d of the build has no name", i)
}
if a.Digest == "" {
return fmt.Errorf("builds: the artifact %q of the build has no digest", a.Name)
}
if p := referenceProblem(a); p != "" {
return fmt.Errorf("builds: the reference %q of the %s %q %s", a.Reference, a.Kind, a.Name, p)
}
if len(a.References) > 0 && !slices.Contains(a.References, a.Reference) {
return fmt.Errorf("builds: the references of the %s %q do not name its reference %q", a.Kind, a.Name, a.Reference)
}
for _, r := range a.References {
if p := referenceProblem(BuiltArtifact{Name: a.Name, Kind: a.Kind, Digest: a.Digest, Reference: r}); p != "" {
return fmt.Errorf("builds: the reference %q of the %s %q %s", r, a.Kind, a.Name, p)
}
}
}
for _, n := range b.Needs {
if !strings.HasPrefix(n, "form:") || len(n) == len("form:") {
return fmt.Errorf("builds: the need %q of the build is not form:<name>", n)
}
}
if len(b.Needs) > 0 && b.State != BuildDone {
return fmt.Errorf("builds: a %s build names needs; only a done key does", b.State)
}
for language, id := range b.Asks {
if b.State != BuildBuilding {
return fmt.Errorf("builds: a %s build names the ask %s; only a building key does", b.State, id)
}
if err := checkAskLanguage(language); err != nil {
return err
}
if err := buildstep.CheckAskID(id); err != nil {
return err
}
}
if b.Waits != "" {
if b.State != BuildQueued {
return fmt.Errorf("builds: a %s build waits for %q; only a queued key waits for a package", b.State, b.Waits)
}
if p := referenceProblem(BuiltArtifact{Kind: "package", Reference: b.Waits}); p != "" {
return fmt.Errorf("builds: the package a key waits for, %q, %s", b.Waits, p)
}
}
switch b.State {
case BuildQueued, BuildBuilding:
if len(b.Artifacts) > 0 || b.Error != "" {
return fmt.Errorf("builds: a %s build carries no artifact and no error", b.State)
}
case BuildDone:
// A module that builds no artifact gets a done key too, and is delivered from it (issue 1613).
if b.Manifest == "" {
return fmt.Errorf("builds: a done build names no manifest digest")
}
if b.Error != "" {
return fmt.Errorf("builds: a done build carries an error")
}
case BuildFailed:
if b.Error == "" {
return fmt.Errorf("builds: a failed build says no error")
}
}
return nil
}
// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without a manifest
// digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a
// recipe version).
func EncodeBuilderBuild(b BuilderBuild) ([]byte, error) {
if err := b.check(); err != nil {
return nil, err
}
b.V = keys.Version
return json.Marshal(b)
}
// DecodeBuilderBuild reads a builder build value; it refuses an unknown version (keys.ErrUnknownVersion) and what
// EncodeBuilderBuild refuses.
func DecodeBuilderBuild(data []byte) (BuilderBuild, error) {
var b BuilderBuild
if err := kept.Decode(data, &b); err != nil {
return BuilderBuild{}, err
}
if err := b.check(); err != nil {
return BuilderBuild{}, err
}
return b, nil
}
// SameBuilderBuild says whether two builds are the same value, so the builder writes only when they differ.
func SameBuilderBuild(a, b BuilderBuild) bool { return reflect.DeepEqual(a, b) }
// TransportFaultOf reads a key's error `transport fault: <class>, <seat> <node>, <cause>` (design 47 G §568, A-76): ok is false
// for an error that does not open so. The node of a fault no node ran (the put of an ask) is `no node`.
func TransportFaultOf(err string) (class, seat, node, cause string, ok bool) {
rest, found := strings.CutPrefix(err, TransportFaultPrefix+" ")
if !found {
return "", "", "", "", false
}
parts := strings.SplitN(rest, ", ", 3)
if len(parts) != 3 {
return "", "", "", "", false
}
seat, node, _ = strings.Cut(parts[1], " ")
return parts[0], seat, node, parts[2], seat != "" && node != ""
}