mesh-delivery/modules/mesh-delivery-builder/builds/builds.go

// Package builds holds what the builder module owns and no seat carries (novox/hq ADR 0634 §1): the layout of its state
// `builds` (a build per module and fingerprint) and the rules its writer keeps, the question a registry seat's holder asks
// it before it stores a publish, its event version.numbered and the entries of its build log. Other modules import it from
// the builder by its published version (ADR 0573 §1); the builder's manifest declares it as a package artifact.
package builds

import (
	"encoding/json"
	"fmt"
	"reflect"
	"slices"
	"strings"

	"git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference"
	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry"
	"git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
	"git.novox.be/novox/mesh-delivery/sdk/buildstep"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms"
	"git.novox.be/novox/mesh-delivery/sdk/kept"
	"git.novox.be/novox/mesh-delivery/sdk/trunk"
	"git.novox.be/novox/mesh-sdk/go/keys"
)

// seats are the names of the registry seats a reference names first, from their contracts.
var seats = reference.Seats{Image: containerimageregistry.Seat, Object: ociartifactregistry.Seat}

// askLanguages are the languages a build seat is named after (node-build-<language>), which a building key's Asks are keyed by.
var askLanguages = []string{"go", "node", "image"}

func checkAskLanguage(language string) error {
	if !slices.Contains(askLanguages, language) {
		return fmt.Errorf("builds: the ask language %q is not go, node or image", language)
	}
	return nil
}

// The builder's keys (novox/hq ADR 0429 clauses 5 and 16, ADR 0437 clause 5, ADR 0405 clauses 2 and 7, ADR 0453
// clause 4): a build is `<module>.<fingerprint key part>` in the builder's bucket, one key per fingerprint, since equal
// fingerprints are equal content and the same source is never built twice. They are written only by the builder part
// from R4c and read by the checker, the assigner and the view. They are not the ended controller's build records
// (buildrecord.go: BuildKey).
//
// **The rules its writer keeps.** A builder claims a fingerprint that has no build key by creating it queued or
// building, a create that fails when the key exists, and moves it by compare-and-set; a builder reclaims only a build
// whose `by` is offline in presence. A failed key is built again as a new revision of the same key (ADR 0429 clause
// 18). Only the builder's credential writes the bucket (clause 5); every other part reads it.
const (
	// BuilderModule is the module of the builder part.
	BuilderModule = "mesh-delivery-builder"
	// BuilderBuildsBucket is the builder's bucket of builds: `<owner module>_<name>`.
	BuilderBuildsBucket = BuilderModule + "_builds"
	// BuilderBuildsHistory is the revisions the bucket keeps per key.
	BuilderBuildsHistory = 5
)

// The states of a build.
const (
	BuildQueued   = "queued"
	BuildBuilding = "building"
	BuildDone     = "done"
	BuildFailed   = "failed"
)

// BuilderBuildKey is the key, in BuilderBuildsBucket, of a module's build at a fingerprint: `<module>.<hex>`, the hex
// being the fingerprint's key part (meshcatalogclient.KeyPart), since a bus key may not hold ':'. A module holding a dot or a
// character a key refuses, and a fingerprint not of the new form, are refused, naming them.
func BuilderBuildKey(module, fp string) (string, error) {
	if err := keys.CheckName("module", module); err != nil {
		return "", err
	}
	part, err := meshcatalogclient.KeyPart(fp)
	if err != nil {
		return "", err
	}
	return module + "." + part, nil
}

// ParseBuilderBuildKey is the inverse of BuilderBuildKey: the module and the fingerprint (`src2:` and the hex).
func ParseBuilderBuildKey(key string) (module, fp string, err error) {
	module, part, ok := strings.Cut(key, ".")
	if !ok {
		return "", "", fmt.Errorf("builds: %q is not a builder build key, <module>.<fingerprint hex>", key)
	}
	if err := keys.CheckName("module", module); err != nil {
		return "", "", fmt.Errorf("builds: %q is not a builder build key: %w", key, err)
	}
	if fp, err = meshcatalogclient.ParseKeyPart(part); err != nil {
		return "", "", fmt.Errorf("builds: %q is not a builder build key: %w", key, err)
	}
	return module, fp, nil
}

// BuiltArtifact is one artifact of a done build: its name, kind, digest and the reference that names where it is, with every
// form it is named in (novox/hq ADR 0618, ADR 0624). The Delivery SDK's forms package holds the type, since the forms read it
// and forms cannot import this package.
type BuiltArtifact = forms.BuiltArtifact

const artifactStore = "artifact-store://"

// referenceProblem says what is wrong with a's reference for its kind, or "".
func referenceProblem(a BuiltArtifact) string {
	r := a.Reference
	if r == "" || a.Kind == "" {
		return ""
	}
	// An upstream copy may be named by the holder's own naming, upstream/<host>/<path>@<digest>, in either form: the
	// holder names it, not the module that asked for it (reference.InUpstream).
	if path, kept := seats.InArtifactStore(r); kept && a.Kind == "upstream" {
		if _, image, ok := reference.InUpstream(path); ok {
			if _, digest, _ := strings.Cut(image, "@"); digest != a.Digest {
				return "does not name its digest " + a.Digest
			}
			return ""
		}
	}
	if seat, path, ok := seats.InRegistrySeat(r); ok {
		want := "container-image-registry"
		if a.Kind == "bundle" || a.Kind == "archive" {
			want = "oci-artifact-registry"
		}
		module, rest, _ := strings.Cut(path, "/")
		if a.Kind == "package" || seat != want || module == "" || rest != a.Name+"@"+a.Digest {
			return "is not " + want + "/<module>/" + a.Name + "@" + a.Digest
		}
		return ""
	}
	switch a.Kind {
	case "bundle", "archive":
		path, ok := strings.CutPrefix(r, artifactStore)
		module, rest, _ := strings.Cut(path, "/")
		if !ok || module == "" || rest != a.Name+"/blobs/"+a.Digest {
			return "is not artifact-store://<module>/" + a.Name + "/blobs/" + a.Digest
		}
	case "image", "upstream":
		path, digest, ok := strings.Cut(r, "@")
		if !ok || digest != a.Digest {
			return "does not name its digest " + a.Digest
		}
		var host, module, name string
		if rest, kept := strings.CutPrefix(path, artifactStore); kept {
			module, name, _ = strings.Cut(rest, "/")
			host = "kept"
		} else {
			parts := strings.Split(path, "/")
			if len(parts) == 3 {
				host, module, name = parts[0], parts[1], parts[2]
			}
		}
		if host == "" || !(host == "kept" || host == "localhost" || strings.ContainsAny(host, ".:")) || module == "" || name != a.Name {
			return "is not <registry>/<module>/" + a.Name + "@" + a.Digest
		}
	case "package":
		at := strings.LastIndex(r, "@")
		if at <= 0 || at == len(r)-1 || r[:at] == "@" {
			return "is not <name>@<version>"
		}
	}
	return ""
}

// Referenced says whether b is done and every artifact names where it is, which the controller delivers from (novox/hq
// ADR 0498 clause 8), and names the artifacts that name no reference. A done key with no artifact (a module that builds
// none, issue 1613) is referenced.
func (b BuilderBuild) Referenced() (ok bool, missing []string) {
	for _, a := range b.Artifacts {
		if a.Reference == "" {
			missing = append(missing, a.Name)
		}
	}
	return b.State == BuildDone && len(missing) == 0, missing
}

// The prefixes of a failed key's error that a build module answers (novox/hq ADR 0498 clauses 2, 4 and 5; ADR 0478
// clause 2), each followed by the artifact and the words of what refused it.
const (
	// NotStored: the push of a bundle, an archive or an upstream copy to the artifact store failed; built again.
	NotStored = "not stored: "
	// NotPinned: an upstream artifact names no digest.
	NotPinned = "not pinned: "
	// VersionTaken: the package registry holds the package's version with another packed file.
	VersionTaken = "version taken: "
	// RefusedKind: no build module builds this kind of artifact in its language.
	RefusedKind = "refused kind: "
	// WaitsFor: a fetch found the module's lock naming a package version the registry does not hold yet; the builder
	// writes the key queued with Waits (novox/hq ADR 0500 clause 6), and it is built once the package is published.
	WaitsFor = "waits for: "
)

// FailedUntilChanged says whether a failed key's error is one that is not built again until the module's fingerprint
// changes (not pinned, version taken, refused kind: the same source fails the same way); every other failed key, a
// push that was not stored included, is built again as a new revision (ADR 0429 clause 18).
func FailedUntilChanged(err string) bool {
	return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind)
}

// ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and
// the build module's recipe version.
type ToolchainUsed struct {
	Version string `json:"version"`
	Recipe  string `json:"recipe"`
}

// BuilderBuild is the value of a builder build key: its State (queued, building, done or failed), By the builder
// instance that holds it, Step what it is doing or did last, Manifest the digest of the module's manifest (the key of
// the indexer's manifests bucket), Artifacts when done, Toolchains by language when it built or is building (a done
// build names them, so two builders that differ are seen: ADR 0405 clause 2), Error when failed, and Waits when queued for a
// package version.
type BuilderBuild struct {
	V          int                      `json:"v"`
	State      string                   `json:"state"`
	By         string                   `json:"by,omitempty"`
	Step       string                   `json:"step,omitempty"`
	Manifest   string                   `json:"manifest,omitempty"`
	Artifacts  []BuiltArtifact          `json:"artifacts,omitempty"`
	Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"`
	Error      string                   `json:"error,omitempty"`
	// Waits is, on a queued key only, the package version `<name>@<version>` the module's lock names and the registry
	// does not yet hold: the key is not claimed until that package's trunk key is done and published (novox/hq ADR 0500
	// clause 6). A reader that does not know the field reads the key as queued.
	Waits string `json:"waits,omitempty"`
	// Asks are, on a building key, the asks the builder put for it by build seat language (AskGo, AskNode, AskImage), each
	// its id (novox/hq ADR 0525 §1; issue 1887 A4b): a builder that takes the key over at its start waits for those asks'
	// steps and never puts them again. A reader that does not know the field reads the key as before.
	Asks map[string]string `json:"asks,omitempty"`
	// Imports are, on a done key, the mesh packages the built programs import, as the build module answered them
	// (BuildAnswer.Imports; novox/hq ADR 0593 §2): the builder carries them onto the build's end (BuildLogEntry.Imports).
	Imports []string `json:"imports,omitempty"`
	// Needs are, on a done key only, the reader capabilities (`form:<name>`) the build's reference forms need of a
	// node-engine, beside the ones its manifest needs (novox/hq ADR 0618 §1). A reader that does not know the field
	// reads the build as needing none.
	Needs []string `json:"needs,omitempty"`
}

func (b BuilderBuild) check() error {
	if err := trunk.CheckImports(b.Imports); err != nil {
		return err
	}
	switch b.State {
	case BuildQueued, BuildBuilding, BuildDone, BuildFailed:
	default:
		return fmt.Errorf("builds: the build state %q is not queued, building, done or failed", b.State)
	}
	if b.Manifest != "" {
		if !keys.IsHex64(b.Manifest) {
			return fmt.Errorf("builds: %q is not a manifest digest, 64 lowercase hex digits", b.Manifest)
		}
	}
	for language, t := range b.Toolchains {
		if t.Version == "" {
			return fmt.Errorf("builds: the toolchain of %q has no version", language)
		}
		if t.Recipe == "" {
			return fmt.Errorf("builds: the toolchain of %q has no recipe version", language)
		}
	}
	for i, a := range b.Artifacts {
		if a.Name == "" {
			return fmt.Errorf("builds: artifact %d of the build has no name", i)
		}
		if a.Digest == "" {
			return fmt.Errorf("builds: the artifact %q of the build has no digest", a.Name)
		}
		if p := referenceProblem(a); p != "" {
			return fmt.Errorf("builds: the reference %q of the %s %q %s", a.Reference, a.Kind, a.Name, p)
		}
		if len(a.References) > 0 && !slices.Contains(a.References, a.Reference) {
			return fmt.Errorf("builds: the references of the %s %q do not name its reference %q", a.Kind, a.Name, a.Reference)
		}
		for _, r := range a.References {
			if p := referenceProblem(BuiltArtifact{Name: a.Name, Kind: a.Kind, Digest: a.Digest, Reference: r}); p != "" {
				return fmt.Errorf("builds: the reference %q of the %s %q %s", r, a.Kind, a.Name, p)
			}
		}
	}
	for _, n := range b.Needs {
		if !strings.HasPrefix(n, "form:") || len(n) == len("form:") {
			return fmt.Errorf("builds: the need %q of the build is not form:<name>", n)
		}
	}
	if len(b.Needs) > 0 && b.State != BuildDone {
		return fmt.Errorf("builds: a %s build names needs; only a done key does", b.State)
	}
	for language, id := range b.Asks {
		if b.State != BuildBuilding {
			return fmt.Errorf("builds: a %s build names the ask %s; only a building key does", b.State, id)
		}
		if err := checkAskLanguage(language); err != nil {
			return err
		}
		if err := buildstep.CheckAskID(id); err != nil {
			return err
		}
	}
	if b.Waits != "" {
		if b.State != BuildQueued {
			return fmt.Errorf("builds: a %s build waits for %q; only a queued key waits for a package", b.State, b.Waits)
		}
		if p := referenceProblem(BuiltArtifact{Kind: "package", Reference: b.Waits}); p != "" {
			return fmt.Errorf("builds: the package a key waits for, %q, %s", b.Waits, p)
		}
	}
	switch b.State {
	case BuildQueued, BuildBuilding:
		if len(b.Artifacts) > 0 || b.Error != "" {
			return fmt.Errorf("builds: a %s build carries no artifact and no error", b.State)
		}
	case BuildDone:
		// A module that builds no artifact gets a done key too, and is delivered from it (issue 1613).
		if b.Manifest == "" {
			return fmt.Errorf("builds: a done build names no manifest digest")
		}
		if b.Error != "" {
			return fmt.Errorf("builds: a done build carries an error")
		}
	case BuildFailed:
		if b.Error == "" {
			return fmt.Errorf("builds: a failed build says no error")
		}
	}
	return nil
}

// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without a manifest
// digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a
// recipe version).
func EncodeBuilderBuild(b BuilderBuild) ([]byte, error) {
	if err := b.check(); err != nil {
		return nil, err
	}
	b.V = keys.Version
	return json.Marshal(b)
}

// DecodeBuilderBuild reads a builder build value; it refuses an unknown version (keys.ErrUnknownVersion) and what
// EncodeBuilderBuild refuses.
func DecodeBuilderBuild(data []byte) (BuilderBuild, error) {
	var b BuilderBuild
	if err := kept.Decode(data, &b); err != nil {
		return BuilderBuild{}, err
	}
	if err := b.check(); err != nil {
		return BuilderBuild{}, err
	}
	return b, nil
}

// SameBuilderBuild says whether two builds are the same value, so the builder writes only when they differ.
func SameBuilderBuild(a, b BuilderBuild) bool { return reflect.DeepEqual(a, b) }