jschoubben
ad1f6e6ac2
not stored, not pinned, version taken and refused kind (moved here from the builder); FailedUntilChanged keeps an unpinned upstream or a taken version from rebuilding in a loop, while a failed push is built again.
259 lines
11 KiB
Go
259 lines
11 KiB
Go
package delivery
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
)
|
|
|
|
// The builder's keys (novox/hq ADR 0429 clauses 5 and 16, ADR 0437 clause 5, ADR 0405 clauses 2 and 7, ADR 0453
|
|
// clause 4): a build is `<module>.<fingerprint key part>` in the builder's bucket, one key per fingerprint, since equal
|
|
// fingerprints are equal content and the same source is never built twice. They are written only by the builder part
|
|
// from R4c and read by the checker, the assigner and the view, and from R4e by the controller. They are not the
|
|
// controller's build records (buildrecord.go: BuildsBucket, BuildKey), which stay as they are.
|
|
//
|
|
// **The rules its writer keeps.** A builder claims a fingerprint that has no build key by creating it queued or
|
|
// building, a create that fails when the key exists, and moves it by compare-and-set; a builder reclaims only a build
|
|
// whose `by` is offline in presence. A failed key is built again as a new revision of the same key (ADR 0429 clause
|
|
// 18). Only the builder's credential writes the bucket (clause 5); every other part reads it.
|
|
const (
|
|
// BuilderModule is the module of the builder part.
|
|
BuilderModule = "mesh-delivery-builder"
|
|
// BuilderBuildsBucket is the builder's bucket of builds: `<owner module>_<name>`.
|
|
BuilderBuildsBucket = BuilderModule + "_builds"
|
|
// BuilderBuildsHistory is the revisions the bucket keeps per key.
|
|
BuilderBuildsHistory = 5
|
|
)
|
|
|
|
// The states of a build.
|
|
const (
|
|
BuildQueued = "queued"
|
|
BuildBuilding = "building"
|
|
BuildDone = "done"
|
|
BuildFailed = "failed"
|
|
)
|
|
|
|
// BuilderBuildKey is the key, in BuilderBuildsBucket, of a module's build at a fingerprint: `<module>.<hex>`, the hex
|
|
// being the fingerprint's key part (FingerprintKeyPart), since a bus key may not hold ':'. A module holding a dot or a
|
|
// character a key refuses, and a fingerprint not of the new form, are refused, naming them.
|
|
func BuilderBuildKey(module, fingerprint string) (string, error) {
|
|
if err := checkName("module", module); err != nil {
|
|
return "", err
|
|
}
|
|
part, err := FingerprintKeyPart(fingerprint)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return module + "." + part, nil
|
|
}
|
|
|
|
// ParseBuilderBuildKey is the inverse of BuilderBuildKey: the module and the fingerprint (`src2:` and the hex).
|
|
func ParseBuilderBuildKey(key string) (module, fingerprint string, err error) {
|
|
module, part, ok := strings.Cut(key, ".")
|
|
if !ok {
|
|
return "", "", fmt.Errorf("delivery: %q is not a builder build key, <module>.<fingerprint hex>", key)
|
|
}
|
|
if err := checkName("module", module); err != nil {
|
|
return "", "", fmt.Errorf("delivery: %q is not a builder build key: %w", key, err)
|
|
}
|
|
if fingerprint, err = ParseFingerprintKeyPart(part); err != nil {
|
|
return "", "", fmt.Errorf("delivery: %q is not a builder build key: %w", key, err)
|
|
}
|
|
return module, fingerprint, nil
|
|
}
|
|
|
|
// BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest (`sha256:` and
|
|
// 64 hex digits) and the reference that names where it is (novox/hq ADR 0498 clauses 1, 5 and 6; the lead's decision
|
|
// L1 of R4e), by kind:
|
|
//
|
|
// bundle, archive artifact-store://<module>/<artifact>/blobs/<digest> the blob pushed to the artifact store
|
|
// image, upstream <registry>/<module>/<artifact>@<digest> the image registry's reference, which
|
|
// artifact-store://<module>/<artifact>@<digest> catalogue.Recorded keeps without its address
|
|
// package <name>@<version> the version published from the trunk
|
|
//
|
|
// <artifact> is Name and <digest> is Digest. A reference is empty for a candidate's image or package, which is never
|
|
// pushed or published (ADR 0498 clause 6), and in a key done before R4e; the controller delivers only from a key whose
|
|
// every artifact names one (Referenced, clause 8). A kind left empty (a key written before kinds) is not checked.
|
|
type BuiltArtifact struct {
|
|
Name string `json:"name"`
|
|
Kind string `json:"kind,omitempty"`
|
|
Digest string `json:"digest"`
|
|
Reference string `json:"reference,omitempty"`
|
|
}
|
|
|
|
const artifactStore = "artifact-store://"
|
|
|
|
// referenceProblem says what is wrong with a's reference for its kind, or "".
|
|
func (a BuiltArtifact) referenceProblem() string {
|
|
r := a.Reference
|
|
if r == "" || a.Kind == "" {
|
|
return ""
|
|
}
|
|
switch a.Kind {
|
|
case "bundle", "archive":
|
|
path, ok := strings.CutPrefix(r, artifactStore)
|
|
module, rest, _ := strings.Cut(path, "/")
|
|
if !ok || module == "" || rest != a.Name+"/blobs/"+a.Digest {
|
|
return "is not artifact-store://<module>/" + a.Name + "/blobs/" + a.Digest
|
|
}
|
|
case "image", "upstream":
|
|
path, digest, ok := strings.Cut(r, "@")
|
|
if !ok || digest != a.Digest {
|
|
return "does not name its digest " + a.Digest
|
|
}
|
|
var host, module, name string
|
|
if rest, kept := strings.CutPrefix(path, artifactStore); kept {
|
|
module, name, _ = strings.Cut(rest, "/")
|
|
host = "kept"
|
|
} else {
|
|
parts := strings.Split(path, "/")
|
|
if len(parts) == 3 {
|
|
host, module, name = parts[0], parts[1], parts[2]
|
|
}
|
|
}
|
|
if host == "" || !(host == "kept" || host == "localhost" || strings.ContainsAny(host, ".:")) || module == "" || name != a.Name {
|
|
return "is not <registry>/<module>/" + a.Name + "@" + a.Digest
|
|
}
|
|
case "package":
|
|
at := strings.LastIndex(r, "@")
|
|
if at <= 0 || at == len(r)-1 || r[:at] == "@" {
|
|
return "is not <name>@<version>"
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// Referenced says whether b is done and every artifact names where it is, which the controller delivers from (novox/hq
|
|
// ADR 0498 clause 8), and names the artifacts that name no reference.
|
|
func (b BuilderBuild) Referenced() (ok bool, missing []string) {
|
|
for _, a := range b.Artifacts {
|
|
if a.Reference == "" {
|
|
missing = append(missing, a.Name)
|
|
}
|
|
}
|
|
return b.State == BuildDone && len(b.Artifacts) > 0 && len(missing) == 0, missing
|
|
}
|
|
|
|
// The prefixes of a failed key's error that a build module answers (novox/hq ADR 0498 clauses 2, 4 and 5; ADR 0478
|
|
// clause 2), each followed by the artifact and the words of what refused it.
|
|
const (
|
|
// NotStored: the push of a bundle, an archive or an upstream copy to the artifact store failed; built again.
|
|
NotStored = "not stored: "
|
|
// NotPinned: an upstream artifact names no digest.
|
|
NotPinned = "not pinned: "
|
|
// VersionTaken: the package registry holds the package's version with another packed file.
|
|
VersionTaken = "version taken: "
|
|
// RefusedKind: no build module builds this kind of artifact in its language.
|
|
RefusedKind = "refused kind: "
|
|
)
|
|
|
|
// FailedUntilChanged says whether a failed key's error is one that is not built again until the module's fingerprint
|
|
// changes (not pinned, version taken, refused kind: the same source fails the same way); every other failed key, a
|
|
// push that was not stored included, is built again as a new revision (ADR 0429 clause 18).
|
|
func FailedUntilChanged(err string) bool {
|
|
return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind)
|
|
}
|
|
|
|
// ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and
|
|
// the build module's recipe version.
|
|
type ToolchainUsed struct {
|
|
Version string `json:"version"`
|
|
Recipe string `json:"recipe"`
|
|
}
|
|
|
|
// BuilderBuild is the value of a builder build key: its State (queued, building, done or failed), By the builder
|
|
// instance that holds it, Step what it is doing or did last, Manifest the digest of the module's manifest (the key of
|
|
// the indexer's manifests bucket), Artifacts when done, Toolchains by language when it built or is building (a done
|
|
// build names them, so two builders that differ are seen: ADR 0405 clause 2), and Error when failed.
|
|
type BuilderBuild struct {
|
|
V int `json:"v"`
|
|
State string `json:"state"`
|
|
By string `json:"by,omitempty"`
|
|
Step string `json:"step,omitempty"`
|
|
Manifest string `json:"manifest,omitempty"`
|
|
Artifacts []BuiltArtifact `json:"artifacts,omitempty"`
|
|
Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
func (b BuilderBuild) check() error {
|
|
switch b.State {
|
|
case BuildQueued, BuildBuilding, BuildDone, BuildFailed:
|
|
default:
|
|
return fmt.Errorf("delivery: the build state %q is not queued, building, done or failed", b.State)
|
|
}
|
|
if b.Manifest != "" {
|
|
if _, err := ManifestsKey(b.Manifest); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for language, t := range b.Toolchains {
|
|
if t.Version == "" {
|
|
return fmt.Errorf("delivery: the toolchain of %q has no version", language)
|
|
}
|
|
if t.Recipe == "" {
|
|
return fmt.Errorf("delivery: the toolchain of %q has no recipe version", language)
|
|
}
|
|
}
|
|
for i, a := range b.Artifacts {
|
|
if a.Name == "" {
|
|
return fmt.Errorf("delivery: artifact %d of the build has no name", i)
|
|
}
|
|
if a.Digest == "" {
|
|
return fmt.Errorf("delivery: the artifact %q of the build has no digest", a.Name)
|
|
}
|
|
if p := a.referenceProblem(); p != "" {
|
|
return fmt.Errorf("delivery: the reference %q of the %s %q %s", a.Reference, a.Kind, a.Name, p)
|
|
}
|
|
}
|
|
switch b.State {
|
|
case BuildQueued, BuildBuilding:
|
|
if len(b.Artifacts) > 0 || b.Error != "" {
|
|
return fmt.Errorf("delivery: a %s build carries no artifact and no error", b.State)
|
|
}
|
|
case BuildDone:
|
|
if len(b.Artifacts) == 0 {
|
|
return fmt.Errorf("delivery: a done build names no artifact")
|
|
}
|
|
if b.Manifest == "" {
|
|
return fmt.Errorf("delivery: a done build names no manifest digest")
|
|
}
|
|
if b.Error != "" {
|
|
return fmt.Errorf("delivery: a done build carries an error")
|
|
}
|
|
case BuildFailed:
|
|
if b.Error == "" {
|
|
return fmt.Errorf("delivery: a failed build says no error")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without artifacts or a
|
|
// manifest digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a
|
|
// recipe version).
|
|
func EncodeBuilderBuild(b BuilderBuild) ([]byte, error) {
|
|
if err := b.check(); err != nil {
|
|
return nil, err
|
|
}
|
|
b.V = Version
|
|
return json.Marshal(b)
|
|
}
|
|
|
|
// DecodeBuilderBuild reads a builder build value; it refuses an unknown version (ErrUnknownVersion) and what
|
|
// EncodeBuilderBuild refuses.
|
|
func DecodeBuilderBuild(data []byte) (BuilderBuild, error) {
|
|
var b BuilderBuild
|
|
if err := decode(data, &b); err != nil {
|
|
return BuilderBuild{}, err
|
|
}
|
|
if err := b.check(); err != nil {
|
|
return BuilderBuild{}, err
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
// SameBuilderBuild says whether two builds are the same value, so the builder writes only when they differ.
|
|
func SameBuilderBuild(a, b BuilderBuild) bool { return reflect.DeepEqual(a, b) }
|