package delivery import ( "encoding/json" "fmt" "reflect" "strings" ) // The builder's keys (novox/hq ADR 0429 clauses 5 and 16, ADR 0437 clause 5, ADR 0405 clauses 2 and 7, ADR 0453 // clause 4): a build is `.` in the builder's bucket, one key per fingerprint, since equal // fingerprints are equal content and the same source is never built twice. They are written only by the builder part // from R4c and read by the checker, the assigner and the view, and from R4e by the controller. They are not the // controller's build records (buildrecord.go: BuildsBucket, BuildKey), which stay as they are. // // **The rules its writer keeps.** A builder claims a fingerprint that has no build key by creating it queued or // building, a create that fails when the key exists, and moves it by compare-and-set; a builder reclaims only a build // whose `by` is offline in presence. A failed key is built again as a new revision of the same key (ADR 0429 clause // 18). Only the builder's credential writes the bucket (clause 5); every other part reads it. const ( // BuilderModule is the module of the builder part. BuilderModule = "mesh-delivery-builder" // BuilderBuildsBucket is the builder's bucket of builds: `_`. BuilderBuildsBucket = BuilderModule + "_builds" // BuilderBuildsHistory is the revisions the bucket keeps per key. BuilderBuildsHistory = 5 ) // The states of a build. const ( BuildQueued = "queued" BuildBuilding = "building" BuildDone = "done" BuildFailed = "failed" ) // BuilderBuildKey is the key, in BuilderBuildsBucket, of a module's build at a fingerprint: `.`, the hex // being the fingerprint's key part (FingerprintKeyPart), since a bus key may not hold ':'. A module holding a dot or a // character a key refuses, and a fingerprint not of the new form, are refused, naming them. func BuilderBuildKey(module, fingerprint string) (string, error) { if err := checkName("module", module); err != nil { return "", err } part, err := FingerprintKeyPart(fingerprint) if err != nil { return "", err } return module + "." + part, nil } // ParseBuilderBuildKey is the inverse of BuilderBuildKey: the module and the fingerprint (`src2:` and the hex). func ParseBuilderBuildKey(key string) (module, fingerprint string, err error) { module, part, ok := strings.Cut(key, ".") if !ok { return "", "", fmt.Errorf("delivery: %q is not a builder build key, .", key) } if err := checkName("module", module); err != nil { return "", "", fmt.Errorf("delivery: %q is not a builder build key: %w", key, err) } if fingerprint, err = ParseFingerprintKeyPart(part); err != nil { return "", "", fmt.Errorf("delivery: %q is not a builder build key: %w", key, err) } return module, fingerprint, nil } // BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest (`sha256:` and // 64 hex digits) and the reference that names where it is (novox/hq ADR 0498 clauses 1, 5 and 6; the lead's decision // L1 of R4e), by kind: // // bundle, archive artifact-store:////blobs/ the blob pushed to the artifact store // image, upstream //@ the image registry's reference, which // artifact-store:///@ catalogue.Recorded keeps without its address // package @ the version published from the trunk // // is Name and is Digest. A reference is empty for a candidate's image or package, which is never // pushed or published (ADR 0498 clause 6), and in a key done before R4e; the controller delivers only from a key whose // every artifact names one (Referenced, clause 8). A kind left empty (a key written before kinds) is not checked. type BuiltArtifact struct { Name string `json:"name"` Kind string `json:"kind,omitempty"` Digest string `json:"digest"` Reference string `json:"reference,omitempty"` } const artifactStore = "artifact-store://" // referenceProblem says what is wrong with a's reference for its kind, or "". func (a BuiltArtifact) referenceProblem() string { r := a.Reference if r == "" || a.Kind == "" { return "" } switch a.Kind { case "bundle", "archive": path, ok := strings.CutPrefix(r, artifactStore) module, rest, _ := strings.Cut(path, "/") if !ok || module == "" || rest != a.Name+"/blobs/"+a.Digest { return "is not artifact-store:///" + a.Name + "/blobs/" + a.Digest } case "image", "upstream": path, digest, ok := strings.Cut(r, "@") if !ok || digest != a.Digest { return "does not name its digest " + a.Digest } var host, module, name string if rest, kept := strings.CutPrefix(path, artifactStore); kept { module, name, _ = strings.Cut(rest, "/") host = "kept" } else { parts := strings.Split(path, "/") if len(parts) == 3 { host, module, name = parts[0], parts[1], parts[2] } } if host == "" || !(host == "kept" || host == "localhost" || strings.ContainsAny(host, ".:")) || module == "" || name != a.Name { return "is not //" + a.Name + "@" + a.Digest } case "package": at := strings.LastIndex(r, "@") if at <= 0 || at == len(r)-1 || r[:at] == "@" { return "is not @" } } return "" } // Referenced says whether b is done and every artifact names where it is, which the controller delivers from (novox/hq // ADR 0498 clause 8), and names the artifacts that name no reference. A done key with no artifact (a module that builds // none, issue 1613) is referenced. func (b BuilderBuild) Referenced() (ok bool, missing []string) { for _, a := range b.Artifacts { if a.Reference == "" { missing = append(missing, a.Name) } } return b.State == BuildDone && len(missing) == 0, missing } // The prefixes of a failed key's error that a build module answers (novox/hq ADR 0498 clauses 2, 4 and 5; ADR 0478 // clause 2), each followed by the artifact and the words of what refused it. const ( // NotStored: the push of a bundle, an archive or an upstream copy to the artifact store failed; built again. NotStored = "not stored: " // NotPinned: an upstream artifact names no digest. NotPinned = "not pinned: " // VersionTaken: the package registry holds the package's version with another packed file. VersionTaken = "version taken: " // RefusedKind: no build module builds this kind of artifact in its language. RefusedKind = "refused kind: " // WaitsFor: a fetch found the module's lock naming a package version the registry does not hold yet; the builder // writes the key queued with Waits (novox/hq ADR 0500 clause 6), and it is built once the package is published. WaitsFor = "waits for: " ) // FailedUntilChanged says whether a failed key's error is one that is not built again until the module's fingerprint // changes (not pinned, version taken, refused kind: the same source fails the same way); every other failed key, a // push that was not stored included, is built again as a new revision (ADR 0429 clause 18). func FailedUntilChanged(err string) bool { return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind) } // ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and // the build module's recipe version. type ToolchainUsed struct { Version string `json:"version"` Recipe string `json:"recipe"` } // BuilderBuild is the value of a builder build key: its State (queued, building, done or failed), By the builder // instance that holds it, Step what it is doing or did last, Manifest the digest of the module's manifest (the key of // the indexer's manifests bucket), Artifacts when done, Toolchains by language when it built or is building (a done // build names them, so two builders that differ are seen: ADR 0405 clause 2), Error when failed, and Waits when queued for a // package version. type BuilderBuild struct { V int `json:"v"` State string `json:"state"` By string `json:"by,omitempty"` Step string `json:"step,omitempty"` Manifest string `json:"manifest,omitempty"` Artifacts []BuiltArtifact `json:"artifacts,omitempty"` Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"` Error string `json:"error,omitempty"` // Waits is, on a queued key only, the package version `@` the module's lock names and the registry // does not yet hold: the key is not claimed until that package's trunk key is done and published (novox/hq ADR 0500 // clause 6). A reader that does not know the field reads the key as queued. Waits string `json:"waits,omitempty"` } func (b BuilderBuild) check() error { switch b.State { case BuildQueued, BuildBuilding, BuildDone, BuildFailed: default: return fmt.Errorf("delivery: the build state %q is not queued, building, done or failed", b.State) } if b.Manifest != "" { if _, err := ManifestsKey(b.Manifest); err != nil { return err } } for language, t := range b.Toolchains { if t.Version == "" { return fmt.Errorf("delivery: the toolchain of %q has no version", language) } if t.Recipe == "" { return fmt.Errorf("delivery: the toolchain of %q has no recipe version", language) } } for i, a := range b.Artifacts { if a.Name == "" { return fmt.Errorf("delivery: artifact %d of the build has no name", i) } if a.Digest == "" { return fmt.Errorf("delivery: the artifact %q of the build has no digest", a.Name) } if p := a.referenceProblem(); p != "" { return fmt.Errorf("delivery: the reference %q of the %s %q %s", a.Reference, a.Kind, a.Name, p) } } if b.Waits != "" { if b.State != BuildQueued { return fmt.Errorf("delivery: a %s build waits for %q; only a queued key waits for a package", b.State, b.Waits) } if p := (BuiltArtifact{Kind: "package", Reference: b.Waits}).referenceProblem(); p != "" { return fmt.Errorf("delivery: the package a key waits for, %q, %s", b.Waits, p) } } switch b.State { case BuildQueued, BuildBuilding: if len(b.Artifacts) > 0 || b.Error != "" { return fmt.Errorf("delivery: a %s build carries no artifact and no error", b.State) } case BuildDone: // A module that builds no artifact gets a done key too, and is delivered from it (issue 1613). if b.Manifest == "" { return fmt.Errorf("delivery: a done build names no manifest digest") } if b.Error != "" { return fmt.Errorf("delivery: a done build carries an error") } case BuildFailed: if b.Error == "" { return fmt.Errorf("delivery: a failed build says no error") } } return nil } // EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without a manifest // digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a // recipe version). func EncodeBuilderBuild(b BuilderBuild) ([]byte, error) { if err := b.check(); err != nil { return nil, err } b.V = Version return json.Marshal(b) } // DecodeBuilderBuild reads a builder build value; it refuses an unknown version (ErrUnknownVersion) and what // EncodeBuilderBuild refuses. func DecodeBuilderBuild(data []byte) (BuilderBuild, error) { var b BuilderBuild if err := decode(data, &b); err != nil { return BuilderBuild{}, err } if err := b.check(); err != nil { return BuilderBuild{}, err } return b, nil } // SameBuilderBuild says whether two builds are the same value, so the builder writes only when they differ. func SameBuilderBuild(a, b BuilderBuild) bool { return reflect.DeepEqual(a, b) }