Lay out the builder's build keys, builds.<module>.<fingerprint>, beside the controller's (issue 1358)
This commit is contained in:
@@ -0,0 +1,172 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The builder's keys (novox/hq ADR 0429 clauses 5 and 16, ADR 0437 clause 5, ADR 0405 clauses 2 and 7, ADR 0453
|
||||
// clause 4): a build is `<module>.<fingerprint key part>` in the builder's bucket, one key per fingerprint, since equal
|
||||
// fingerprints are equal content and the same source is never built twice. They are written only by the builder part
|
||||
// from R4c and read by the checker, the assigner and the view, and from R4e by the controller. They are not the
|
||||
// controller's build records (buildrecord.go: BuildsBucket, BuildKey), which stay as they are.
|
||||
//
|
||||
// **The rules its writer keeps.** A builder claims a fingerprint that has no build key by creating it queued or
|
||||
// building, a create that fails when the key exists, and moves it by compare-and-set; a builder reclaims only a build
|
||||
// whose `by` is offline in presence. A failed key is built again as a new revision of the same key (ADR 0429 clause
|
||||
// 18). Only the builder's credential writes the bucket (clause 5); every other part reads it.
|
||||
const (
|
||||
// BuilderModule is the module of the builder part.
|
||||
BuilderModule = "mesh-delivery-builder"
|
||||
// BuilderBuildsBucket is the builder's bucket of builds: `<owner module>_<name>`.
|
||||
BuilderBuildsBucket = BuilderModule + "_builds"
|
||||
// BuilderBuildsHistory is the revisions the bucket keeps per key.
|
||||
BuilderBuildsHistory = 5
|
||||
)
|
||||
|
||||
// The states of a build.
|
||||
const (
|
||||
BuildQueued = "queued"
|
||||
BuildBuilding = "building"
|
||||
BuildDone = "done"
|
||||
BuildFailed = "failed"
|
||||
)
|
||||
|
||||
// BuilderBuildKey is the key, in BuilderBuildsBucket, of a module's build at a fingerprint: `<module>.<hex>`, the hex
|
||||
// being the fingerprint's key part (FingerprintKeyPart), since a bus key may not hold ':'. A module holding a dot or a
|
||||
// character a key refuses, and a fingerprint not of the new form, are refused, naming them.
|
||||
func BuilderBuildKey(module, fingerprint string) (string, error) {
|
||||
if err := checkName("module", module); err != nil {
|
||||
return "", err
|
||||
}
|
||||
part, err := FingerprintKeyPart(fingerprint)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return module + "." + part, nil
|
||||
}
|
||||
|
||||
// ParseBuilderBuildKey is the inverse of BuilderBuildKey: the module and the fingerprint (`src2:` and the hex).
|
||||
func ParseBuilderBuildKey(key string) (module, fingerprint string, err error) {
|
||||
module, part, ok := strings.Cut(key, ".")
|
||||
if !ok {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a builder build key, <module>.<fingerprint hex>", key)
|
||||
}
|
||||
if err := checkName("module", module); err != nil {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a builder build key: %w", key, err)
|
||||
}
|
||||
if fingerprint, err = ParseFingerprintKeyPart(part); err != nil {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a builder build key: %w", key, err)
|
||||
}
|
||||
return module, fingerprint, nil
|
||||
}
|
||||
|
||||
// BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest and, for an
|
||||
// image, the reference it was pushed to.
|
||||
type BuiltArtifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
Digest string `json:"digest"`
|
||||
Reference string `json:"reference,omitempty"`
|
||||
}
|
||||
|
||||
// ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and
|
||||
// the build module's recipe version.
|
||||
type ToolchainUsed struct {
|
||||
Version string `json:"version"`
|
||||
Recipe string `json:"recipe"`
|
||||
}
|
||||
|
||||
// BuilderBuild is the value of a builder build key: its State (queued, building, done or failed), By the builder
|
||||
// instance that holds it, Step what it is doing or did last, Manifest the digest of the module's manifest (the key of
|
||||
// the indexer's manifests bucket), Artifacts when done, Toolchains by language when it built or is building (a done
|
||||
// build names them, so two builders that differ are seen: ADR 0405 clause 2), and Error when failed.
|
||||
type BuilderBuild struct {
|
||||
V int `json:"v"`
|
||||
State string `json:"state"`
|
||||
By string `json:"by,omitempty"`
|
||||
Step string `json:"step,omitempty"`
|
||||
Manifest string `json:"manifest,omitempty"`
|
||||
Artifacts []BuiltArtifact `json:"artifacts,omitempty"`
|
||||
Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (b BuilderBuild) check() error {
|
||||
switch b.State {
|
||||
case BuildQueued, BuildBuilding, BuildDone, BuildFailed:
|
||||
default:
|
||||
return fmt.Errorf("delivery: the build state %q is not queued, building, done or failed", b.State)
|
||||
}
|
||||
if b.Manifest != "" {
|
||||
if _, err := ManifestsKey(b.Manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for language, t := range b.Toolchains {
|
||||
if t.Version == "" {
|
||||
return fmt.Errorf("delivery: the toolchain of %q has no version", language)
|
||||
}
|
||||
if t.Recipe == "" {
|
||||
return fmt.Errorf("delivery: the toolchain of %q has no recipe version", language)
|
||||
}
|
||||
}
|
||||
for i, a := range b.Artifacts {
|
||||
if a.Name == "" {
|
||||
return fmt.Errorf("delivery: artifact %d of the build has no name", i)
|
||||
}
|
||||
if a.Digest == "" {
|
||||
return fmt.Errorf("delivery: the artifact %q of the build has no digest", a.Name)
|
||||
}
|
||||
}
|
||||
switch b.State {
|
||||
case BuildQueued, BuildBuilding:
|
||||
if len(b.Artifacts) > 0 || b.Error != "" {
|
||||
return fmt.Errorf("delivery: a %s build carries no artifact and no error", b.State)
|
||||
}
|
||||
case BuildDone:
|
||||
if len(b.Artifacts) == 0 {
|
||||
return fmt.Errorf("delivery: a done build names no artifact")
|
||||
}
|
||||
if b.Manifest == "" {
|
||||
return fmt.Errorf("delivery: a done build names no manifest digest")
|
||||
}
|
||||
if b.Error != "" {
|
||||
return fmt.Errorf("delivery: a done build carries an error")
|
||||
}
|
||||
case BuildFailed:
|
||||
if b.Error == "" {
|
||||
return fmt.Errorf("delivery: a failed build says no error")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without artifacts or a
|
||||
// manifest digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a
|
||||
// recipe version).
|
||||
func EncodeBuilderBuild(b BuilderBuild) ([]byte, error) {
|
||||
if err := b.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b.V = Version
|
||||
return json.Marshal(b)
|
||||
}
|
||||
|
||||
// DecodeBuilderBuild reads a builder build value; it refuses an unknown version (ErrUnknownVersion) and what
|
||||
// EncodeBuilderBuild refuses.
|
||||
func DecodeBuilderBuild(data []byte) (BuilderBuild, error) {
|
||||
var b BuilderBuild
|
||||
if err := decode(data, &b); err != nil {
|
||||
return BuilderBuild{}, err
|
||||
}
|
||||
if err := b.check(); err != nil {
|
||||
return BuilderBuild{}, err
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// SameBuilderBuild says whether two builds are the same value, so the builder writes only when they differ.
|
||||
func SameBuilderBuild(a, b BuilderBuild) bool { return reflect.DeepEqual(a, b) }
|
||||
@@ -0,0 +1,187 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func doneBuild() BuilderBuild {
|
||||
return BuilderBuild{
|
||||
State: BuildDone,
|
||||
By: "builder.novox.1",
|
||||
Step: "pushed",
|
||||
Manifest: digestOf('4'),
|
||||
Artifacts: []BuiltArtifact{
|
||||
{Name: "code", Kind: "bundle", Digest: "sha256:" + digestOf('5')},
|
||||
{Name: "server", Kind: "image", Digest: "sha256:" + digestOf('6'), Reference: "registry.example/mail@sha256:" + digestOf('6')},
|
||||
},
|
||||
Toolchains: map[string]ToolchainUsed{
|
||||
"go": {Version: "1.26.0", Recipe: "3"},
|
||||
"typescript": {Version: "5.9.2", Recipe: "1"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func builderSamples() map[string]BuilderBuild {
|
||||
failed := BuilderBuild{State: BuildFailed, By: "builder.novox.1", Step: "test", Error: "go test: 2 failed"}
|
||||
return map[string]BuilderBuild{
|
||||
"queued": {State: BuildQueued, Manifest: digestOf('4')},
|
||||
"building": {State: BuildBuilding, By: "builder.novox.1", Step: "fetch", Manifest: digestOf('4')},
|
||||
"done": doneBuild(),
|
||||
"failed": failed,
|
||||
}
|
||||
}
|
||||
|
||||
// 1358/AC1: the key is <module>.<hex>, parsing gives both back; a module with a dot, an empty one, and a fingerprint not
|
||||
// of the new form are refused, naming it.
|
||||
func TestBuilderBuildKey(t *testing.T) {
|
||||
fp := fp2of('a')
|
||||
key, err := BuilderBuildKey("mesh-mail", fp)
|
||||
if err != nil || key != "mesh-mail."+strings.Repeat("a", 64) {
|
||||
t.Fatalf("key = %q, %v", key, err)
|
||||
}
|
||||
module, back, err := ParseBuilderBuildKey(key)
|
||||
if err != nil || module != "mesh-mail" || back != fp {
|
||||
t.Errorf("parse = %q, %q, %v", module, back, err)
|
||||
}
|
||||
src1, _ := SourceFingerprint(SourceInputs{Module: "m", Tree: "t"})
|
||||
for module, fp := range map[string]string{"a.b": fp, "": fp, "mesh-mail": src1, "mail ": fp} {
|
||||
if _, err := BuilderBuildKey(module, fp); err == nil {
|
||||
t.Errorf("BuilderBuildKey(%q, %q) accepted", module, fp)
|
||||
}
|
||||
}
|
||||
if _, err := BuilderBuildKey("a.b", fp); err == nil || !strings.Contains(err.Error(), "a.b") {
|
||||
t.Errorf("a dotted module is not named: %v", err)
|
||||
}
|
||||
if _, err := BuilderBuildKey("mesh-mail", src1); err == nil || !strings.Contains(err.Error(), src1) {
|
||||
t.Errorf("a src1 fingerprint is not named: %v", err)
|
||||
}
|
||||
for _, bad := range []string{"", "mesh-mail", "mesh-mail.", "mesh-mail.abc", "mesh-mail." + strings.Repeat("A", 64), ".x", "mesh-mail.build.7", "a.b." + strings.Repeat("a", 64)} {
|
||||
if _, _, err := ParseBuilderBuildKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") {
|
||||
t.Errorf("ParseBuilderBuildKey(%q) = %v, want a refusal naming it", bad, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1358/AC2: a build value in each state encodes and decodes unchanged.
|
||||
func TestBuilderBuildRoundTrip(t *testing.T) {
|
||||
for name, b := range builderSamples() {
|
||||
data, err := EncodeBuilderBuild(b)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if !strings.HasPrefix(string(data), `{"v":1,`) {
|
||||
t.Errorf("%s at %s", name, data)
|
||||
}
|
||||
got, err := DecodeBuilderBuild(data)
|
||||
b.V = Version
|
||||
if err != nil || !SameBuilderBuild(got, b) {
|
||||
t.Errorf("%s: %+v %v", name, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1358/AC3: done without artifacts or without a manifest digest, failed without an error, an artifact without a digest
|
||||
// and an unknown state are refused.
|
||||
func TestBuilderBuildChecked(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
edit func(*BuilderBuild)
|
||||
words []string
|
||||
}{
|
||||
"done without artifacts": {func(b *BuilderBuild) { b.Artifacts = nil }, []string{"artifact"}},
|
||||
"done without a manifest": {func(b *BuilderBuild) { b.Manifest = "" }, []string{"manifest"}},
|
||||
"a malformed manifest": {func(b *BuilderBuild) { b.Manifest = "abc" }, []string{"abc"}},
|
||||
"an artifact without a digest": {func(b *BuilderBuild) { b.Artifacts[1].Digest = "" }, []string{"digest", "server"}},
|
||||
"an artifact without a name": {func(b *BuilderBuild) { b.Artifacts[0].Name = "" }, []string{"name"}},
|
||||
"an unknown state": {func(b *BuilderBuild) { b.State = "paused" }, []string{"paused"}},
|
||||
"a toolchain without a version": {func(b *BuilderBuild) { b.Toolchains["go"] = ToolchainUsed{Recipe: "3"} }, []string{"go", "version"}},
|
||||
"a toolchain without a recipe": {func(b *BuilderBuild) { b.Toolchains["go"] = ToolchainUsed{Version: "1.26.0"} }, []string{"go", "recipe"}},
|
||||
}
|
||||
for name, c := range cases {
|
||||
b := doneBuild()
|
||||
c.edit(&b)
|
||||
_, err := EncodeBuilderBuild(b)
|
||||
if err == nil {
|
||||
t.Errorf("%s accepted", name)
|
||||
continue
|
||||
}
|
||||
for _, w := range c.words {
|
||||
if !strings.Contains(err.Error(), w) {
|
||||
t.Errorf("%s: error %q does not name %q", name, err, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
f := builderSamples()["failed"]
|
||||
f.Error = ""
|
||||
if _, err := EncodeBuilderBuild(f); err == nil || !strings.Contains(err.Error(), "error") {
|
||||
t.Errorf("failed without an error: %v", err)
|
||||
}
|
||||
// a queued or building value may not already carry an outcome
|
||||
q := builderSamples()["queued"]
|
||||
q.Artifacts = doneBuild().Artifacts
|
||||
if _, err := EncodeBuilderBuild(q); err == nil {
|
||||
t.Error("a queued build with artifacts accepted")
|
||||
}
|
||||
// decode refuses what encode refuses
|
||||
if _, err := DecodeBuilderBuild([]byte(`{"v":1,"state":"done","manifest":"` + digestOf('4') + `"}`)); err == nil {
|
||||
t.Error("decoded a done build without artifacts")
|
||||
}
|
||||
}
|
||||
|
||||
// 1358/AC4: a done value names the toolchain version per language and the recipe version; two values that differ only
|
||||
// there are not the same.
|
||||
func TestBuilderBuildNamesItsToolchains(t *testing.T) {
|
||||
a, b, c := doneBuild(), doneBuild(), doneBuild()
|
||||
b.Toolchains["go"] = ToolchainUsed{Version: "1.26.1", Recipe: "3"}
|
||||
c.Toolchains["go"] = ToolchainUsed{Version: "1.26.0", Recipe: "4"}
|
||||
if SameBuilderBuild(a, b) || SameBuilderBuild(a, c) || SameBuilderBuild(b, c) {
|
||||
t.Error("values differing only in a toolchain's version or recipe are the same")
|
||||
}
|
||||
if !SameBuilderBuild(a, doneBuild()) {
|
||||
t.Error("equal values differ")
|
||||
}
|
||||
data, err := EncodeBuilderBuild(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := DecodeBuilderBuild(data)
|
||||
if err != nil || !reflect.DeepEqual(got.Toolchains, a.Toolchains) {
|
||||
t.Errorf("toolchains = %+v, %v", got.Toolchains, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1358/AC5: a value of another version, or none, is refused with ErrUnknownVersion.
|
||||
func TestBuilderBuildVersion(t *testing.T) {
|
||||
for _, in := range []string{`{"v":2,"state":"queued"}`, `{"state":"queued"}`} {
|
||||
if _, err := DecodeBuilderBuild([]byte(in)); !errors.Is(err, ErrUnknownVersion) {
|
||||
t.Errorf("DecodeBuilderBuild(%s) = %v", in, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1358/AC6: the builder's bucket and module do not collide with the controller's build layout (buildrecord.go), which
|
||||
// this change leaves as it was.
|
||||
func TestBuilderLayoutDoesNotCollideWithTheControllers(t *testing.T) {
|
||||
if BuilderBuildsBucket != "mesh-delivery-builder_builds" || BuilderBuildsHistory != 5 || BuilderModule != "mesh-delivery-builder" {
|
||||
t.Errorf("bucket %s history %d module %s", BuilderBuildsBucket, BuilderBuildsHistory, BuilderModule)
|
||||
}
|
||||
if BuilderBuildsBucket == BuildsBucket || BuilderModule == "mesh-controller" {
|
||||
t.Error("the builder's bucket is the controller's")
|
||||
}
|
||||
if BuildsBucket != "mesh-controller_builds" || BuildKey("m", "7") != "m.build.7" || NewestKey("m") != "m.newest" {
|
||||
t.Error("the controller's layout changed")
|
||||
}
|
||||
// the two key forms are never read as each other
|
||||
key, _ := BuilderBuildKey("m", fp2of('a'))
|
||||
if key == BuildKey("m", "a") || key == NewestKey("m") {
|
||||
t.Errorf("builder key %q equals a controller key", key)
|
||||
}
|
||||
if _, _, err := ParseBuilderBuildKey(BuildKey("m", digestOf('a'))); err == nil {
|
||||
t.Error("a controller build key parsed as the builder's")
|
||||
}
|
||||
if _, _, err := ParseBuilderBuildKey(NewestKey("m")); err == nil {
|
||||
t.Error("a controller newest key parsed as the builder's")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user