From 21cccc6a8db1b725b5b914ca754c8553254b2a97 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 01:46:16 +0000 Subject: [PATCH] Lay out the builder's build keys, builds.., beside the controller's (issue 1358) --- go/delivery/builder.go | 172 +++++++++++++++++++++++++++++++++ go/delivery/builder_test.go | 187 ++++++++++++++++++++++++++++++++++++ 2 files changed, 359 insertions(+) create mode 100644 go/delivery/builder.go create mode 100644 go/delivery/builder_test.go diff --git a/go/delivery/builder.go b/go/delivery/builder.go new file mode 100644 index 0000000..205e5d5 --- /dev/null +++ b/go/delivery/builder.go @@ -0,0 +1,172 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" +) + +// The builder's keys (novox/hq ADR 0429 clauses 5 and 16, ADR 0437 clause 5, ADR 0405 clauses 2 and 7, ADR 0453 +// clause 4): a build is `.` in the builder's bucket, one key per fingerprint, since equal +// fingerprints are equal content and the same source is never built twice. They are written only by the builder part +// from R4c and read by the checker, the assigner and the view, and from R4e by the controller. They are not the +// controller's build records (buildrecord.go: BuildsBucket, BuildKey), which stay as they are. +// +// **The rules its writer keeps.** A builder claims a fingerprint that has no build key by creating it queued or +// building, a create that fails when the key exists, and moves it by compare-and-set; a builder reclaims only a build +// whose `by` is offline in presence. A failed key is built again as a new revision of the same key (ADR 0429 clause +// 18). Only the builder's credential writes the bucket (clause 5); every other part reads it. +const ( + // BuilderModule is the module of the builder part. + BuilderModule = "mesh-delivery-builder" + // BuilderBuildsBucket is the builder's bucket of builds: `_`. + BuilderBuildsBucket = BuilderModule + "_builds" + // BuilderBuildsHistory is the revisions the bucket keeps per key. + BuilderBuildsHistory = 5 +) + +// The states of a build. +const ( + BuildQueued = "queued" + BuildBuilding = "building" + BuildDone = "done" + BuildFailed = "failed" +) + +// BuilderBuildKey is the key, in BuilderBuildsBucket, of a module's build at a fingerprint: `.`, the hex +// being the fingerprint's key part (FingerprintKeyPart), since a bus key may not hold ':'. A module holding a dot or a +// character a key refuses, and a fingerprint not of the new form, are refused, naming them. +func BuilderBuildKey(module, fingerprint string) (string, error) { + if err := checkName("module", module); err != nil { + return "", err + } + part, err := FingerprintKeyPart(fingerprint) + if err != nil { + return "", err + } + return module + "." + part, nil +} + +// ParseBuilderBuildKey is the inverse of BuilderBuildKey: the module and the fingerprint (`src2:` and the hex). +func ParseBuilderBuildKey(key string) (module, fingerprint string, err error) { + module, part, ok := strings.Cut(key, ".") + if !ok { + return "", "", fmt.Errorf("delivery: %q is not a builder build key, .", key) + } + if err := checkName("module", module); err != nil { + return "", "", fmt.Errorf("delivery: %q is not a builder build key: %w", key, err) + } + if fingerprint, err = ParseFingerprintKeyPart(part); err != nil { + return "", "", fmt.Errorf("delivery: %q is not a builder build key: %w", key, err) + } + return module, fingerprint, nil +} + +// BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest and, for an +// image, the reference it was pushed to. +type BuiltArtifact struct { + Name string `json:"name"` + Kind string `json:"kind,omitempty"` + Digest string `json:"digest"` + Reference string `json:"reference,omitempty"` +} + +// ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and +// the build module's recipe version. +type ToolchainUsed struct { + Version string `json:"version"` + Recipe string `json:"recipe"` +} + +// BuilderBuild is the value of a builder build key: its State (queued, building, done or failed), By the builder +// instance that holds it, Step what it is doing or did last, Manifest the digest of the module's manifest (the key of +// the indexer's manifests bucket), Artifacts when done, Toolchains by language when it built or is building (a done +// build names them, so two builders that differ are seen: ADR 0405 clause 2), and Error when failed. +type BuilderBuild struct { + V int `json:"v"` + State string `json:"state"` + By string `json:"by,omitempty"` + Step string `json:"step,omitempty"` + Manifest string `json:"manifest,omitempty"` + Artifacts []BuiltArtifact `json:"artifacts,omitempty"` + Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"` + Error string `json:"error,omitempty"` +} + +func (b BuilderBuild) check() error { + switch b.State { + case BuildQueued, BuildBuilding, BuildDone, BuildFailed: + default: + return fmt.Errorf("delivery: the build state %q is not queued, building, done or failed", b.State) + } + if b.Manifest != "" { + if _, err := ManifestsKey(b.Manifest); err != nil { + return err + } + } + for language, t := range b.Toolchains { + if t.Version == "" { + return fmt.Errorf("delivery: the toolchain of %q has no version", language) + } + if t.Recipe == "" { + return fmt.Errorf("delivery: the toolchain of %q has no recipe version", language) + } + } + for i, a := range b.Artifacts { + if a.Name == "" { + return fmt.Errorf("delivery: artifact %d of the build has no name", i) + } + if a.Digest == "" { + return fmt.Errorf("delivery: the artifact %q of the build has no digest", a.Name) + } + } + switch b.State { + case BuildQueued, BuildBuilding: + if len(b.Artifacts) > 0 || b.Error != "" { + return fmt.Errorf("delivery: a %s build carries no artifact and no error", b.State) + } + case BuildDone: + if len(b.Artifacts) == 0 { + return fmt.Errorf("delivery: a done build names no artifact") + } + if b.Manifest == "" { + return fmt.Errorf("delivery: a done build names no manifest digest") + } + if b.Error != "" { + return fmt.Errorf("delivery: a done build carries an error") + } + case BuildFailed: + if b.Error == "" { + return fmt.Errorf("delivery: a failed build says no error") + } + } + return nil +} + +// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without artifacts or a +// manifest digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a +// recipe version). +func EncodeBuilderBuild(b BuilderBuild) ([]byte, error) { + if err := b.check(); err != nil { + return nil, err + } + b.V = Version + return json.Marshal(b) +} + +// DecodeBuilderBuild reads a builder build value; it refuses an unknown version (ErrUnknownVersion) and what +// EncodeBuilderBuild refuses. +func DecodeBuilderBuild(data []byte) (BuilderBuild, error) { + var b BuilderBuild + if err := decode(data, &b); err != nil { + return BuilderBuild{}, err + } + if err := b.check(); err != nil { + return BuilderBuild{}, err + } + return b, nil +} + +// SameBuilderBuild says whether two builds are the same value, so the builder writes only when they differ. +func SameBuilderBuild(a, b BuilderBuild) bool { return reflect.DeepEqual(a, b) } diff --git a/go/delivery/builder_test.go b/go/delivery/builder_test.go new file mode 100644 index 0000000..2731819 --- /dev/null +++ b/go/delivery/builder_test.go @@ -0,0 +1,187 @@ +package delivery + +import ( + "errors" + "reflect" + "strings" + "testing" +) + +func doneBuild() BuilderBuild { + return BuilderBuild{ + State: BuildDone, + By: "builder.novox.1", + Step: "pushed", + Manifest: digestOf('4'), + Artifacts: []BuiltArtifact{ + {Name: "code", Kind: "bundle", Digest: "sha256:" + digestOf('5')}, + {Name: "server", Kind: "image", Digest: "sha256:" + digestOf('6'), Reference: "registry.example/mail@sha256:" + digestOf('6')}, + }, + Toolchains: map[string]ToolchainUsed{ + "go": {Version: "1.26.0", Recipe: "3"}, + "typescript": {Version: "5.9.2", Recipe: "1"}, + }, + } +} + +func builderSamples() map[string]BuilderBuild { + failed := BuilderBuild{State: BuildFailed, By: "builder.novox.1", Step: "test", Error: "go test: 2 failed"} + return map[string]BuilderBuild{ + "queued": {State: BuildQueued, Manifest: digestOf('4')}, + "building": {State: BuildBuilding, By: "builder.novox.1", Step: "fetch", Manifest: digestOf('4')}, + "done": doneBuild(), + "failed": failed, + } +} + +// 1358/AC1: the key is ., parsing gives both back; a module with a dot, an empty one, and a fingerprint not +// of the new form are refused, naming it. +func TestBuilderBuildKey(t *testing.T) { + fp := fp2of('a') + key, err := BuilderBuildKey("mesh-mail", fp) + if err != nil || key != "mesh-mail."+strings.Repeat("a", 64) { + t.Fatalf("key = %q, %v", key, err) + } + module, back, err := ParseBuilderBuildKey(key) + if err != nil || module != "mesh-mail" || back != fp { + t.Errorf("parse = %q, %q, %v", module, back, err) + } + src1, _ := SourceFingerprint(SourceInputs{Module: "m", Tree: "t"}) + for module, fp := range map[string]string{"a.b": fp, "": fp, "mesh-mail": src1, "mail ": fp} { + if _, err := BuilderBuildKey(module, fp); err == nil { + t.Errorf("BuilderBuildKey(%q, %q) accepted", module, fp) + } + } + if _, err := BuilderBuildKey("a.b", fp); err == nil || !strings.Contains(err.Error(), "a.b") { + t.Errorf("a dotted module is not named: %v", err) + } + if _, err := BuilderBuildKey("mesh-mail", src1); err == nil || !strings.Contains(err.Error(), src1) { + t.Errorf("a src1 fingerprint is not named: %v", err) + } + for _, bad := range []string{"", "mesh-mail", "mesh-mail.", "mesh-mail.abc", "mesh-mail." + strings.Repeat("A", 64), ".x", "mesh-mail.build.7", "a.b." + strings.Repeat("a", 64)} { + if _, _, err := ParseBuilderBuildKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") { + t.Errorf("ParseBuilderBuildKey(%q) = %v, want a refusal naming it", bad, err) + } + } +} + +// 1358/AC2: a build value in each state encodes and decodes unchanged. +func TestBuilderBuildRoundTrip(t *testing.T) { + for name, b := range builderSamples() { + data, err := EncodeBuilderBuild(b) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if !strings.HasPrefix(string(data), `{"v":1,`) { + t.Errorf("%s at %s", name, data) + } + got, err := DecodeBuilderBuild(data) + b.V = Version + if err != nil || !SameBuilderBuild(got, b) { + t.Errorf("%s: %+v %v", name, got, err) + } + } +} + +// 1358/AC3: done without artifacts or without a manifest digest, failed without an error, an artifact without a digest +// and an unknown state are refused. +func TestBuilderBuildChecked(t *testing.T) { + cases := map[string]struct { + edit func(*BuilderBuild) + words []string + }{ + "done without artifacts": {func(b *BuilderBuild) { b.Artifacts = nil }, []string{"artifact"}}, + "done without a manifest": {func(b *BuilderBuild) { b.Manifest = "" }, []string{"manifest"}}, + "a malformed manifest": {func(b *BuilderBuild) { b.Manifest = "abc" }, []string{"abc"}}, + "an artifact without a digest": {func(b *BuilderBuild) { b.Artifacts[1].Digest = "" }, []string{"digest", "server"}}, + "an artifact without a name": {func(b *BuilderBuild) { b.Artifacts[0].Name = "" }, []string{"name"}}, + "an unknown state": {func(b *BuilderBuild) { b.State = "paused" }, []string{"paused"}}, + "a toolchain without a version": {func(b *BuilderBuild) { b.Toolchains["go"] = ToolchainUsed{Recipe: "3"} }, []string{"go", "version"}}, + "a toolchain without a recipe": {func(b *BuilderBuild) { b.Toolchains["go"] = ToolchainUsed{Version: "1.26.0"} }, []string{"go", "recipe"}}, + } + for name, c := range cases { + b := doneBuild() + c.edit(&b) + _, err := EncodeBuilderBuild(b) + if err == nil { + t.Errorf("%s accepted", name) + continue + } + for _, w := range c.words { + if !strings.Contains(err.Error(), w) { + t.Errorf("%s: error %q does not name %q", name, err, w) + } + } + } + f := builderSamples()["failed"] + f.Error = "" + if _, err := EncodeBuilderBuild(f); err == nil || !strings.Contains(err.Error(), "error") { + t.Errorf("failed without an error: %v", err) + } + // a queued or building value may not already carry an outcome + q := builderSamples()["queued"] + q.Artifacts = doneBuild().Artifacts + if _, err := EncodeBuilderBuild(q); err == nil { + t.Error("a queued build with artifacts accepted") + } + // decode refuses what encode refuses + if _, err := DecodeBuilderBuild([]byte(`{"v":1,"state":"done","manifest":"` + digestOf('4') + `"}`)); err == nil { + t.Error("decoded a done build without artifacts") + } +} + +// 1358/AC4: a done value names the toolchain version per language and the recipe version; two values that differ only +// there are not the same. +func TestBuilderBuildNamesItsToolchains(t *testing.T) { + a, b, c := doneBuild(), doneBuild(), doneBuild() + b.Toolchains["go"] = ToolchainUsed{Version: "1.26.1", Recipe: "3"} + c.Toolchains["go"] = ToolchainUsed{Version: "1.26.0", Recipe: "4"} + if SameBuilderBuild(a, b) || SameBuilderBuild(a, c) || SameBuilderBuild(b, c) { + t.Error("values differing only in a toolchain's version or recipe are the same") + } + if !SameBuilderBuild(a, doneBuild()) { + t.Error("equal values differ") + } + data, err := EncodeBuilderBuild(a) + if err != nil { + t.Fatal(err) + } + got, err := DecodeBuilderBuild(data) + if err != nil || !reflect.DeepEqual(got.Toolchains, a.Toolchains) { + t.Errorf("toolchains = %+v, %v", got.Toolchains, err) + } +} + +// 1358/AC5: a value of another version, or none, is refused with ErrUnknownVersion. +func TestBuilderBuildVersion(t *testing.T) { + for _, in := range []string{`{"v":2,"state":"queued"}`, `{"state":"queued"}`} { + if _, err := DecodeBuilderBuild([]byte(in)); !errors.Is(err, ErrUnknownVersion) { + t.Errorf("DecodeBuilderBuild(%s) = %v", in, err) + } + } +} + +// 1358/AC6: the builder's bucket and module do not collide with the controller's build layout (buildrecord.go), which +// this change leaves as it was. +func TestBuilderLayoutDoesNotCollideWithTheControllers(t *testing.T) { + if BuilderBuildsBucket != "mesh-delivery-builder_builds" || BuilderBuildsHistory != 5 || BuilderModule != "mesh-delivery-builder" { + t.Errorf("bucket %s history %d module %s", BuilderBuildsBucket, BuilderBuildsHistory, BuilderModule) + } + if BuilderBuildsBucket == BuildsBucket || BuilderModule == "mesh-controller" { + t.Error("the builder's bucket is the controller's") + } + if BuildsBucket != "mesh-controller_builds" || BuildKey("m", "7") != "m.build.7" || NewestKey("m") != "m.newest" { + t.Error("the controller's layout changed") + } + // the two key forms are never read as each other + key, _ := BuilderBuildKey("m", fp2of('a')) + if key == BuildKey("m", "a") || key == NewestKey("m") { + t.Errorf("builder key %q equals a controller key", key) + } + if _, _, err := ParseBuilderBuildKey(BuildKey("m", digestOf('a'))); err == nil { + t.Error("a controller build key parsed as the builder's") + } + if _, _, err := ParseBuilderBuildKey(NewestKey("m")); err == nil { + t.Error("a controller newest key parsed as the builder's") + } +}