diff --git a/modules/mesh-delivery-builder/builds/builds.go b/modules/mesh-delivery-builder/builds/builds.go new file mode 100644 index 00000000..04bd7751 --- /dev/null +++ b/modules/mesh-delivery-builder/builds/builds.go @@ -0,0 +1,372 @@ +// Package builds holds what the builder module owns and no seat carries (novox/hq ADR 0634 §1): the layout of its state +// `builds` (a build per module and fingerprint) and the rules its writer keeps, the question a registry seat's holder asks +// it before it stores a publish, its event version.numbered and the entries of its build log. Other modules import it from +// the builder by its published version (ADR 0573 §1); the builder's manifest declares it as a package artifact. +package builds + +import ( + "encoding/json" + "fmt" + "reflect" + "slices" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-delivery/sdk/fingerprint" + "git.novox.be/novox/mesh-delivery/sdk/kept" + "git.novox.be/novox/mesh-delivery/sdk/reference" + "git.novox.be/novox/mesh-delivery/sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// seats are the names of the registry seats a reference names first, from their contracts. +var seats = reference.Seats{Image: containerimageregistry.Seat, Object: ociartifactregistry.Seat} + +// askLanguages are the languages a build seat is named after (node-build-), which a building key's Asks are keyed by. +var askLanguages = []string{"go", "node", "image"} + +func checkAskLanguage(language string) error { + if !slices.Contains(askLanguages, language) { + return fmt.Errorf("builds: the ask language %q is not go, node or image", language) + } + return nil +} + +// The builder's keys (novox/hq ADR 0429 clauses 5 and 16, ADR 0437 clause 5, ADR 0405 clauses 2 and 7, ADR 0453 +// clause 4): a build is `.` in the builder's bucket, one key per fingerprint, since equal +// fingerprints are equal content and the same source is never built twice. They are written only by the builder part +// from R4c and read by the checker, the assigner and the view. They are not the ended controller's build records +// (buildrecord.go: BuildKey). +// +// **The rules its writer keeps.** A builder claims a fingerprint that has no build key by creating it queued or +// building, a create that fails when the key exists, and moves it by compare-and-set; a builder reclaims only a build +// whose `by` is offline in presence. A failed key is built again as a new revision of the same key (ADR 0429 clause +// 18). Only the builder's credential writes the bucket (clause 5); every other part reads it. +const ( + // BuilderModule is the module of the builder part. + BuilderModule = "mesh-delivery-builder" + // BuilderBuildsBucket is the builder's bucket of builds: `_`. + BuilderBuildsBucket = BuilderModule + "_builds" + // BuilderBuildsHistory is the revisions the bucket keeps per key. + BuilderBuildsHistory = 5 +) + +// The states of a build. +const ( + BuildQueued = "queued" + BuildBuilding = "building" + BuildDone = "done" + BuildFailed = "failed" +) + +// BuilderBuildKey is the key, in BuilderBuildsBucket, of a module's build at a fingerprint: `.`, the hex +// being the fingerprint's key part (fingerprint.KeyPart), since a bus key may not hold ':'. A module holding a dot or a +// character a key refuses, and a fingerprint not of the new form, are refused, naming them. +func BuilderBuildKey(module, fp string) (string, error) { + if err := kept.CheckName("module", module); err != nil { + return "", err + } + part, err := fingerprint.KeyPart(fp) + if err != nil { + return "", err + } + return module + "." + part, nil +} + +// ParseBuilderBuildKey is the inverse of BuilderBuildKey: the module and the fingerprint (`src2:` and the hex). +func ParseBuilderBuildKey(key string) (module, fp string, err error) { + module, part, ok := strings.Cut(key, ".") + if !ok { + return "", "", fmt.Errorf("builds: %q is not a builder build key, .", key) + } + if err := kept.CheckName("module", module); err != nil { + return "", "", fmt.Errorf("builds: %q is not a builder build key: %w", key, err) + } + if fp, err = fingerprint.ParseKeyPart(part); err != nil { + return "", "", fmt.Errorf("builds: %q is not a builder build key: %w", key, err) + } + return module, fp, nil +} + +// BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest (`sha256:` and +// 64 hex digits) and the reference that names where it is (novox/hq ADR 0498 clauses 1, 5 and 6; the lead's decision +// L1 of R4e), by kind: +// +// bundle, archive artifact-store:////blobs/ the blob pushed to the artifact store +// image, upstream //@ the image registry's reference, which +// artifact-store:///@ catalogue.Recorded keeps without its address +// package @ the version published from the trunk +// +// From the registry milestone a done key names the reference its seat answered (novox/hq ADR 0605 §9, design 51 *The +// done key names the reference the seat answered*): `//@`, the registry seat it +// came through (oci-artifact-registry for a bundle or an archive, container-image-registry for an image or an upstream +// copy), and `@` for a package as before. A key written in the older form keeps it; this reads both, and +// no build is built again for its form (seats.InArtifactStore reads both the same way). +// +// is Name and is Digest. A reference is empty for a candidate's image or package, which is never +// pushed or published (ADR 0498 clause 6), and in a key done before R4e; the controller delivers only from a key whose +// every artifact names one (Referenced, clause 8). A kind left empty (a key written before kinds) is not checked. +type BuiltArtifact struct { + Name string `json:"name"` + Kind string `json:"kind,omitempty"` + Digest string `json:"digest"` + Reference string `json:"reference,omitempty"` + // References are every form this artifact is named in, Reference among them (novox/hq ADR 0618, ADR 0624): a build + // keeps each form that a counted node-engine states, away nodes included, and Reference stays the form every one + // of them reads. A reader that does not know the field reads Reference alone. + References []string `json:"references,omitempty"` +} + +const artifactStore = "artifact-store://" + +// referenceProblem says what is wrong with a's reference for its kind, or "". +func (a BuiltArtifact) referenceProblem() string { + r := a.Reference + if r == "" || a.Kind == "" { + return "" + } + // An upstream copy may be named by the holder's own naming, upstream//@, in either form: the + // holder names it, not the module that asked for it (reference.InUpstream). + if path, kept := seats.InArtifactStore(r); kept && a.Kind == "upstream" { + if _, image, ok := reference.InUpstream(path); ok { + if _, digest, _ := strings.Cut(image, "@"); digest != a.Digest { + return "does not name its digest " + a.Digest + } + return "" + } + } + if seat, path, ok := seats.InRegistrySeat(r); ok { + want := "container-image-registry" + if a.Kind == "bundle" || a.Kind == "archive" { + want = "oci-artifact-registry" + } + module, rest, _ := strings.Cut(path, "/") + if a.Kind == "package" || seat != want || module == "" || rest != a.Name+"@"+a.Digest { + return "is not " + want + "//" + a.Name + "@" + a.Digest + } + return "" + } + switch a.Kind { + case "bundle", "archive": + path, ok := strings.CutPrefix(r, artifactStore) + module, rest, _ := strings.Cut(path, "/") + if !ok || module == "" || rest != a.Name+"/blobs/"+a.Digest { + return "is not artifact-store:///" + a.Name + "/blobs/" + a.Digest + } + case "image", "upstream": + path, digest, ok := strings.Cut(r, "@") + if !ok || digest != a.Digest { + return "does not name its digest " + a.Digest + } + var host, module, name string + if rest, kept := strings.CutPrefix(path, artifactStore); kept { + module, name, _ = strings.Cut(rest, "/") + host = "kept" + } else { + parts := strings.Split(path, "/") + if len(parts) == 3 { + host, module, name = parts[0], parts[1], parts[2] + } + } + if host == "" || !(host == "kept" || host == "localhost" || strings.ContainsAny(host, ".:")) || module == "" || name != a.Name { + return "is not //" + a.Name + "@" + a.Digest + } + case "package": + at := strings.LastIndex(r, "@") + if at <= 0 || at == len(r)-1 || r[:at] == "@" { + return "is not @" + } + } + return "" +} + +// Referenced says whether b is done and every artifact names where it is, which the controller delivers from (novox/hq +// ADR 0498 clause 8), and names the artifacts that name no reference. A done key with no artifact (a module that builds +// none, issue 1613) is referenced. +func (b BuilderBuild) Referenced() (ok bool, missing []string) { + for _, a := range b.Artifacts { + if a.Reference == "" { + missing = append(missing, a.Name) + } + } + return b.State == BuildDone && len(missing) == 0, missing +} + +// The prefixes of a failed key's error that a build module answers (novox/hq ADR 0498 clauses 2, 4 and 5; ADR 0478 +// clause 2), each followed by the artifact and the words of what refused it. +const ( + // NotStored: the push of a bundle, an archive or an upstream copy to the artifact store failed; built again. + NotStored = "not stored: " + // NotPinned: an upstream artifact names no digest. + NotPinned = "not pinned: " + // VersionTaken: the package registry holds the package's version with another packed file. + VersionTaken = "version taken: " + // RefusedKind: no build module builds this kind of artifact in its language. + RefusedKind = "refused kind: " + // WaitsFor: a fetch found the module's lock naming a package version the registry does not hold yet; the builder + // writes the key queued with Waits (novox/hq ADR 0500 clause 6), and it is built once the package is published. + WaitsFor = "waits for: " +) + +// FailedUntilChanged says whether a failed key's error is one that is not built again until the module's fingerprint +// changes (not pinned, version taken, refused kind: the same source fails the same way); every other failed key, a +// push that was not stored included, is built again as a new revision (ADR 0429 clause 18). +func FailedUntilChanged(err string) bool { + return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind) +} + +// ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and +// the build module's recipe version. +type ToolchainUsed struct { + Version string `json:"version"` + Recipe string `json:"recipe"` +} + +// BuilderBuild is the value of a builder build key: its State (queued, building, done or failed), By the builder +// instance that holds it, Step what it is doing or did last, Manifest the digest of the module's manifest (the key of +// the indexer's manifests bucket), Artifacts when done, Toolchains by language when it built or is building (a done +// build names them, so two builders that differ are seen: ADR 0405 clause 2), Error when failed, and Waits when queued for a +// package version. +type BuilderBuild struct { + V int `json:"v"` + State string `json:"state"` + By string `json:"by,omitempty"` + Step string `json:"step,omitempty"` + Manifest string `json:"manifest,omitempty"` + Artifacts []BuiltArtifact `json:"artifacts,omitempty"` + Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"` + Error string `json:"error,omitempty"` + // Waits is, on a queued key only, the package version `@` the module's lock names and the registry + // does not yet hold: the key is not claimed until that package's trunk key is done and published (novox/hq ADR 0500 + // clause 6). A reader that does not know the field reads the key as queued. + Waits string `json:"waits,omitempty"` + // Asks are, on a building key, the asks the builder put for it by build seat language (AskGo, AskNode, AskImage), each + // its id (novox/hq ADR 0525 §1; issue 1887 A4b): a builder that takes the key over at its start waits for those asks' + // steps and never puts them again. A reader that does not know the field reads the key as before. + Asks map[string]string `json:"asks,omitempty"` + // Imports are, on a done key, the mesh packages the built programs import, as the build module answered them + // (BuildAnswer.Imports; novox/hq ADR 0593 §2): the builder carries them onto the build's end (BuildLogEntry.Imports). + Imports []string `json:"imports,omitempty"` + // Needs are, on a done key only, the reader capabilities (`form:`) the build's reference forms need of a + // node-engine, beside the ones its manifest needs (novox/hq ADR 0618 §1). A reader that does not know the field + // reads the build as needing none. + Needs []string `json:"needs,omitempty"` +} + +func (b BuilderBuild) check() error { + if err := trunk.CheckImports(b.Imports); err != nil { + return err + } + switch b.State { + case BuildQueued, BuildBuilding, BuildDone, BuildFailed: + default: + return fmt.Errorf("builds: the build state %q is not queued, building, done or failed", b.State) + } + if b.Manifest != "" { + if !kept.IsHex64(b.Manifest) { + return fmt.Errorf("builds: %q is not a manifest digest, 64 lowercase hex digits", b.Manifest) + } + } + for language, t := range b.Toolchains { + if t.Version == "" { + return fmt.Errorf("builds: the toolchain of %q has no version", language) + } + if t.Recipe == "" { + return fmt.Errorf("builds: the toolchain of %q has no recipe version", language) + } + } + for i, a := range b.Artifacts { + if a.Name == "" { + return fmt.Errorf("builds: artifact %d of the build has no name", i) + } + if a.Digest == "" { + return fmt.Errorf("builds: the artifact %q of the build has no digest", a.Name) + } + if p := a.referenceProblem(); p != "" { + return fmt.Errorf("builds: the reference %q of the %s %q %s", a.Reference, a.Kind, a.Name, p) + } + if len(a.References) > 0 && !slices.Contains(a.References, a.Reference) { + return fmt.Errorf("builds: the references of the %s %q do not name its reference %q", a.Kind, a.Name, a.Reference) + } + for _, r := range a.References { + if p := (BuiltArtifact{Name: a.Name, Kind: a.Kind, Digest: a.Digest, Reference: r}).referenceProblem(); p != "" { + return fmt.Errorf("builds: the reference %q of the %s %q %s", r, a.Kind, a.Name, p) + } + } + } + for _, n := range b.Needs { + if !strings.HasPrefix(n, "form:") || len(n) == len("form:") { + return fmt.Errorf("builds: the need %q of the build is not form:", n) + } + } + if len(b.Needs) > 0 && b.State != BuildDone { + return fmt.Errorf("builds: a %s build names needs; only a done key does", b.State) + } + for language, id := range b.Asks { + if b.State != BuildBuilding { + return fmt.Errorf("builds: a %s build names the ask %s; only a building key does", b.State, id) + } + if err := checkAskLanguage(language); err != nil { + return err + } + if err := buildstep.CheckAskID(id); err != nil { + return err + } + } + if b.Waits != "" { + if b.State != BuildQueued { + return fmt.Errorf("builds: a %s build waits for %q; only a queued key waits for a package", b.State, b.Waits) + } + if p := (BuiltArtifact{Kind: "package", Reference: b.Waits}).referenceProblem(); p != "" { + return fmt.Errorf("builds: the package a key waits for, %q, %s", b.Waits, p) + } + } + switch b.State { + case BuildQueued, BuildBuilding: + if len(b.Artifacts) > 0 || b.Error != "" { + return fmt.Errorf("builds: a %s build carries no artifact and no error", b.State) + } + case BuildDone: + // A module that builds no artifact gets a done key too, and is delivered from it (issue 1613). + if b.Manifest == "" { + return fmt.Errorf("builds: a done build names no manifest digest") + } + if b.Error != "" { + return fmt.Errorf("builds: a done build carries an error") + } + case BuildFailed: + if b.Error == "" { + return fmt.Errorf("builds: a failed build says no error") + } + } + return nil +} + +// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without a manifest +// digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a +// recipe version). +func EncodeBuilderBuild(b BuilderBuild) ([]byte, error) { + if err := b.check(); err != nil { + return nil, err + } + b.V = keys.Version + return json.Marshal(b) +} + +// DecodeBuilderBuild reads a builder build value; it refuses an unknown version (kept.ErrUnknownVersion) and what +// EncodeBuilderBuild refuses. +func DecodeBuilderBuild(data []byte) (BuilderBuild, error) { + var b BuilderBuild + if err := kept.Decode(data, &b); err != nil { + return BuilderBuild{}, err + } + if err := b.check(); err != nil { + return BuilderBuild{}, err + } + return b, nil +} + +// SameBuilderBuild says whether two builds are the same value, so the builder writes only when they differ. +func SameBuilderBuild(a, b BuilderBuild) bool { return reflect.DeepEqual(a, b) } diff --git a/modules/mesh-delivery-builder/builds/builds_test.go b/modules/mesh-delivery-builder/builds/builds_test.go new file mode 100644 index 00000000..e49c966e --- /dev/null +++ b/modules/mesh-delivery-builder/builds/builds_test.go @@ -0,0 +1,224 @@ +package builds + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/sdk/buildrecord" + "git.novox.be/novox/mesh-delivery/sdk/kept" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +func digestOf(c byte) string { return strings.Repeat(string(c), 64) } + +func fp2of(c byte) string { return "src2:" + digestOf(c) } + +func doneBuild() BuilderBuild { + return BuilderBuild{ + State: BuildDone, + By: "builder.novox.1", + Step: "pushed", + Manifest: digestOf('4'), + Artifacts: []BuiltArtifact{ + {Name: "code", Kind: "bundle", Digest: "sha256:" + digestOf('5')}, + {Name: "server", Kind: "image", Digest: "sha256:" + digestOf('6'), Reference: "registry.example/mail/server@sha256:" + digestOf('6')}, + }, + Toolchains: map[string]ToolchainUsed{ + "go": {Version: "1.26.0", Recipe: "3"}, + "typescript": {Version: "5.9.2", Recipe: "1"}, + }, + } +} + +func builderSamples() map[string]BuilderBuild { + failed := BuilderBuild{State: BuildFailed, By: "builder.novox.1", Step: "test", Error: "go test: 2 failed"} + both := doneBuild() + both.Needs = []string{"form:artifact-registry"} + both.Artifacts[1].References = []string{both.Artifacts[1].Reference, "artifact-store://mail/server@" + both.Artifacts[1].Digest} + return map[string]BuilderBuild{ + "done with needs and both forms": both, + "queued": {State: BuildQueued, Manifest: digestOf('4')}, + "building": {State: BuildBuilding, By: "builder.novox.1", Step: "fetch", Manifest: digestOf('4')}, + "done": doneBuild(), + "failed": failed, + } +} + +// 1358/AC1: the key is ., parsing gives both back; a module with a dot, an empty one, and a fingerprint not +// of the new form are refused, naming it. +func TestBuilderBuildKey(t *testing.T) { + fp := fp2of('a') + key, err := BuilderBuildKey("mesh-mail", fp) + if err != nil || key != "mesh-mail."+strings.Repeat("a", 64) { + t.Fatalf("key = %q, %v", key, err) + } + module, back, err := ParseBuilderBuildKey(key) + if err != nil || module != "mesh-mail" || back != fp { + t.Errorf("parse = %q, %q, %v", module, back, err) + } + src1 := "src1:" + strings.Repeat("1", 64) + for module, fp := range map[string]string{"a.b": fp, "": fp, "mesh-mail": src1, "mail ": fp} { + if _, err := BuilderBuildKey(module, fp); err == nil { + t.Errorf("BuilderBuildKey(%q, %q) accepted", module, fp) + } + } + if _, err := BuilderBuildKey("a.b", fp); err == nil || !strings.Contains(err.Error(), "a.b") { + t.Errorf("a dotted module is not named: %v", err) + } + if _, err := BuilderBuildKey("mesh-mail", src1); err == nil || !strings.Contains(err.Error(), src1) { + t.Errorf("a src1 fingerprint is not named: %v", err) + } + for _, bad := range []string{"", "mesh-mail", "mesh-mail.", "mesh-mail.abc", "mesh-mail." + strings.Repeat("A", 64), ".x", "mesh-mail.build.7", "a.b." + strings.Repeat("a", 64)} { + if _, _, err := ParseBuilderBuildKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") { + t.Errorf("ParseBuilderBuildKey(%q) = %v, want a refusal naming it", bad, err) + } + } +} + +// 1358/AC2: a build value in each state encodes and decodes unchanged. +func TestBuilderBuildRoundTrip(t *testing.T) { + for name, b := range builderSamples() { + data, err := EncodeBuilderBuild(b) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if !strings.HasPrefix(string(data), `{"v":1,`) { + t.Errorf("%s at %s", name, data) + } + got, err := DecodeBuilderBuild(data) + b.V = keys.Version + if err != nil || !SameBuilderBuild(got, b) { + t.Errorf("%s: %+v %v", name, got, err) + } + } +} + +// 1358/AC3: done without a manifest digest, failed without an error, an artifact without a digest +// and an unknown state are refused. +func TestBuilderBuildChecked(t *testing.T) { + cases := map[string]struct { + edit func(*BuilderBuild) + words []string + }{ + "done without a manifest": {func(b *BuilderBuild) { b.Manifest = "" }, []string{"manifest"}}, + "a malformed manifest": {func(b *BuilderBuild) { b.Manifest = "abc" }, []string{"abc"}}, + "an artifact without a digest": {func(b *BuilderBuild) { b.Artifacts[1].Digest = "" }, []string{"digest", "server"}}, + "an artifact without a name": {func(b *BuilderBuild) { b.Artifacts[0].Name = "" }, []string{"name"}}, + "a need that is not a form": {func(b *BuilderBuild) { b.Needs = []string{"word:x"} }, []string{"word:x", "form:"}}, + "a reference not among the references": {func(b *BuilderBuild) { + b.Artifacts[1].References = []string{"artifact-store://mail/server@" + b.Artifacts[1].Digest} + }, []string{"server", "references"}}, + "a reference of another artifact": {func(b *BuilderBuild) { + b.Artifacts[1].References = []string{b.Artifacts[1].Reference, "registry.example/mail/other@" + b.Artifacts[1].Digest} + }, []string{"server", "other"}}, + "an unknown state": {func(b *BuilderBuild) { b.State = "paused" }, []string{"paused"}}, + "a toolchain without a version": {func(b *BuilderBuild) { b.Toolchains["go"] = ToolchainUsed{Recipe: "3"} }, []string{"go", "version"}}, + "a toolchain without a recipe": {func(b *BuilderBuild) { b.Toolchains["go"] = ToolchainUsed{Version: "1.26.0"} }, []string{"go", "recipe"}}, + } + for name, c := range cases { + b := doneBuild() + c.edit(&b) + _, err := EncodeBuilderBuild(b) + if err == nil { + t.Errorf("%s accepted", name) + continue + } + for _, w := range c.words { + if !strings.Contains(err.Error(), w) { + t.Errorf("%s: error %q does not name %q", name, err, w) + } + } + } + f := builderSamples()["failed"] + f.Error = "" + if _, err := EncodeBuilderBuild(f); err == nil || !strings.Contains(err.Error(), "error") { + t.Errorf("failed without an error: %v", err) + } + // a queued or building value may not already carry an outcome + q := builderSamples()["queued"] + q.Needs = []string{"form:x"} + if _, err := EncodeBuilderBuild(q); err == nil { + t.Error("a queued build with needs accepted") + } + q.Needs = nil + q.Artifacts = doneBuild().Artifacts + if _, err := EncodeBuilderBuild(q); err == nil { + t.Error("a queued build with artifacts accepted") + } + // decode refuses what encode refuses + if _, err := DecodeBuilderBuild([]byte(`{"v":1,"state":"done"}`)); err == nil { + t.Error("decoded a done build without a manifest digest") + } +} + +// 1358/AC4: a done value names the toolchain version per language and the recipe version; two values that differ only +// there are not the same. +func TestBuilderBuildNamesItsToolchains(t *testing.T) { + a, b, c := doneBuild(), doneBuild(), doneBuild() + b.Toolchains["go"] = ToolchainUsed{Version: "1.26.1", Recipe: "3"} + c.Toolchains["go"] = ToolchainUsed{Version: "1.26.0", Recipe: "4"} + if SameBuilderBuild(a, b) || SameBuilderBuild(a, c) || SameBuilderBuild(b, c) { + t.Error("values differing only in a toolchain's version or recipe are the same") + } + if !SameBuilderBuild(a, doneBuild()) { + t.Error("equal values differ") + } + data, err := EncodeBuilderBuild(a) + if err != nil { + t.Fatal(err) + } + got, err := DecodeBuilderBuild(data) + if err != nil || !reflect.DeepEqual(got.Toolchains, a.Toolchains) { + t.Errorf("toolchains = %+v, %v", got.Toolchains, err) + } +} + +// 1358/AC5: a value of another version, or none, is refused with kept.ErrUnknownVersion. +func TestBuilderBuildVersion(t *testing.T) { + for _, in := range []string{`{"v":2,"state":"queued"}`, `{"state":"queued"}`} { + if _, err := DecodeBuilderBuild([]byte(in)); !errors.Is(err, kept.ErrUnknownVersion) { + t.Errorf("DecodeBuilderBuild(%s) = %v", in, err) + } + } + // A holder's question before it stores (ADR 0605 §4): asked at the builder's address with name, version and digest; + // an answer without recorded is an error, never a yes. + for answer, want := range map[string]string{`{"recorded":true}`: "true ", `{"recorded":false,"why":"other digest"}`: "false other digest", + `{"name":"x"}`: "error", `not json`: "error"} { + b := BuilderVersions{Ask: func(addr string, body any) (json.RawMessage, error) { + if a := body.(map[string]any); addr != BuilderVersionsAddress || a["name"] != "n" || a["version"] != "0.1.2" || a["digest"] != "sha256:d" { + t.Errorf("asked %s %v", addr, body) + } + return json.RawMessage(answer), nil + }} + ok, why, err := b.Recorded("n", "0.1.2", "sha256:d") + if got := fmt.Sprintf("%v %s", ok, why); err != nil && want != "error" || err == nil && got != want { + t.Errorf("answer %s: %s, %v; want %s", answer, got, err, want) + } + } +} + +// 1358/AC6: the builder's bucket and module do not collide with the controller's build layout (buildrecord.go), which +// this change leaves as it was. +func TestBuilderLayoutDoesNotCollideWithTheControllers(t *testing.T) { + if BuilderBuildsBucket != "mesh-delivery-builder_builds" || BuilderBuildsHistory != 5 || BuilderModule != "mesh-delivery-builder" { + t.Errorf("bucket %s history %d module %s", BuilderBuildsBucket, BuilderBuildsHistory, BuilderModule) + } + if buildrecord.BuildKey("m", "7") != "m.build.7" || buildrecord.NewestKey("m") != "m.newest" { + t.Error("the controller's layout changed") + } + // the two key forms are never read as each other + key, _ := BuilderBuildKey("m", fp2of('a')) + if key == buildrecord.BuildKey("m", "a") || key == buildrecord.NewestKey("m") { + t.Errorf("builder key %q equals a controller key", key) + } + if _, _, err := ParseBuilderBuildKey(buildrecord.BuildKey("m", digestOf('a'))); err == nil { + t.Error("a controller build key parsed as the builder's") + } + if _, _, err := ParseBuilderBuildKey(buildrecord.NewestKey("m")); err == nil { + t.Error("a controller newest key parsed as the builder's") + } +} diff --git a/modules/mesh-delivery-builder/builds/go.mod b/modules/mesh-delivery-builder/builds/go.mod new file mode 100644 index 00000000..8fc38072 --- /dev/null +++ b/modules/mesh-delivery-builder/builds/go.mod @@ -0,0 +1,10 @@ +module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0 + git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry v0.0.0 + git.novox.be/novox/mesh-delivery/sdk v0.0.0 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174359-05b3e2e1b94f +) diff --git a/modules/mesh-delivery-builder/builds/log.go b/modules/mesh-delivery-builder/builds/log.go new file mode 100644 index 00000000..d7344eee --- /dev/null +++ b/modules/mesh-delivery-builder/builds/log.go @@ -0,0 +1,177 @@ +package builds + +import ( + "encoding/json" + "fmt" + "sort" + "time" + + "git.novox.be/novox/mesh-delivery/sdk/kept" + "git.novox.be/novox/mesh-delivery/sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// The builder's log (clause 1; the lead and the operator, 2026-10-29): the builder's own module log (ADR 0297) `build-log`, +// kept 30 days (max-age-days), one entry under the key `` when a build of a fingerprint starts and one when it +// ends, so the newest entry of a key is the module's newest start or end. The builder's tools read it through the +// runtime's log verbs; another module reads it by naming BuildLogRead in its manifest's `reads`: a bundle through the +// runtime (stdio.Log(BuildLogRead)), a process on its own credential (process.Conn.ReadLog), each by newest per key, +// from a time, or following new entries. The catalogue module asks the builder's tool BuildsSinceTool instead. +const ( + BuilderLog = "build-log" + BuildLogRead = BuilderModule + "." + BuilderLog + BuildStartedEvent = "build.started" + BuildEndedEvent = "build.ended" + // BuildSaidEvent is a build's narration between its start and its end (design 51: a build narrates its publish, + // asked, waiting on a seat, stored, refused, at least every five minutes while it waits): its module, fingerprint, + // line and why. + BuildSaidEvent = "build.said" + // BuilderLogDays is how long the log keeps its entries. + BuilderLogDays = 30 + // BuildsSinceTool is the builder's tool answering every ended build since a time, in order (clause 9): its + // argument `since` (RFC 3339), its answer BuildsSince. + BuildsSinceTool = "builder_builds_since" +) + +// BuildLogKey is the key of a module's entries in the builder's log: the module's name, refused when a log key could not +// hold it. +func BuildLogKey(module string) (string, error) { + if err := kept.CheckName("module", module); err != nil { + return "", err + } + return module, nil +} + +// BuildLogEntry is one entry of the builder's log: the module, the fingerprint, the source revision, where the module is +// built from (its repository, its directory there, the commit and the ref), the build node and the time; at the end, +// the outcome, the manifest built (its digest, read from the indexer's state `manifests` under ManifestsKey), what it was +// built against (the bases' references, as the controller's built-before said them) and each artifact made +// (BuiltArtifact, the done key's: its name, kind, digest and reference). The repository, directory and commit are what +// the indexer's PathOf, the outside-built registrations and the assigner's running commits read; the manifest, against +// and made what the catalogue module registers a build from (the lead, 2026-10-29). +type BuildLogEntry struct { + V int `json:"v"` + Event string `json:"event"` + Module string `json:"module"` + FP string `json:"fp"` + Source string `json:"source"` + Repository string `json:"repository"` + Directory string `json:"directory"` + Commit string `json:"commit"` + Ref string `json:"ref,omitempty"` + Node string `json:"node"` + At time.Time `json:"at"` + Outcome string `json:"outcome,omitempty"` + Manifest string `json:"manifest,omitempty"` + Against []string `json:"against,omitempty"` + Made []BuiltArtifact `json:"made,omitempty"` + // Imports are, at a build's end, the mesh packages (import paths) the built programs import, their tests left out, + // sorted and each once, as the build module answered them (BuildAnswer.Imports; novox/hq ADR 0593 §2): the indexer + // keeps them on the module (KeptInputs2.Imports) when the build's fingerprint is the module's. + Imports []string `json:"imports,omitempty"` + // OutOfStep are, at a build's end, the generated files its test step found out of step with what their generator + // writes, as the build module answered them (its test step's `out-of-step`): the indexer raises a finding naming them + // for a trunk build, and only a clean build of the same module clears it. A start or a narration names none. + OutOfStep []string `json:"out-of-step,omitempty"` + // Line is a narration's words, Why what it narrates (asked, waiting, stored, refused): a build.said entry's alone. + Line string `json:"line,omitempty"` + Why string `json:"why,omitempty"` +} + +// BuildsSince is the answer of BuildsSinceTool: every ended build since a time, oldest first, and whether the log held +// more than one answer carries (then ask again from the last one's time). +type BuildsSince struct { + Since time.Time `json:"since"` + Builds []BuildLogEntry `json:"builds"` + More bool `json:"more,omitempty"` +} + +// NewestPerModule is each module's newest entry of entries (one kind, read from the last per subject or whole), by +// module: the reader's helper for the five newest and the running commits. +func NewestPerModule(entries []BuildLogEntry) map[string]BuildLogEntry { + out := map[string]BuildLogEntry{} + for _, e := range entries { + if have, ok := out[e.Module]; !ok || e.At.After(have.At) { + out[e.Module] = e + } + } + return out +} + +// EndedSince is, of entries, the ended builds at or after since, oldest first: the reader's helper for history and for +// BuildsSinceTool's answer. +func EndedSince(entries []BuildLogEntry, since time.Time) []BuildLogEntry { + var out []BuildLogEntry + for _, e := range entries { + if e.Event == BuildEndedEvent && !e.At.Before(since) { + out = append(out, e) + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) }) + return out +} + +// EncodeBuildLogEntry is e as JSON at Version; it refuses an entry a reader could not place. +func EncodeBuildLogEntry(e BuildLogEntry) ([]byte, error) { + e.V = keys.Version + switch { + case e.Event == BuildSaidEvent: + switch { + case e.Module == "" || e.FP == "" || e.At.IsZero() || e.Line == "": + return nil, fmt.Errorf("builds: a build's narration names its module, fingerprint, line and time") + case e.Outcome != "" || len(e.Made) > 0 || e.Manifest != "" || len(e.Against) > 0 || len(e.Imports) > 0 || len(e.OutOfStep) > 0: + return nil, fmt.Errorf("builds: a build's narration names no outcome, manifest, base, artifact, import or file out of step") + } + return json.Marshal(e) + case e.Event != BuildStartedEvent && e.Event != BuildEndedEvent: + return nil, fmt.Errorf("builds: a build log entry is %s, %s or %s, not %q", BuildStartedEvent, BuildSaidEvent, BuildEndedEvent, e.Event) + case e.Line != "" || e.Why != "": + return nil, fmt.Errorf("builds: a build's start or end carries no narration's line") + case e.Module == "" || e.FP == "" || e.Node == "" || e.At.IsZero() || e.Repository == "" || e.Commit == "": + return nil, fmt.Errorf("builds: a build log entry names its module, fingerprint, repository, commit, node and time") + case e.Event == BuildEndedEvent && e.Outcome == "": + return nil, fmt.Errorf("builds: a build's end names its outcome") + case e.Event == BuildStartedEvent && (e.Outcome != "" || len(e.Made) > 0 || e.Manifest != "" || len(e.Against) > 0 || len(e.Imports) > 0 || + len(e.OutOfStep) > 0): + return nil, fmt.Errorf("builds: a build's start names no outcome, manifest, base, artifact, import or file out of step") + } + if err := trunk.CheckImports(e.Imports); err != nil { + return nil, err + } + return json.Marshal(e) +} + +// DecodeBuildLogEntry reads one entry. +func DecodeBuildLogEntry(data []byte) (BuildLogEntry, error) { + var e BuildLogEntry + if err := json.Unmarshal(data, &e); err != nil { + return BuildLogEntry{}, fmt.Errorf("builds: a build log entry is not readable: %w", err) + } + if e.V != keys.Version { + return BuildLogEntry{}, fmt.Errorf("builds: a build log entry of version %d, and this reads %d", e.V, keys.Version) + } + return e, nil +} + +// BuildDuration is a build's duration, its end less its start (clause 1): false unless both are the same module's +// build of the same fingerprint, the start first. +func BuildDuration(start, end BuildLogEntry) (time.Duration, bool) { + if start.Event != BuildStartedEvent || end.Event != BuildEndedEvent || start.Module != end.Module || start.FP != end.FP || + end.At.Before(start.At) { + return 0, false + } + return end.At.Sub(start.At), true +} + +// IsBuildEntry says whether an entry of the builder's log is a build's start or end, read from its event alone, so an entry of +// another kind (a build's narration, build.said) is passed over before its form is read. One whose event cannot be read is +// taken as a build's, so its decoding says what is wrong with it. +func IsBuildEntry(value []byte) bool { + var e struct { + Event *string `json:"event"` + } + if json.Unmarshal(value, &e) != nil || e.Event == nil { + return true + } + return *e.Event == BuildStartedEvent || *e.Event == BuildEndedEvent +} diff --git a/modules/mesh-delivery-builder/builds/log_test.go b/modules/mesh-delivery-builder/builds/log_test.go new file mode 100644 index 00000000..4f62c5e0 --- /dev/null +++ b/modules/mesh-delivery-builder/builds/log_test.go @@ -0,0 +1,60 @@ +package builds + +import ( + "testing" + "time" +) + +func TestTheBuildersLogKeepsAStartAndAnEndAndTheirDuration(t *testing.T) { + // ADR 0571 clause 1: the builder's log, a start and an end, and the duration between them. + at := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + start := BuildLogEntry{Event: BuildStartedEvent, Module: "letta", FP: "src2:a", Source: "abc", Repository: "novox/mesh-catalog", + Directory: "modules/letta", Commit: "abc123", Node: "ace", At: at} + end := start + end.Event, end.At, end.Outcome, end.Manifest, end.Against = BuildEndedEvent, at.Add(3*time.Minute), "done", "sha256:m", []string{"artifact-store://base/image@sha256:b"} + end.Made = []BuiltArtifact{{Name: "image", Kind: "image", Digest: "sha256:1", Reference: "artifact-store://letta/image@sha256:1"}} + for _, e := range []BuildLogEntry{start, end} { + raw, err := EncodeBuildLogEntry(e) + back, derr := DecodeBuildLogEntry(raw) + if err != nil || derr != nil || back.Event != e.Event || back.Outcome != e.Outcome || back.Directory != "modules/letta" || + back.Repository != "novox/mesh-catalog" || back.Commit != "abc123" { + t.Errorf("%s read back as %+v, %v %v", e.Event, back, err, derr) + } + } + if d, ok := BuildDuration(start, end); !ok || d != 3*time.Minute { + t.Errorf("the build took %s, %v", d, ok) + } + older := end + older.At, older.FP = at.Add(-time.Hour), "src2:b" + other := end + other.Module = "mesh-catalog" + if n := NewestPerModule([]BuildLogEntry{older, end, other}); n["letta"].FP != "src2:a" || len(n) != 2 { + t.Errorf("each module's newest is %v", n) + } + if e := EndedSince([]BuildLogEntry{end, start, older}, at.Add(-2*time.Hour)); len(e) != 2 || e[0].FP != "src2:b" { + t.Errorf("the ended builds since are %v", e) + } + if k, err := BuildLogKey("letta"); k != "letta" || err != nil || BuildLogRead != "mesh-delivery-builder.build-log" { + t.Errorf("the key is %q, %v", k, err) + } + if _, err := BuildLogKey("a.b"); err == nil { + t.Error("a module with a dot was taken as a key") + } + for _, edit := range []func(*BuildLogEntry){func(e *BuildLogEntry) { e.Outcome = "" }, func(e *BuildLogEntry) { e.Commit = "" }, + func(e *BuildLogEntry) { e.Event, e.Outcome, e.Against = BuildStartedEvent, "", nil }} { + bad := end + edit(&bad) + if _, err := EncodeBuildLogEntry(bad); err == nil { + t.Errorf("%+v was encoded", bad) + } + } +} + +func TestOnlyABuildsStartOrEndIsABuildEntry(t *testing.T) { + for entry, want := range map[string]bool{`{"event":"build.started"}`: true, `{"event":"build.ended"}`: true, `{"event":"build.said"}`: false, + `not json`: true, `{}`: true} { + if got := IsBuildEntry([]byte(entry)); got != want { + t.Errorf("IsBuildEntry(%s) = %v", entry, got) + } + } +} diff --git a/modules/mesh-delivery-builder/builds/numbered.go b/modules/mesh-delivery-builder/builds/numbered.go new file mode 100644 index 00000000..0eca558a --- /dev/null +++ b/modules/mesh-delivery-builder/builds/numbered.go @@ -0,0 +1,57 @@ +package builds + +import ( + "encoding/json" + "fmt" +) + +// The builder's event version.numbered (design 51 *Events of the builder*, *The floor*): said once per version the builder +// numbers, on its own family `version` (seven days), apart from its builds' ends, so a waiter on a build's end does not +// wake on it. A build module consumes it to publish a build's version with the held digest, which stores nothing. +const ( + // VersionNumberedEvent is the event's local name. + VersionNumberedEvent = "version.numbered" + // VersionNumberedSubject is its subject, as a consumer's filter names it. + VersionNumberedSubject = "mesh.mod." + BuilderModule + ".event." + VersionNumberedEvent +) + +// VersionNumbered is the body of VersionNumberedEvent: a numbered version of a name, the build it was numbered for, and +// what the build module needs to publish it. +type VersionNumbered struct { + // Name is the numbered name, as builder_versions lists it; Kind its artifact's kind. + Name string `json:"name"` + Kind string `json:"kind"` + Version string `json:"version"` + Tag string `json:"tag,omitempty"` + // Module and Fingerprint are the build the version was numbered for; Commit its source commit. + Module string `json:"module"` + Fingerprint string `json:"fingerprint"` + Commit string `json:"commit,omitempty"` + // Digest is the built artifact's, when the build held one; Seat the registry seat it is published through. + Digest string `json:"digest,omitempty"` + Seat string `json:"seat,omitempty"` + // Head is the head the version answers, when it answers a tag ask. + Head string `json:"head,omitempty"` + // Again is when builder_publish_again said it once more, so the bus's dedup does not drop it. + Again string `json:"again,omitempty"` +} + +// EncodeVersionNumbered writes the body, refusing one that names no name, version, module or fingerprint. +func EncodeVersionNumbered(n VersionNumbered) ([]byte, error) { + if n.Name == "" || n.Version == "" || n.Module == "" || n.Fingerprint == "" { + return nil, fmt.Errorf("builds: %s names its name, version, module and fingerprint", VersionNumberedEvent) + } + return json.Marshal(n) +} + +// DecodeVersionNumbered reads the body, with the same refusal. +func DecodeVersionNumbered(data []byte) (VersionNumbered, error) { + var n VersionNumbered + if err := json.Unmarshal(data, &n); err != nil { + return VersionNumbered{}, fmt.Errorf("builds: %s is not readable: %w", VersionNumberedEvent, err) + } + if _, err := EncodeVersionNumbered(n); err != nil { + return VersionNumbered{}, err + } + return n, nil +} diff --git a/modules/mesh-delivery-builder/builds/reference_test.go b/modules/mesh-delivery-builder/builds/reference_test.go new file mode 100644 index 00000000..8032e036 --- /dev/null +++ b/modules/mesh-delivery-builder/builds/reference_test.go @@ -0,0 +1,157 @@ +package builds + +import ( + "strings" + "testing" +) + +func artifactOf(name, kind, digest, reference string) BuiltArtifact { + return BuiltArtifact{Name: name, Kind: kind, Digest: digest, Reference: reference} +} + +// A done key names where each artifact is (novox/hq ADR 0498 clauses 1, 5 and 6; the lead's decision L1 of R4e): a +// bundle or an archive by its artifact-store blob, an image or an upstream copy by the registry's reference to its +// digest (or the kept artifact-store form), a package by name and version. A reference of the wrong form for its kind, +// or naming another digest or artifact, is refused, naming the artifact. +func TestADoneKeyNamesEachArtifactsReferenceInTheFormOfItsKind(t *testing.T) { + d := "sha256:" + digestOf('6') + good := []BuiltArtifact{ + artifactOf("code", "bundle", d, "artifact-store://mail/code/blobs/"+d), + artifactOf("files", "archive", d, "artifact-store://mail/files/blobs/"+d), + artifactOf("server", "image", d, "10.0.0.1:5000/mail/server@"+d), + artifactOf("server", "image", d, "artifact-store://mail/server@"+d), + artifactOf("nats", "upstream", d, "registry.example/nats/nats@"+d), + artifactOf("lib", "package", d, "@novox/mesh-sdk@0.1.15"), + artifactOf("lib", "package", d, "left-pad@1.3.0"), + artifactOf("old", "", d, "anything at all"), + // The form a registry seat answers (ADR 0605 §9), beside the older one. + artifactOf("code", "bundle", d, "oci-artifact-registry/mail/code@"+d), + artifactOf("files", "archive", d, "oci-artifact-registry/mail/files@"+d), + artifactOf("server", "image", d, "container-image-registry/mail/server@"+d), + artifactOf("nats", "upstream", d, "container-image-registry/nats/nats@"+d), + // An upstream copy under the holder's own naming, upstream//, in either form (ADR 0624 §1). + artifactOf("app", "upstream", d, "container-image-registry/upstream/quay.io:443/org/app@"+d), + artifactOf("app", "upstream", d, "artifact-store://upstream/quay.io:443/org/app@"+d), + } + for _, a := range good { + b := doneBuild() + b.Artifacts = []BuiltArtifact{a} + if _, err := EncodeBuilderBuild(b); err != nil { + t.Errorf("%+v was refused: %v", a, err) + } + } + other := "sha256:" + digestOf('7') + bad := []BuiltArtifact{ + artifactOf("code", "bundle", d, "artifact-store://mail/code/blobs/"+other), + artifactOf("code", "bundle", d, "artifact-store://mail/other/blobs/"+d), + artifactOf("code", "bundle", d, "10.0.0.1:5000/mail/code@"+d), + artifactOf("server", "image", d, "10.0.0.1:5000/mail@"+d), + artifactOf("server", "image", d, "10.0.0.1:5000/mail/server@"+other), + artifactOf("server", "image", d, "mail/server@"+d), + artifactOf("server", "image", d, "artifact-store://mail/server/blobs/"+d), + artifactOf("lib", "package", d, "@novox/mesh-sdk"), + artifactOf("lib", "package", d, "0.1.15"), + artifactOf("code", "bundle", d, "container-image-registry/mail/code@"+d), + artifactOf("server", "image", d, "oci-artifact-registry/mail/server@"+d), + artifactOf("server", "image", d, "container-image-registry/mail/server@"+other), + artifactOf("server", "image", d, "container-image-registry/mail/other@"+d), + artifactOf("lib", "package", d, "container-image-registry/mail/lib@"+d), + artifactOf("app", "upstream", d, "container-image-registry/upstream/quay.io:443/org/app@"+other), + artifactOf("app", "upstream", d, "artifact-store://upstream/quay.io:443/org/app@"+other), + artifactOf("app", "upstream", d, "oci-artifact-registry/upstream/quay.io:443/org/app@"+d), + artifactOf("server", "image", d, "container-image-registry/upstream/quay.io:443/org/server@"+d), + } + for _, a := range bad { + b := doneBuild() + b.Artifacts = []BuiltArtifact{a} + if _, err := EncodeBuilderBuild(b); err == nil || !strings.Contains(err.Error(), `"`+a.Name+`"`) { + t.Errorf("%+v: %v, wanted a refusal naming the artifact", a, err) + } + } +} + +// The controller delivers only from a done key whose every artifact names a reference (ADR 0498 clause 8): Referenced +// says so and names the artifacts without one. +func TestReferencedNamesTheArtifactsWithoutAReference(t *testing.T) { + d := "sha256:" + digestOf('6') + b := doneBuild() + b.Artifacts = []BuiltArtifact{ + artifactOf("code", "bundle", d, "artifact-store://mail/code/blobs/"+d), + artifactOf("server", "image", d, ""), + } + if ok, missing := b.Referenced(); ok || len(missing) != 1 || missing[0] != "server" { + t.Fatalf("Referenced = %v, %v; want false, [server]", ok, missing) + } + b.Artifacts[1].Reference = "10.0.0.1:5000/mail/server@" + d + if ok, missing := b.Referenced(); !ok || len(missing) != 0 { + t.Fatalf("Referenced = %v, %v; want true", ok, missing) + } + if ok, _ := (BuilderBuild{State: BuildQueued}).Referenced(); ok { + t.Fatal("a queued key is referenced") + } +} + +// The errors a build module answers open with a fixed prefix (ADR 0498 clauses 2, 4 and 5; ADR 0478 clause 2), and three +// of them are not built again until the module's fingerprint changes; a failed push is. +func TestOnlyAPushFailureIsBuiltAgainBeforeTheFingerprintChanges(t *testing.T) { + for _, c := range []struct { + err string + again bool + }{ + {NotStored + "code: the store said 500", true}, + {NotPinned + "nats: upstream names no digest", false}, + {VersionTaken + "@novox/mesh-sdk@0.1.15 holds another file", false}, + {RefusedKind + "a package in go", false}, + {"go test: 2 failed", true}, + } { + if got := FailedUntilChanged(c.err); got == c.again { + t.Errorf("FailedUntilChanged(%q) = %v", c.err, got) + } + } + if NotStored != "not stored: " || NotPinned != "not pinned: " || VersionTaken != "version taken: " || RefusedKind != "refused kind: " { + t.Fatal("a prefix changed its words") + } + // A Node fetch that finds the lock naming a package version the registry does not hold fails with WaitsFor; the + // builder then writes the key queued with Waits, and the error is not one that waits for the fingerprint to change. + if WaitsFor != "waits for: " || FailedUntilChanged(WaitsFor+"@novox/mesh-sdk@0.1.15") { + t.Fatalf("WaitsFor = %q", WaitsFor) + } +} + +// A key whose module's lock names a version of the group's package not yet published waits, queued, saying which +// (ADR 0500 clause 6): an older reader reads it as queued, and only a queued key waits. +func TestAKeyWaitsQueuedForAPackageVersion(t *testing.T) { + b := BuilderBuild{State: BuildQueued, Manifest: digestOf('4'), Waits: "@novox/mesh-sdk@0.1.15"} + data, err := EncodeBuilderBuild(b) + if err != nil { + t.Fatal(err) + } + back, err := DecodeBuilderBuild(data) + if err != nil || back.Waits != b.Waits || back.State != BuildQueued { + t.Fatalf("decoded %+v, %v", back, err) + } + building := BuilderBuild{State: BuildBuilding, By: "x", Waits: "@novox/mesh-sdk@0.1.15"} + if _, err := EncodeBuilderBuild(building); err == nil { + t.Fatal("a building key that waits was accepted") + } + if _, err := EncodeBuilderBuild(BuilderBuild{State: BuildQueued, Waits: "mesh-sdk"}); err == nil { + t.Fatal("a wait naming no version was accepted") + } +} + +// A module that builds no artifact gets a done key too (issue 1613), and it is delivered: Referenced is ok for a done +// key with no artifact, since no artifact lacks a reference; a queued one with none still is not. +func TestADoneKeyWithNoArtifactIsReferenced(t *testing.T) { + b := BuilderBuild{State: BuildDone, Manifest: digestOf('4')} + data, err := EncodeBuilderBuild(b) + if err != nil { + t.Fatalf("a done key with no artifact was refused: %v", err) + } + back, err := DecodeBuilderBuild(data) + if err != nil { + t.Fatal(err) + } + if ok, missing := back.Referenced(); !ok || len(missing) != 0 { + t.Fatalf("Referenced = %v, %v; want true", ok, missing) + } +} diff --git a/modules/mesh-delivery-builder/builds/versions.go b/modules/mesh-delivery-builder/builds/versions.go new file mode 100644 index 00000000..a8abcf45 --- /dev/null +++ b/modules/mesh-delivery-builder/builds/versions.go @@ -0,0 +1,38 @@ +package builds + +import ( + "encoding/json" + "fmt" +) + +// The holder's question before it stores a publish (novox/hq ADR 0605 §4, design 51 *Publishing*): did the builder +// number this version with this digest? Every registry seat's holder asks it the same way, so the asking lives here +// once (ADR 0627 §2), beside the builder's layouts; no holder writes it again. +// +// The builder answers it today only as its module's tool builder_versions: no seat in the seat set names the builder's +// role yet. Once one does, BuilderVersionsAddress becomes that seat's verb and nothing else changes for a holder. +const BuilderVersionsAddress = BuilderModule + ".builder_versions" + +// AskFunc is the runtime's request and reply over the bus (mesh-sdk stdio.Ask), given by the holder so this package does +// not depend on the runtime. +type AskFunc func(address string, body any) (json.RawMessage, error) + +// BuilderVersions asks the builder whether it numbered a version with a digest. It answers the holder's Check: +// recorded, and why not when it is not. +type BuilderVersions struct{ Ask AskFunc } + +// Recorded asks the builder about name@version with digest. An answer without `recorded` is an error, never a yes. +func (b BuilderVersions) Recorded(name, version, digest string) (bool, string, error) { + raw, err := b.Ask(BuilderVersionsAddress, map[string]any{"name": name, "version": version, "digest": digest}) + if err != nil { + return false, "", err + } + var a struct { + Recorded *bool `json:"recorded"` + Why string `json:"why"` + } + if err := json.Unmarshal(raw, &a); err != nil || a.Recorded == nil { + return false, "", fmt.Errorf("builder_versions answered no recorded: %s", raw) + } + return *a.Recorded, a.Why, nil +} diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 3bf21661..504beb60 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -167,6 +167,15 @@ "loads": [ "mesh-delivery-builder-tools" ] + }, + { + "name": "builds", + "kind": "package", + "language": "go", + "paths": [ + "builds" + ], + "series": "0.1" } ] } diff --git a/sdk/buildstep/steps.go b/sdk/buildstep/steps.go index 6a6a306a..e39bb9f9 100644 --- a/sdk/buildstep/steps.go +++ b/sdk/buildstep/steps.go @@ -187,3 +187,12 @@ type Ask struct { Fetch map[string]any `json:"fetch"` } `json:"compose"` } + +// CheckAskID refuses an id that is no ask's: base-36 milliseconds and six hex digits, as NewID gives. Its form keeps it apart from +// a module's or a node's name in a key. +func CheckAskID(id string) error { + if !askIDOK.MatchString(id) { + return fmt.Errorf("buildstep: %q is not an ask id (base-36 milliseconds and six hex digits)", id) + } + return nil +} diff --git a/sdk/kept/kept.go b/sdk/kept/kept.go new file mode 100644 index 00000000..1d907361 --- /dev/null +++ b/sdk/kept/kept.go @@ -0,0 +1,73 @@ +// Package kept holds what the layouts of the Delivery domain's kept values share: the version check on reading one, the +// rule a name must keep to be a key part, and the form of a digest. It names no seat and no module (ADR 0634 §2). The +// version itself (keys.Version) and the escaping of a name (keys.EscapeName) are mesh-sdk's, since more than one domain uses +// them. +package kept + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// ErrUnknownVersion is what Decode returns for a value whose version is missing or not keys.Version. Test it with errors.Is. +var ErrUnknownVersion = errors.New("kept: unknown value version") + +var nameOK = regexp.MustCompile(`^[-/_=a-zA-Z0-9]+$`) + +// CheckName refuses a name that is empty or holds a dot, a wildcard, whitespace or a character a bus key refuses. +func CheckName(kind, name string) error { + if name == "" { + return fmt.Errorf("kept: the %s name is empty", kind) + } + if !nameOK.MatchString(name) { + return fmt.Errorf("kept: the %s name %q holds a dot, a wildcard, whitespace or a character a key refuses", kind, name) + } + return nil +} + +var hex64 = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// IsHex64 says whether s is 64 lowercase hex digits. +func IsHex64(s string) bool { return hex64.MatchString(s) } + +// CheckVersion refuses a value with no version or one this reader does not read. +func CheckVersion(data []byte) error { + var probe struct { + V *int `json:"v"` + } + if err := json.Unmarshal(data, &probe); err != nil { + return fmt.Errorf("kept: value is not JSON: %w", err) + } + if probe.V == nil { + return fmt.Errorf("%w: no version in the value, this reader reads %d", ErrUnknownVersion, keys.Version) + } + if *probe.V != keys.Version { + return fmt.Errorf("%w: the value is version %d, this reader reads %d", ErrUnknownVersion, *probe.V, keys.Version) + } + return nil +} + +// Decode reads data into into after CheckVersion. +func Decode(data []byte, into any) error { + if err := CheckVersion(data); err != nil { + return err + } + if err := json.Unmarshal(data, into); err != nil { + return fmt.Errorf("kept: value does not fit its shape: %w", err) + } + return nil +} + +var digestForm = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + +// CheckDigest refuses a digest that is not sha256:<64 lowercase hex digits>, saying what it is of. +func CheckDigest(digest, of string) error { + if !digestForm.MatchString(digest) { + return fmt.Errorf("kept: %s %q is not sha256:<64 lowercase hex digits>", of, digest) + } + return nil +} diff --git a/sdk/kept/kept_test.go b/sdk/kept/kept_test.go new file mode 100644 index 00000000..aeff9ec0 --- /dev/null +++ b/sdk/kept/kept_test.go @@ -0,0 +1,31 @@ +package kept + +import ( + "errors" + "strings" + "testing" +) + +func TestAKeptValueIsReadOnlyAtItsVersionAndAKeyPartOnlyOfKeyCharacters(t *testing.T) { + var v struct{ V int } + for data, ok := range map[string]bool{`{"v":1}`: true, `{"v":2}`: false, `{}`: false} { + err := Decode([]byte(data), &v) + if (err == nil) != ok || (!ok && !errors.Is(err, ErrUnknownVersion)) { + t.Errorf("Decode(%s) = %v", data, err) + } + } + if Decode([]byte(`nope`), &v) == nil { + t.Error("a value that is no JSON is read") + } + for name, ok := range map[string]bool{"a-b_c/d=": true, "": false, "a.b": false, "a b": false, "a*": false} { + if (CheckName("module", name) == nil) != ok { + t.Errorf("CheckName(%q) = %v", name, CheckName("module", name)) + } + } + if !IsHex64(strings.Repeat("a", 64)) || IsHex64(strings.Repeat("A", 64)) || IsHex64("a") { + t.Error("IsHex64 reads a wrong form") + } + if CheckDigest("sha256:"+strings.Repeat("0", 64), "x") != nil || CheckDigest("sha256:0", "x") == nil { + t.Error("CheckDigest reads a wrong form") + } +} diff --git a/sdk/trunk/packages.go b/sdk/trunk/packages.go index 442767e0..4d5f653b 100644 --- a/sdk/trunk/packages.go +++ b/sdk/trunk/packages.go @@ -1,6 +1,7 @@ package trunk import ( + "fmt" "strings" ) @@ -26,3 +27,17 @@ func PackageRepository(pkg string) (repository, dir string, ok bool) { } return "novox/" + name, dir, true } + +// CheckImports refuses an import set (the imports of a module's kept inputs, of a build's answer and of a build log entry) +// that holds a path that is no mesh package's, or is not sorted and each once. +func CheckImports(imports []string) error { + for i, pkg := range imports { + if _, _, ok := PackageRepository(pkg); !ok || strings.ContainsAny(pkg, " \t\n@") { + return fmt.Errorf("trunk: the import %q is no mesh package's path (it starts with %s)", pkg, ForgeModulePrefix) + } + if i > 0 && imports[i-1] >= pkg { + return fmt.Errorf("trunk: the imports are not sorted and each once (%s after %s)", pkg, imports[i-1]) + } + } + return nil +} diff --git a/sdk/trunk/trunk_test.go b/sdk/trunk/trunk_test.go index 7ea841e8..c0c6b53f 100644 --- a/sdk/trunk/trunk_test.go +++ b/sdk/trunk/trunk_test.go @@ -33,3 +33,15 @@ func TestSameRepository(t *testing.T) { // 1281/AC1 t.Errorf("RepositoryName = %q", got) } } + +func TestAnImportSetHoldsMeshPackagesSortedAndEachOnce(t *testing.T) { + a, b := ForgeModulePrefix+"mesh-sdk/go", ForgeModulePrefix+"mesh-delivery/sdk" + if err := CheckImports([]string{b, a}); err != nil { + t.Errorf("a sorted set is refused: %v", err) + } + for name, bad := range map[string][]string{"unsorted": {a, b}, "twice": {a, a}, "not a mesh package": {"github.com/x/y"}, "a version": {a + "@v1"}} { + if CheckImports(bad) == nil { + t.Errorf("%s: %v is taken", name, bad) + } + } +}