diff --git a/modules/mesh-build-go/README.md b/modules/mesh-build-go/README.md index 6e73d03d..2a18e494 100644 --- a/modules/mesh-build-go/README.md +++ b/modules/mesh-build-go/README.md @@ -142,7 +142,11 @@ trunk build's, in chunks, only what the holder lacks; the holder asks the builde seat's events. The step answers the seat's form `oci-artifact-registry//@` (ADR 0605 §9); the builder writes the older form beside it while a counted node-engine reads only that (ADR 0624); a publish the holder refuses, or a seat nothing holds, fails the step `not stored: : `. The packed bytes never stay on the node. -The module writes into no store itself and requires no `artifact-store`. The go build runs shut off in bubblewrap (ADR 0448, `isolate.go`): no +The module writes into no store itself and requires no `artifact-store`. A Go package (kind `package`, language `go`; `gopackage.go`) is +the directory its one path names packed as Go's module zip, every file under `@/`, its digest the +zip's and its `h1:` sum answered beside it; a trunk build the builder gave a version publishes it through +`go-package-registry` under its Go module path and that version (the manifest `invokes` both verbs) and answers +`@`; a candidate's is packed under `v0.0.0` and publishes nothing. The go build runs shut off in bubblewrap (ADR 0448, `isolate.go`): no network, none of the account, the caches beneath a throwaway layer unless the step is told `publish` (a trunk build). An artifact that is neither a Go bundle nor an archive fails the step naming it. diff --git a/modules/mesh-build-go/buildstep/askrun.go b/modules/mesh-build-go/buildstep/askrun.go index 60bc385e..e63ff9be 100644 --- a/modules/mesh-build-go/buildstep/askrun.go +++ b/modules/mesh-build-go/buildstep/askrun.go @@ -175,6 +175,13 @@ func (s AskSteps) build(ctx context.Context, job delivery.BuildJob) (any, error) if job.Publish { args["publish"] = true } + // The seat it is published through and the version the builder gave it (design 51 *Publishing*, CAT-BG2). + if a.Seat != "" { + args["seat"] = a.Seat + } + if a.Version != "" { + args["version"] = a.Version + } Phase(ctx, "build "+a.Artifact) out, err = s.Build(ctx, args) } diff --git a/modules/mesh-build-go/buildstep/askrun_test.go b/modules/mesh-build-go/buildstep/askrun_test.go index b0fce397..fb75f23f 100644 --- a/modules/mesh-build-go/buildstep/askrun_test.go +++ b/modules/mesh-build-go/buildstep/askrun_test.go @@ -70,7 +70,8 @@ func ask(job any) delivery.StepAsk { } // 1887/A3: a build ask fetches once, builds each artifact from the workspace with publish, copies a pinned one without a -// fetch, and answers each artifact's digest and reference. +// fetch, and answers each artifact's digest and reference; an artifact's seat and the version the builder gave it reach +// its build (CAT-BG2). func TestRunAskBuilds(t *testing.T) { r := &recorder{fetch: map[string]any{"workspace": "/w/1"}} steps := r.steps() @@ -85,7 +86,7 @@ func TestRunAskBuilds(t *testing.T) { // Issue 1585: the ask's package versions reach the fetch, and the ask's own fetch arguments stay as they were. a := ask(delivery.BuildJob{Module: "m", Path: "modules/m", Publish: true, Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}, - Artifacts: []delivery.AskArtifact{{Artifact: "base", From: "registry/b@sha256:00"}, {Artifact: "bundle"}, {Artifact: "lib"}}}) + Artifacts: []delivery.AskArtifact{{Artifact: "base", From: "registry/b@sha256:00"}, {Artifact: "bundle"}, {Artifact: "lib", Seat: "go-package-registry", Version: "v0.1.15"}}}) a.Packages = delivery.PackageVersions{"git.novox.be/novox/mesh-sdk/go": runCommit} out, err := RunAsk(context.Background(), steps, a) if err != nil { @@ -98,7 +99,7 @@ func TestRunAskBuilds(t *testing.T) { want := `copy {"artifact":"base","from":"registry/b@sha256:00","module":"m"} fetch {"commit":"` + runCommit + `","packages":{"git.novox.be/novox/mesh-sdk/go":"` + runCommit + `"},"repository":"novox/x"} build {"artifact":"bundle","module":"modules/m","publish":true,"workspace":"/w/1"} -build {"artifact":"lib","module":"modules/m","publish":true,"workspace":"/w/1"} +build {"artifact":"lib","module":"modules/m","publish":true,"seat":"go-package-registry","version":"v0.1.15","workspace":"/w/1"} imports {"module":"modules/m","workspace":"/w/1"}` if got != want { t.Fatalf("calls:\n%s\nwant:\n%s", got, want) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build.go b/modules/mesh-build-go/cmd/mesh-build-go/build.go index a2787d8c..fe5f1847 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/build.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/build.go @@ -38,20 +38,25 @@ type BuildResult struct { // §9): oci-artifact-registry//@. The builder writes the older form beside it while a // counted node-engine reads only that (ADR 0624 §1, §3). Reference string `json:"reference"` + // Sum is a Go package's h1: hash, Go's go.sum hash of its module zip (CAT-BG2); empty for every other artifact. + Sum string `json:"sum,omitempty"` } func (m *Module) buildTool() StepTool { return StepTool{ Name: "build_go_build", - Description: "Build one Go bundle or archive artifact of a module of a fetched workspace with the build agent's recipe, by the " + - "installed Go and no network, and pack it. Answers a step id at once; the step ends on its key with the " + - "artifact's digest, size and the archive's path.", + Description: "Build one Go bundle, archive or Go package artifact of a module of a fetched workspace with the build agent's " + + "recipe, by the installed Go and no network, and pack it (a Go package as Go's module zip); a bundle or archive is " + + "published on oci-artifact-registry, a Go package given a version on go-package-registry under its module path. " + + "Answers a step id at once; the step ends on its key with the artifact's digest, size, reference and a Go package's h1: sum.", Input: map[string]any{ "type": "object", "properties": map[string]any{ "workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"}, "module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"}, "artifact": map[string]any{"type": "string", "description": "the artifact's name in the module's manifest"}, + "version": map[string]any{"type": "string", "description": "the version the builder gave a package at the trunk's claim; absent for a candidate, whose package is not published"}, + "seat": map[string]any{"type": "string", "description": "the registry seat the artifact is published through, as the build ask names it"}, "publish": map[string]any{"type": "boolean", "description": "true for a merged trunk commit alone: the build writes the shared Go caches; without it (a candidate) they are read beneath a throwaway layer"}, }, "required": []string{"workspace", "module", "artifact"}, @@ -110,6 +115,11 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { if art == nil { return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", ")) } + if art.Kind == catalogue.ArtifactPackage && art.Language == "go" { + version, _ := args["version"].(string) + seat, _ := args["seat"].(string) + return m.buildGoPackage(ctx, workspace, dir, *art, version, seat) + } if art.Kind == "archive" { return m.buildArchive(ctx, workspace, module, manifest.Module, dir, *art) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/gopackage.go b/modules/mesh-build-go/cmd/mesh-build-go/gopackage.go new file mode 100644 index 00000000..9116637e --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/gopackage.go @@ -0,0 +1,93 @@ +package main + +// The build of a Go package (CAT-BG2; novox/hq design 51 *Publishing*, ADR 0605 §3, §9): the directory the artifact +// names (its one path, else the module's own) packed as Go's module zip, every file under @/, +// as Go's module proxy protocol serves it; its digest is the zip's sha256, its h1: sum (Go's go.sum hash) beside it. A +// trunk build given a version by the builder publishes it through go-package-registry under its Go module path (the +// name the forge's Go seat takes, the lead's ruling 3 on A3k) and that version; a candidate's publishes nothing. The +// zip is made in this process, after nothing ran: packing a tree runs no code of it. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + "golang.org/x/mod/sumdb/dirhash" + modzip "golang.org/x/mod/zip" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// GoPackageSeat is the registry seat a Go package is published on. +const GoPackageSeat = "go-package-registry" + +// candidateVersion is the version a candidate's zip is laid out under: no version was given, and nothing is published. +const candidateVersion = "v0.0.0" + +func (m *Module) buildGoPackage(ctx context.Context, workspace, dir string, art catalogue.Artifact, version, seat string) (BuildResult, error) { + root := dir + switch len(art.Paths) { + case 0: + case 1: + root = filepath.Join(dir, filepath.FromSlash(strings.TrimSuffix(art.Paths[0], "/"))) + default: + return BuildResult{}, fmt.Errorf("the Go package %s names %d paths; a Go package is one directory holding its go.mod", art.Name, len(art.Paths)) + } + if err := m.insideModule(workspace, dir, root); err != nil { + return BuildResult{}, fmt.Errorf("the directory of %s: %w", art.Name, err) + } + raw, err := os.ReadFile(filepath.Join(root, "go.mod")) + if err != nil { + return BuildResult{}, fmt.Errorf("the Go package %s has no go.mod in %s: %w", art.Name, art.Paths, err) + } + path := modfile.ModulePath(raw) + if path == "" { + return BuildResult{}, fmt.Errorf("the go.mod of the Go package %s names no module", art.Name) + } + if version != "" && seat != GoPackageSeat { + return BuildResult{}, fmt.Errorf("the Go package %s is given the version %s and the seat %q: a Go package is published through %s", art.Name, version, seat, GoPackageSeat) + } + zipVersion := candidateVersion + if version != "" { + zipVersion = "v" + strings.TrimPrefix(version, "v") + } + file, err := os.CreateTemp(filepath.Join(m.Work, workspace), "gopackage-*.zip") + if err != nil { + return BuildResult{}, err + } + defer os.Remove(file.Name()) // the packed file never stays on the node (the lead's decision L9) + defer file.Close() + if err := modzip.CreateFromDir(file, module.Version{Path: path, Version: zipVersion}, root); err != nil { + return BuildResult{}, fmt.Errorf("packing the Go package %s (%s@%s): %w", art.Name, path, zipVersion, err) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return BuildResult{}, err + } + body, err := io.ReadAll(file) + if err != nil { + return BuildResult{}, err + } + sum, err := dirhash.HashZip(file.Name(), dirhash.Hash1) + if err != nil { + return BuildResult{}, fmt.Errorf("the h1: sum of %s: %w", art.Name, err) + } + d := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(d[:]) + res := BuildResult{Artifact: art.Name, Digest: digest, Size: len(body), Sum: sum} + if version == "" { + return res, nil + } + if err := m.publishOn(ctx, seat, path, "package", version, digest, transferName(path, version, digest), body); err != nil { + return BuildResult{}, fmt.Errorf("%s%s: %v", delivery.NotStored, art.Name, err) + } + res.Reference = path + "@" + version + return res, nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go index 333e3d6a..71e81ed1 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go @@ -167,9 +167,11 @@ func TestTheModuleInvokesTheArtifactSeatsVerbsAndWritesNoStore(t *testing.T) { if slices.Contains(m.Requires, "artifact-store") { t.Errorf("requires %v still name artifact-store", m.Requires) } - for _, verb := range []string{"publish", "publish-part"} { - if !slices.Contains(m.Invokes, "seat:"+ArtifactSeat+"."+verb) { - t.Errorf("invokes %v lack the seat's %s", m.Invokes, verb) + for _, seat := range []string{ArtifactSeat, GoPackageSeat} { // a Go package's too (CAT-BG2) + for _, verb := range []string{"publish", "publish-part"} { + if !slices.Contains(m.Invokes, "seat:"+seat+"."+verb) { + t.Errorf("invokes %v lack the seat %s's %s", m.Invokes, seat, verb) + } } } for _, r := range m.Resources { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/publish.go b/modules/mesh-build-go/cmd/mesh-build-go/publish.go index f8c2e59c..c5f638a6 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/publish.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/publish.go @@ -30,15 +30,20 @@ func transferName(module, artifact, digest string) string { return strings.NewReplacer("/", "-", ".", "-", " ", "-", "*", "-", ">", "-").Replace(module + "-" + artifact + "-" + head) } -// publishArchive publishes body, of digest, as / of kind (bundle or archive) on the seat: through -// Call when a test set it, else the runtime's request over the bus (stdio.Ask). +// publishArchive publishes body, of digest, as / of kind (bundle or archive) on the artifact seat. func (m *Module) publishArchive(ctx context.Context, module, artifact, kind, digest string, body []byte) error { + return m.publishOn(ctx, ArtifactSeat, module+"/"+artifact, kind, "", digest, transferName(module, artifact, digest), body) +} + +// publishOn publishes body, of digest, as name (at version, when it has one) of kind on seat, in one part: through Call +// when a test set it, else the runtime's request over the bus (stdio.Ask). +func (m *Module) publishOn(ctx context.Context, seat, name, kind, version, digest, transfer string, body []byte) error { call := m.Call if call == nil { call = stdio.Ask } - in := registry.PublishInput{Name: module + "/" + artifact, Kind: kind, Digest: digest, Transfer: transferName(module, artifact, digest)} - _, err := publish.Publish(ctx, publish.Call(call), ArtifactSeat, in, []publish.Part{{Digest: digest, Size: int64(len(body)), + in := registry.PublishInput{Name: name, Kind: kind, Version: version, Digest: digest, Transfer: transfer} + _, err := publish.Publish(ctx, publish.Call(call), seat, in, []publish.Part{{Digest: digest, Size: int64(len(body)), Open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(body)), nil }}}) return err } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go b/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go index 09b56fa8..138f6fbf 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go @@ -1,6 +1,8 @@ package main import ( + "archive/zip" + "bytes" "crypto/sha256" "encoding/base64" "encoding/hex" @@ -30,6 +32,8 @@ type fakeStore struct { open map[string]registry.PublishInput chunks map[string][]byte asked []string + // published are the publishes whose last chunk was stored, in order. + published []registry.PublishInput } func withStore(t *testing.T, m *Module) *fakeStore { @@ -40,16 +44,20 @@ func withStore(t *testing.T, m *Module) *fakeStore { defer s.mu.Unlock() s.asked = append(s.asked, key) raw, _ := json.Marshal(body) - switch key { - case "seat:" + ArtifactSeat + ".publish": + seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".") + if seat != ArtifactSeat && seat != GoPackageSeat { + return nil, errors.New("nats: no responders available for request") + } + switch verb { + case "publish": var in registry.PublishInput _ = json.Unmarshal(raw, &in) - if in.Kind != "bundle" && in.Kind != "archive" || in.Transfer == "" || strings.ContainsAny(in.Transfer, "/. *>") { + if (seat == ArtifactSeat) != (in.Kind == "bundle" || in.Kind == "archive") || in.Transfer == "" || strings.ContainsAny(in.Transfer, "/. *>") { t.Errorf("publish of kind %q, transfer %q", in.Kind, in.Transfer) } s.open[in.Transfer] = in return json.Marshal(registry.PublishAnswer{Transfer: in.Transfer, Lacks: []string{in.Digest}}) - case "seat:" + ArtifactSeat + ".publish-part": + case "publish-part": var c registry.PublishPartInput _ = json.Unmarshal(raw, &c) data, _ := base64.StdEncoding.DecodeString(c.Data) @@ -67,6 +75,7 @@ func withStore(t *testing.T, m *Module) *fakeStore { t.Errorf("published %s with bytes of another digest", in.Digest) } s.blobs[in.Name+"@"+in.Digest] = whole + s.published = append(s.published, in) pushed.Store(in.Digest, whole) return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part, Complete: true}) } @@ -145,3 +154,74 @@ func TestThePackedFileLeavesTheNodeOnceItsPushAnswered(t *testing.T) { } } } + +// CAT-BG2 (REG S23, ADR 0605 §3, §9; the lead's ruling 3 on A3k): a Go package is built as Go's module zip of the +// directory its artifact names, every file under @/, its digest the zip's and its h1: sum beside +// it. A trunk build given a version publishes it through go-package-registry under its Go module path and that version, +// and answers @; a candidate's (no version) publishes nothing and answers its digest alone. A +// directory with no go.mod fails the step naming it. +func TestAGoPackageIsBuiltAsAModuleZipAndPublishedUnderItsModulePath(t *testing.T) { + const pkg = `{"name":"go","kind":"package","language":"go","paths":["go"],"series":"0.1"}` + m, states, _ := buildModule(t, manifestWith(pkg), "unused") + src := filepath.Join(m.Work, "ws1", "tree", "modules", "demo", "go") + for name, body := range map[string]string{"go.mod": "module git.example/demo/go\n\ngo 1.26\n", "a.go": "package demo\n", + "vendor/x/x.go": "package x\n"} { + if err := os.MkdirAll(filepath.Dir(filepath.Join(src, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + run := func(args map[string]any) (BuildResult, delivery.Step) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == "build_go_build" { + if _, err := tool.Run(args); err != nil { + t.Fatal(err) + } + } + } + st := states.waitEnd(t) + var res BuildResult + if st.State == delivery.StepPassed { + _ = json.Unmarshal(st.Result, &res) + } + return res, st + } + store := withStore(t, m) + args := map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "go"} + cand, st := run(args) + if st.State != delivery.StepPassed || !strings.HasPrefix(cand.Digest, "sha256:") || cand.Reference != "" || len(store.asked) != 0 { + t.Fatalf("a candidate's Go package: %s %q, %+v, asked %v", st.State, st.Reason, cand, store.asked) + } + args["version"], args["seat"] = "v0.1.15", GoPackageSeat + res, st := run(args) + if st.State != delivery.StepPassed || res.Reference != "git.example/demo/go@v0.1.15" || !strings.HasPrefix(res.Sum, "h1:") { + t.Fatalf("a trunk Go package: %s %q, %+v", st.State, st.Reason, res) + } + if len(store.published) != 1 { + t.Fatalf("published %+v", store.published) + } + in := store.published[0] + if in.Name != "git.example/demo/go" || in.Version != "v0.1.15" || in.Kind != "package" || in.Digest != res.Digest { + t.Errorf("the publish: %+v", in) + } + z, err := zip.NewReader(bytes.NewReader(store.blobs[in.Name+"@"+in.Digest]), int64(len(store.blobs[in.Name+"@"+in.Digest]))) + if err != nil { + t.Fatal(err) + } + var names []string + for _, f := range z.File { + names = append(names, f.Name) + } + if strings.Join(names, " ") != "git.example/demo/go@v0.1.15/a.go git.example/demo/go@v0.1.15/go.mod" { + t.Errorf("the zip holds %v", names) + } + if err := os.Remove(filepath.Join(src, "go.mod")); err != nil { + t.Fatal(err) + } + if _, st := run(args); st.State != delivery.StepFailed || !strings.Contains(st.Reason, "go.mod") { + t.Errorf("no go.mod: %s %q", st.State, st.Reason) + } +} diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 272415a7..fa05d2ec 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -9,6 +9,7 @@ require ( github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 github.com/therootcompany/xz v1.0.1 + golang.org/x/mod v0.42.0 ) require ( diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index 52c1ca63..32a87eb9 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -178,3 +178,5 @@ google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 1a0d198b..b12732d2 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -74,7 +74,9 @@ }, "invokes": [ "seat:oci-artifact-registry.publish", - "seat:oci-artifact-registry.publish-part" + "seat:oci-artifact-registry.publish-part", + "seat:go-package-registry.publish", + "seat:go-package-registry.publish-part" ], "own-secrets": { "broker": "${dir:state}/broker"