diff --git a/modules/mesh-build-go/buildstep/publish.go b/modules/mesh-build-go/buildstep/publish.go new file mode 100644 index 000000000..647a8b47c --- /dev/null +++ b/modules/mesh-build-go/buildstep/publish.go @@ -0,0 +1,46 @@ +package buildstep + +// The publish of a done build's file through a registry seat (novox/hq design 51 *Publishing*, ADR 0611; CAT-BG1, +// CAT-BN3): from the build module's own process, after the build step ended, through the seat's verbs publish and +// publish-part, which each build module's manifest `invokes`, in chunks, only what the holder lacks. The holder checks +// the build with the builder and says published or refused on the seat's events. One place for every build module, so +// the move to the registry clients is made here once. + +import ( + "bytes" + "context" + "encoding/json" + "io" + "strings" + + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + "git.novox.be/novox/mesh-sdk/go/publish" +) + +// The registry seats a build module publishes on. +const ( + ArtifactSeat = "oci-artifact-registry" + GoSeat = "go-package-registry" + NpmSeat = "npm-package-registry" +) + +// SeatCall asks one verb of a seat, `seat:.`, with a body, and answers its JSON (the bundle's stdio.Ask). +type SeatCall = func(key string, body any) (json.RawMessage, error) + +// TransferName is a transfer's name, the build module's own: the name, a qualifier and the digest's head, safe as a +// transfer id. +func TransferName(name, qualifier, digest string) string { + head := strings.TrimPrefix(digest, "sha256:") + if len(head) > 16 { + head = head[:16] + } + return strings.NewReplacer("/", "-", ".", "-", " ", "-", "*", "-", ">", "-", "@", "-").Replace(name + "-" + qualifier + "-" + head) +} + +// PublishOne publishes body, of digest, as name (at version, when it has one) of kind on seat, in one part. +func PublishOne(ctx context.Context, call SeatCall, seat, name, kind, version, digest, transfer string, body []byte) error { + in := reg.PublishInput{Name: name, Kind: kind, Version: version, Digest: digest, Transfer: transfer} + _, err := publish.Publish(ctx, publish.Call(call), seat, in, []publish.Part{{Digest: digest, Size: int64(len(body)), + Open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(body)), nil }}}) + return err +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/publish.go b/modules/mesh-build-go/cmd/mesh-build-go/publish.go index c5f638a6e..993a045f7 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/publish.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/publish.go @@ -2,32 +2,25 @@ package main // The publish of a done build's archive (novox/hq design 51 *Publishing*, ADR 0611; CAT-BG1): from this module's own // process, after the build step ended, through the seat oci-artifact-registry's verbs publish and publish-part, which -// the manifest `invokes`, with mesh-sdk's one sender (publish.Publish): the archive in chunks, only what the holder +// the manifest `invokes`, with the build modules' one sender (buildstep.PublishOne): the archive in chunks, only what the holder // lacks. No build step reaches the store, and this module no longer writes into it directly (push.go is gone): the // holder checks the build with the builder and says published or refused on the seat's events. import ( - "bytes" "context" "encoding/json" - "io" - "strings" - registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/publish" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" ) // ArtifactSeat is the registry seat a bundle's or an archive's archive is published on. -const ArtifactSeat = "oci-artifact-registry" +const ArtifactSeat = buildstep.ArtifactSeat -// transferName is the transfer's name, this module's own: the artifact and its digest's head, safe as a transfer id. +// transferName is the transfer's name, this module's own (buildstep.TransferName). func transferName(module, artifact, digest string) string { - head := strings.TrimPrefix(digest, "sha256:") - if len(head) > 16 { - head = head[:16] - } - return strings.NewReplacer("/", "-", ".", "-", " ", "-", "*", "-", ">", "-").Replace(module + "-" + artifact + "-" + head) + return buildstep.TransferName(module, artifact, digest) } // publishArchive publishes body, of digest, as / of kind (bundle or archive) on the artifact seat. @@ -35,17 +28,14 @@ func (m *Module) publishArchive(ctx context.Context, module, artifact, kind, dig return m.publishOn(ctx, ArtifactSeat, module+"/"+artifact, kind, "", digest, transferName(module, artifact, digest), body) } -// publishOn publishes body, of digest, as name (at version, when it has one) of kind on seat, in one part: through Call -// when a test set it, else the runtime's request over the bus (stdio.Ask). +// publishOn publishes body through buildstep.PublishOne: through Call when a test set it, else the runtime's request +// over the bus (stdio.Ask). func (m *Module) publishOn(ctx context.Context, seat, name, kind, version, digest, transfer string, body []byte) error { call := m.Call if call == nil { call = stdio.Ask } - in := registry.PublishInput{Name: name, Kind: kind, Version: version, Digest: digest, Transfer: transfer} - _, err := publish.Publish(ctx, publish.Call(call), seat, in, []publish.Part{{Digest: digest, Size: int64(len(body)), - Open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(body)), nil }}}) - return err + return buildstep.PublishOne(ctx, call, seat, name, kind, version, digest, transfer, body) } // callFunc is the runtime's request and reply over the bus, as stdio.Ask makes it. diff --git a/modules/mesh-build-node/README.md b/modules/mesh-build-node/README.md index 9f1b1a54b..c0307486e 100644 --- a/modules/mesh-build-node/README.md +++ b/modules/mesh-build-node/README.md @@ -24,9 +24,9 @@ the module runs `npm ci --ignore-scripts` there once (a directory without its `. **How the registry is reached** (the choice of issue 1488): the lock holds @novox/mesh-sdk at the forge's public address, and npm 12 installs a locked tarball only from the registry it is configured with. The module therefore -requires `npm-package-registry` (read access, provided by gitea) and, like the lab module, takes its binding and sealed -secret as files in its home. The install gets a private npm config file with the `@novox` scope at the binding's -`public-root` (else scheme, address, port and npm path) and the bound account's credential, and removes it afterwards; +requires `npm-package-registry` (provided by gitea, form none: reads are anonymous) and, like the lab module, takes its +binding as a file in its home. The install gets a private npm config file with the `@novox` scope at the binding's +`public-root` (else scheme, address, port and npm path), with no credential, and removes it afterwards; every other package resolves from the public registry. This is the way today's build agent names it. The runtime tree for bundles with externals (the pruned sdk and nats today in `/app/runtime`) is not installed @@ -83,14 +83,16 @@ toolchain install: a long step answers at once with its id and ends on `.< toolchain's packages (not typescript or esbuild) copied to `stage-/node_modules`: the build agent's depth (`/app/modules/module`, `/app/node_modules`), so esbuild's `../../node_modules/...` path comments, and the bundle's digest, equal its. Copies, not links, because the mesh makes no symlink. A bundle that keeps an external gets them in its own `node_modules`. - Once the build ended, this process pushes the packed archive to the artifact store (novox/hq ADR 0498 §1 §2, mesh-sdk's - `buildstep.PushBlob`, the address in `${dir:state}/registry`) under `/` by its digest, a candidate's as a - trunk build's, and answers `reference: artifact-store:////blobs/`; a push that fails fails the - step `not stored: : `. The packed bytes never stay on the node. tsc and esbuild, like the + Once the build ended, this process publishes the packed archive through the seat `oci-artifact-registry` (design 51 + *Publishing*; buildstep's one sender, `buildstep.PublishOne`, the manifest `invokes` publish and publish-part) under + `/` by its digest, a candidate's as a trunk build's, and answers the seat's reference + `oci-artifact-registry//@`; a publish the holder refuses, or a seat nothing holds, fails the + step `not stored: : `. The packed bytes never stay on the node. tsc and esbuild, like the type-check and the test script, run shut off in bubblewrap (ADR 0448): no network, none of the account, the toolchain read-only, the npm cache beneath a throwaway layer unless the step is told `publish` (a trunk build). A `package` artifact (ADR 0498 §5) is built by its own `build` script and packed with `npm pack --ignore-scripts`, both - shut off; its digest is the packed file's. Only with `publish` is it published, by this process with the registry's - write credential (never in a workspace or a step's mounts): a version the registry lacks is published, the same - packed file already there is done without a publish, another file under that version fails `version taken: - @`; the reference `@` is answered only then. A candidate answers its digest alone. + shut off; its digest is the packed file's. A trunk build the builder gave a version packs it under that version (its + package.json's `version` set for the pack, then put back) and publishes it through `npm-package-registry`; the holder + asks the builder, answers held for the same file and refuses another under that version. The reference + `@` is answered only then. A candidate, or a trunk build not numbered, answers its digest alone. + The module holds no credential for the registry: reads are anonymous (ADR 0346) and writes go through the seat. diff --git a/modules/mesh-build-node/cmd/mesh-build-node/build.go b/modules/mesh-build-node/cmd/mesh-build-node/build.go index 4627c2c7a..e6a4750dc 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/build.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/build.go @@ -19,6 +19,7 @@ import ( "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" sdkstep "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -88,7 +89,9 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { return nil, err } if a.Kind == "package" { - return m.buildPackage(ctx, workspace, module, root, dir, a, publish) + version, _ := args["version"].(string) + seat, _ := args["seat"].(string) + return m.buildPackage(ctx, workspace, module, root, dir, a, publish, version, seat) } if why, err := lockProblem(dir, module); err != nil || why != "" { if err == nil { @@ -139,9 +142,9 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { if err != nil { return nil, fmt.Errorf("packing %s: %w", a.Name, err) } - // Pushed from this process once the build ended, a candidate's as a trunk build's (ADR 0498 §1 §2): the step answers - // the reference, or fails not stored. The packed bytes are never written to the node (the lead's decision L9). - if err := m.push(ctx, a.module+"/"+a.Name, digest, archive); err != nil { + // Published from this process once the build ended, a candidate's as a trunk build's (ADR 0498 §1 §2; CAT-BN3): the + // step answers the seat's reference, or fails not stored. The packed bytes are never written to the node (L9). + if err := m.publishOn(ctx, buildstep.ArtifactSeat, a.module+"/"+a.Name, "bundle", "", digest, archive); err != nil { return nil, fmt.Errorf("%s%s: %v", delivery.NotStored, a.Name, err) } files := []string{} @@ -154,7 +157,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { }) sort.Strings(files) return BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: digest, Bytes: len(archive), - Reference: buildstep.BlobReference(a.module, a.Name, digest), Files: files}, nil + Reference: catalogue.SeatReference(buildstep.ArtifactSeat, a.module, a.Name, digest), Files: files}, nil } // bundle makes every entrypoint and every launcher ONE file, with esbuild (builder.go:1971-2039): an entrypoint is diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go index ee023a450..c8d58beb4 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go @@ -192,7 +192,7 @@ func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *test files["modules/m/package-lock.json"] = `{}` m, f := fetchMachine(t, files, map[string]string{"@novox/other/index.js": "o", "lodash/index.js": "l"}) m.Toolchain.Registry = func() (Registry, error) { - return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "s3cret-pw!"}, nil + return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil } var npmrc string inner := f.effect @@ -209,20 +209,17 @@ func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *test if err != nil { t.Fatal(err) } - if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/") || !strings.Contains(npmrc, ":_auth=") { + // Reads are anonymous (ADR 0346, CAT-BN3): the config names the scope's registry and no credential. + if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/") || strings.Contains(npmrc, "_auth") { t.Errorf("npm ci was given the config %q", npmrc) } - // The password holds characters a workspace's random name never does, so a name like "mvpwby…" is no false match. - if strings.Contains(strings.Join(npmCalls(f)[0].args, " "), "s3cret-pw!") { - t.Errorf("the credential is on npm's command line") - } r := out.(FetchResult) if _, err := os.Stat(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "node_modules", "@novox", "other", "index.js")); err != nil { t.Errorf("@novox/other was not installed: %v", err) } for _, e := range mustReadDir(t, filepath.Join(m.Work, r.Workspace)) { if strings.HasPrefix(e, "npm-") { - t.Errorf("the scratch install (and its credential file) is left: %s", e) + t.Errorf("the scratch install is left: %s", e) } } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/isolate.go b/modules/mesh-build-node/cmd/mesh-build-node/isolate.go index f4fceff44..b00afcc35 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/isolate.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/isolate.go @@ -2,7 +2,7 @@ package main // The run steps shut off (novox/hq ADR 0448; issue 1309, lead decision L2 of R4e): the type-check, the package's test // script, tsc and esbuild run the change's code, so each runs through mesh-sdk's buildstep.Isolated, in bubblewrap: no -// network, none of the account (its home, its ssh key, its registry secret, its bus login, the runtime directory), the +// network, none of the account (its home, its ssh key, its bus login, the runtime directory), the // workspace read-write, the toolchain read-only, the shared npm cache read-only beneath a throwaway writable layer, or // read-write for a trunk build alone (publish). The fetch step (git, npm ci of the module's own packages) holds the key // and the network and runs none of the change's code, so it runs as before. diff --git a/modules/mesh-build-node/cmd/mesh-build-node/main.go b/modules/mesh-build-node/cmd/mesh-build-node/main.go index 8f2e338d9..60dd9c8ec 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/main.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/main.go @@ -36,7 +36,7 @@ func main() { } paths := Paths{Home: home, Cache: CacheDir} registry := func() (Registry, error) { - return ReadRegistry(filepath.Join(home, "package-registry.json"), filepath.Join(home, "package-registry.secret")) + return ReadRegistry(filepath.Join(home, "package-registry.json")) } m := &Module{ Node: strings.TrimSpace(os.Getenv("MESH_NODE")), @@ -55,7 +55,6 @@ func main() { Ctx: ctx, Say: logf, Bubblewrap: ProbeBubblewrap, - RegistryFile: filepath.Join(home, "registry"), } m.Floor = floorOf(m.Node) m.Settings = filepath.Join(home, "settings.json") diff --git a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go index 3da4725cd..5987c4ab8 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go @@ -213,37 +213,45 @@ func TestTheModuleDeclaresBubblewrapAndTheCapabilityItNeeds(t *testing.T) { } } -// 1587-S5 (ADR 0498 §2): the module requires the artifact store and is told its address in a file of its home, which no -// run step is given (a run step has no network either, ADR 0448). -func TestTheModuleRequiresTheArtifactStoreAndIsToldItsAddress(t *testing.T) { +// CAT-BN3 (design 51 *Publishing*, ADR 0346): the module publishes only through the registry seats' verbs, which it +// invokes; it requires no store, places no store's address, and holds no credential for the package registry: reads +// are anonymous (the provision's form none), so it contributes nothing to it and is sealed no secret. +func TestTheModuleInvokesTheRegistrySeatsAndHoldsNoWriteCredential(t *testing.T) { m := readManifest(t) - if !slices.Contains(m.Requires, "artifact-store") { - t.Errorf("requires %v lack artifact-store", m.Requires) + for _, seat := range []string{"oci-artifact-registry", "npm-package-registry"} { + for _, verb := range []string{"publish", "publish-part"} { + if !slices.Contains(m.Invokes, "seat:"+seat+"."+verb) { + t.Errorf("invokes %v lack the seat %s's %s", m.Invokes, seat, verb) + } + } } - // Decoding leaves the template (files/registry) unread in the resource, so its text is read here. - r := resource(t, m, "registry") - content, err := os.ReadFile(filepath.Join(moduleDir, "files", "registry")) - if err != nil || r["type"] != "file" || r["path"] != "${dir:state}/registry" || r["template"] != "files/registry" || - !strings.Contains(string(content), "${seat:mesh-artifact-store:reach}") { - t.Errorf("the registry resource is %v (template %q, %v)", r, content, err) + if slices.Contains(m.Requires, "artifact-store") { + t.Errorf("requires %v still name artifact-store", m.Requires) + } + for _, r := range m.Resources { + if r["id"] == "registry" { + t.Errorf("the store's address is still placed: %v", r) + } + } + if _, err := os.Stat(filepath.Join(moduleDir, "files", "registry")); err == nil { + t.Error("files/registry is still shipped") } -} - -// 1587-S17 (ADR 0498 §5): the module publishes a trunk package itself, so its contribution to the package registry asks -// for write access; the credential stays with the module's own process (a run step has neither the home nor a network). -func TestTheModuleContributesWriteAccessToThePackageRegistry(t *testing.T) { raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) if err != nil { t.Fatal(err) } var mj struct { - Contributes map[string]map[string]any `json:"contributes"` + Contributes map[string]any `json:"contributes"` + Secrets map[string]string `json:"secrets"` } if err := json.Unmarshal(raw, &mj); err != nil { t.Fatal(err) } - if got := mj.Contributes["npm-package-registry"]["access"]; got != "write" { - t.Errorf("npm-package-registry access = %v, want write", got) + if _, ok := mj.Contributes["npm-package-registry"]; ok { + t.Errorf("contributes %v still ask the package registry for access", mj.Contributes) + } + if _, ok := mj.Secrets["npm-package-registry"]; ok { + t.Errorf("secrets %v still seal the package registry's credential", mj.Secrets) } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/module.go b/modules/mesh-build-node/cmd/mesh-build-node/module.go index 1932be8c8..92075c579 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/module.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/module.go @@ -57,10 +57,9 @@ type Module struct { Say func(format string, a ...any) // HTTP reads and publishes to the package registry; a client of its own when nil. A test serves its own. HTTP *http.Client - // RegistryFile holds the artifact store's address (host:port), placed by the manifest; Push puts a blob there - // (buildstep.PushBlob when nil; a test's fake store). - RegistryFile string - Push func(ctx context.Context, registry, repository, digest string, body []byte) error + // Call asks a registry seat's verb over the bus, the publish of a done build (publish.go): stdio.Ask when nil; a + // test's fake holder. + Call buildstep.SeatCall // Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is // unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing. Bubblewrap func(ctx context.Context) (string, error) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/package.go b/modules/mesh-build-node/cmd/mesh-build-node/package.go index 0663f180a..cbf214e3f 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/package.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/package.go @@ -97,9 +97,8 @@ func (m *Module) installDev(ctx context.Context, root, dir, rel string) error { return writeShims(filepath.Join(dir, "node_modules"), shimDir(root, rel)) } -// npmrcIn writes the npm configuration for the mesh's registry, with its credential, for one command of this process; -// the caller removes it. It lies in this process's temporary directory, never in a workspace: a run step is given the -// workspace and a /tmp of its own, so no step's mounts hold the credential (ADR 0498 §5, ADR 0500 §4). +// npmrcIn writes the npm configuration for the mesh's registry, read anonymously, for one command of this process; the +// caller removes it. It lies in this process's temporary directory, never in a workspace (ADR 0498 §5, ADR 0500 §4). func (m *Module) npmrcIn(root string) (string, error) { if m.Toolchain == nil || m.Toolchain.Registry == nil { return "", fmt.Errorf("the module has no package registry") @@ -188,7 +187,14 @@ func writeShims(nm, into string) error { // buildAndPack builds the package in dir with its own build script, if it has one, and packs it with npm pack, both // shut off (write: a trunk build's, which alone writes the shared cache). The packed file is read and removed: it never // stays on the node. -func (m *Module) buildAndPack(ctx context.Context, root, dir, rel string, write bool) (packed, error) { +func (m *Module) buildAndPack(ctx context.Context, root, dir, rel string, write bool, version string) (packed, error) { + if version != "" { + restore, err := setPackageVersion(dir, version) + if err != nil { + return packed{}, err + } + defer restore() + } p, err := readPackageJSON(dir) if err != nil { return packed{}, err @@ -259,7 +265,7 @@ func (m *Module) packGroup(ctx context.Context, root string, run forgekey.Runner if err := m.installDev(ctx, root, dir, "package"); err != nil { return packed{}, err } - return m.buildAndPack(ctx, root, dir, "package", false) + return m.buildAndPack(ctx, root, dir, "package", false, "") } // sdkLocked is what a module's lock names of the SDK: its version, its integrity and where it was resolved from; ok is @@ -356,7 +362,6 @@ func (m *Module) registryVersion(ctx context.Context, name, version string) (tar if err != nil { return "", "", false, err } - reg.authorize(req) resp, err := m.httpClient().Do(req) if err != nil { return "", "", false, fmt.Errorf("asking the registry for %s: %w", name, err) @@ -386,15 +391,10 @@ func (m *Module) registryPackage(ctx context.Context, name, version string) ([]b if err != nil || !held { return nil, held, err } - reg, err := m.Toolchain.Registry() - if err != nil { - return nil, false, err - } req, err := http.NewRequestWithContext(ctx, http.MethodGet, tarball, nil) if err != nil { return nil, false, err } - reg.authorize(req) resp, err := m.httpClient().Do(req) if err != nil { return nil, false, fmt.Errorf("downloading %s@%s: %w", name, version, err) @@ -465,63 +465,60 @@ func fetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, return buildstep.FetchCommit(ctx, run, dir, remote, commit, ref, m.FetchOptions...) } -// buildPackage builds and packs a package artifact (ADR 0498 §5): its digest is the packed file's. Only a trunk build -// (publish) publishes it, from this module's own process with the registry's write credential, which no run step is -// given: a version the registry lacks is published; the same packed file already there is done without a publish; -// another packed file under that version fails `version taken:`. The reference is @, answered only when -// the registry holds this build's file; a candidate answers its digest alone. -func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir string, a artifact, publish bool) (any, error) { +// buildPackage builds and packs a package artifact (ADR 0498 §5): its digest is the packed file's. A trunk build +// (publish) the builder gave a version packs it under that version and publishes it through npm-package-registry +// (design 51 *Publishing*, CAT-BN3), from this module's own process, which holds no write credential: the holder asks +// the builder, answers held for the same file and refuses another under that version. The reference is +// @, answered only when the holder took it; a candidate, or a trunk build not numbered, answers its +// digest alone. +func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir string, a artifact, publish bool, version, seat string) (any, error) { rel := module if rel == "" { rel = "." } - p, err := m.buildAndPack(ctx, root, dir, rel, publish) + if !publish { + version = "" + } + if seat == "" { + seat = buildstep.NpmSeat + } + if version != "" && seat != buildstep.NpmSeat { + return nil, fmt.Errorf("the package %s is published through %s, not %s: this module publishes npm packages", a.Name, buildstep.NpmSeat, seat) + } + p, err := m.buildAndPack(ctx, root, dir, rel, publish, version) if err != nil { return nil, err } res := BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: p.Digest, Bytes: len(p.tgz)} - if publish { - if res.Reference, err = m.publish(ctx, root, p); err != nil { - return nil, err - } + if version == "" { + return res, nil } + if err := m.publishOn(ctx, seat, p.Name, "package", p.Version, p.Digest, p.tgz); err != nil { + return nil, fmt.Errorf("%s%s: %v", delivery.NotStored, a.Name, err) + } + res.Reference = p.Name + "@" + p.Version return res, nil } -// publish puts a packed package into the registry unless it holds that version, and answers @. -func (m *Module) publish(ctx context.Context, root string, p packed) (string, error) { - ref := p.Name + "@" + p.Version - _, integrity, held, err := m.registryVersion(ctx, p.Name, p.Version) +// setPackageVersion writes version as dir's package.json `version` (the builder's, design 51 *The versions*), so npm +// packs the package under it; restore puts the file back as it was. +func setPackageVersion(dir, version string) (restore func(), err error) { + path := filepath.Join(dir, "package.json") + raw, err := os.ReadFile(path) if err != nil { - return "", err + return nil, fmt.Errorf("the package's package.json: %w", err) } - if held { - if integrity == p.Integrity { - return ref, nil // this very file, published before: done without a publish - } - return "", fmt.Errorf("%s%s: the registry holds it as %s, and this build packs %s", delivery.VersionTaken, ref, integrity, p.Integrity) + var doc map[string]json.RawMessage + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("the package's package.json does not parse: %w", err) } - file, err := os.CreateTemp(root, "publish-*.tgz") + doc["version"], _ = json.Marshal(version) + out, err := json.MarshalIndent(doc, "", " ") if err != nil { - return "", err + return nil, err } - defer os.Remove(file.Name()) // the packed file leaves the node once its publish answered - if _, err := file.Write(p.tgz); err != nil { - file.Close() - return "", err + if err := os.WriteFile(path, append(out, '\n'), 0o644); err != nil { + return nil, err } - if err := file.Close(); err != nil { - return "", err - } - npmrc, err := m.npmrcIn(root) - if err != nil { - return "", err - } - defer os.Remove(npmrc) - env := append(m.baseEnv(), "npm_config_cache="+m.npmCache()) - r := m.Run(ctx, env, "npm", "publish", file.Name(), "--ignore-scripts", "--userconfig", npmrc) - if why := ranFailed(r); why != "" { - return "", fmt.Errorf("publishing %s: %s", ref, why) - } - return ref, nil + return func() { _ = os.WriteFile(path, raw, 0o644) }, nil } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/publish_integration_test.go b/modules/mesh-build-node/cmd/mesh-build-node/publish_integration_test.go index c8420b882..6f7d57106 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/publish_integration_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/publish_integration_test.go @@ -4,82 +4,18 @@ package main import ( "context" - "encoding/base64" - "encoding/json" - "io" - "net/http" - "net/http/httptest" + "os" "os/exec" "path/filepath" "strings" - "sync" "testing" "git.novox.be/novox/mesh-sdk/go/forgekey" ) -// npmRegistry is a registry of the test's own speaking npm's publish protocol: a PUT of the package's document with the -// packed file attached, then GET of the document and the file; it asks for the account's credential. -type npmRegistry struct { - mu sync.Mutex - versions map[string]map[string]any - files map[string][]byte - publishes int -} - -func (r *npmRegistry) serve(t *testing.T) *httptest.Server { - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, q *http.Request) { - r.mu.Lock() - defer r.mu.Unlock() - if u, p, ok := q.BasicAuth(); !ok || u != "builder" || p != "the write credential" { - w.WriteHeader(http.StatusUnauthorized) - return - } - doc := strings.TrimPrefix(q.URL.EscapedPath(), "/npm/") - switch { - case q.Method == http.MethodPut && (doc == "@novox%2fmesh-sdk" || doc == "@novox%2Fmesh-sdk"): - var body struct { - Versions map[string]map[string]any `json:"versions"` - Attachments map[string]struct { - Data string `json:"data"` - } `json:"_attachments"` - } - raw, _ := io.ReadAll(q.Body) - if err := json.Unmarshal(raw, &body); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - for v, meta := range body.Versions { - for name, a := range body.Attachments { - data, _ := base64.StdEncoding.DecodeString(a.Data) - r.files[name] = data - meta["dist"] = map[string]any{"tarball": srv.URL + "/files/" + name, "integrity": integrityOf(data)} - } - r.versions[v] = meta - } - r.publishes++ - w.WriteHeader(http.StatusCreated) - w.Write([]byte(`{"ok":true}`)) - case q.Method == http.MethodGet && (doc == "@novox%2fmesh-sdk" || doc == "@novox%2Fmesh-sdk"): - if len(r.versions) == 0 { - http.NotFound(w, q) - return - } - json.NewEncoder(w).Encode(map[string]any{"name": "@novox/mesh-sdk", "versions": r.versions}) - case strings.HasPrefix(q.URL.Path, "/files/"): - w.Write(r.files[strings.TrimPrefix(q.URL.Path, "/files/")]) - default: - http.NotFound(w, q) - } - })) - t.Cleanup(srv.Close) - return srv -} - -// 1587-S20 (ADR 0498 §5): the real npm publishes a trunk build's package to a registry speaking npm's protocol, with -// the module's credential, once: a second trunk build of the same package finds its file there and publishes nothing; -// a candidate publishes nothing at all. +// 1587-S20 (ADR 0498 §5), CAT-BN3 (design 51 *Publishing*): the real npm packs a trunk build's package under the +// version the builder gave it, and the module publishes that packed file through npm-package-registry's verbs, with no +// credential and no npm publish; the workspace's package.json is put back. A candidate publishes nothing at all. func TestARealTrunkPackageIsPublishedOnce(t *testing.T) { needBubblewrap(t) for _, tool := range []string{"node", "npm"} { @@ -89,34 +25,39 @@ func TestARealTrunkPackageIsPublishedOnce(t *testing.T) { } m, _, work := newMachine(t) m.Run = forgekey.ExecRunner - reg := &npmRegistry{versions: map[string]map[string]any{}, files: map[string][]byte{}} - srv := reg.serve(t) - m.Toolchain.Registry = func() (Registry, error) { - return Registry{URL: srv.URL + "/npm/", Username: "builder", Password: "the write credential"}, nil - } - m.HTTP = srv.Client() + store := withStore(t, m) files := map[string]string{} for n, c := range sdkFixture { files[n] = c } workspaceWith(t, work, files) dir := filepath.Join(work, "w1", "tree") + before, _ := os.ReadFile(filepath.Join(dir, "package.json")) if err := m.installDev(context.Background(), filepath.Join(work, "w1"), dir, "."); err != nil { t.Fatal(err) } args := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib"} - if out, err := m.build(context.Background(), args); err != nil || out.(BuildResult).Reference != "" || reg.publishes != 0 { - t.Fatalf("a candidate: %+v %v, %d publishes", out, err, reg.publishes) + if out, err := m.build(context.Background(), args); err != nil || out.(BuildResult).Reference != "" || len(store.published) != 0 { + t.Fatalf("a candidate: %+v %v, %d published", out, err, len(store.published)) } - args["publish"] = true + args["publish"], args["version"], args["seat"] = true, "9.9.10", "npm-package-registry" out, err := m.build(context.Background(), args) - if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@9.9.9" || reg.publishes != 1 { - t.Fatalf("a trunk build: %+v %v, %d publishes", out, err, reg.publishes) + if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@9.9.10" || len(store.published) != 1 { + t.Fatalf("a trunk build: %+v %v, %d published", out, err, len(store.published)) } - if got := digestOf(reg.files["@novox/mesh-sdk-9.9.9.tgz"]); got != out.(BuildResult).Digest { - t.Errorf("the registry holds %s, the build packed %s (files %d)", got, out.(BuildResult).Digest, len(reg.files)) + p := store.published[0] + tgz := store.blobs[p.Name+"@"+p.Digest] + if p.Name != "@novox/mesh-sdk" || p.Version != "9.9.10" || p.Kind != "package" || digestOf(tgz) != out.(BuildResult).Digest { + t.Errorf("published %+v (%d bytes), the build packed %s", p, len(tgz), out.(BuildResult).Digest) } - if again, err := m.build(context.Background(), args); err != nil || again.(BuildResult).Reference != "@novox/mesh-sdk@9.9.9" || reg.publishes != 1 { - t.Errorf("the same package again: %+v %v, %d publishes", again, err, reg.publishes) + into := t.TempDir() + if err := unpackPackage(tgz, into); err != nil { + t.Fatal(err) + } + if raw, _ := os.ReadFile(filepath.Join(into, "package.json")); !strings.Contains(string(raw), `"version": "9.9.10"`) { + t.Errorf("the packed package.json is %s, want the builder's version", raw) + } + if after, _ := os.ReadFile(filepath.Join(dir, "package.json")); string(after) != string(before) { + t.Errorf("the workspace's package.json was not put back: %s", after) } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/publish_test.go b/modules/mesh-build-node/cmd/mesh-build-node/publish_test.go index e1ab72cb1..54a0b70a8 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/publish_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/publish_test.go @@ -2,9 +2,9 @@ package main import ( "context" + "errors" "os" "path/filepath" - "slices" "strings" "testing" @@ -69,26 +69,37 @@ func TestACandidatesPackageIsBuiltAndPackedAndNeverPublished(t *testing.T) { } } -// ADR 0498 §5: a trunk build (publish) publishes a version the registry lacks and answers @; the same -// packed file already there is done without a publish; another file under that version fails `version taken:`. The -// packed file is gone from the node afterwards. +// ADR 0498 §5, CAT-BN3 (design 51 *Publishing*): a trunk build the builder gave a version packs the package under that +// version and publishes it through npm-package-registry, never by npm publish nor with a credential, and answers +// @; its package.json is put back. A trunk build not numbered publishes nothing; a refusal fails it +// not stored. The packed file is gone from the node afterwards. func TestATrunkPackageIsPublishedOnceAndAVersionTakenIsRefused(t *testing.T) { - tgz := tgzOf(t, sdkPackage, "0.1.15") - args := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib", "publish": true} + tgz := tgzOf(t, sdkPackage, "0.1.16") + args := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib", "publish": true, "version": "0.1.16", "seat": "npm-package-registry"} m, f, work := packageMachine(t, tgz) - fakeRegistry(t, m, nil) - out, err := m.build(context.Background(), args) - if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@0.1.15" || len(publishes(f)) != 1 { - t.Fatalf("an absent version: %+v %v, %d publishes", out, err, len(publishes(f))) - } - if p := publishes(f)[0]; !slices.Contains(p.args, "--ignore-scripts") || !strings.HasSuffix(p.args[1], ".tgz") { - t.Errorf("published with %v", p.args) - } - // ADR 0500 §4: the registry's credential is never in a workspace, which a run step is given. - for i, a := range publishes(f)[0].args { - if a == "--userconfig" && strings.HasPrefix(publishes(f)[0].args[i+1], work) { - t.Errorf("the credential was written into the workspace: %s", publishes(f)[0].args[i+1]) + var packedAs string + inner := f.effect + f.effect = func(c call) { + if c.name == "npm" && len(c.args) > 0 && c.args[0] == "pack" { + raw, _ := os.ReadFile(filepath.Join(work, "w1", "tree", "package.json")) + packedAs = string(raw) } + inner(c) + } + store := withStore(t, m) + out, err := m.build(context.Background(), args) + if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@0.1.16" || len(publishes(f)) != 0 { + t.Fatalf("a numbered trunk package: %+v %v, %d npm publishes", out, err, len(publishes(f))) + } + if !strings.Contains(packedAs, `"version": "0.1.16"`) { + t.Errorf("packed with package.json %s, want the builder's version", packedAs) + } + if raw, _ := os.ReadFile(filepath.Join(work, "w1", "tree", "package.json")); !strings.Contains(string(raw), `"version":"0.1.15"`) { + t.Errorf("package.json was not put back: %s", raw) + } + if len(store.published) != 1 || store.published[0].Name != "@novox/mesh-sdk" || store.published[0].Version != "0.1.16" || + store.published[0].Kind != "package" || store.seats[0] != "npm-package-registry" || store.published[0].Digest != digestOf(tgz) { + t.Errorf("published %+v through %v", store.published, store.seats) } var left []string filepath.WalkDir(work, func(p string, d os.DirEntry, err error) error { @@ -102,15 +113,15 @@ func TestATrunkPackageIsPublishedOnceAndAVersionTakenIsRefused(t *testing.T) { } m, f, _ = packageMachine(t, tgz) - fakeRegistry(t, m, map[string][]byte{"0.1.15": tgz}) - if out, err := m.build(context.Background(), args); err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@0.1.15" || len(publishes(f)) != 0 { - t.Errorf("the same file published before: %+v %v, %d publishes", out, err, len(publishes(f))) + store = withStore(t, m) + unnumbered := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib", "publish": true} + if out, err := m.build(context.Background(), unnumbered); err != nil || out.(BuildResult).Reference != "" || len(store.published) != 0 || len(publishes(f)) != 0 { + t.Errorf("a trunk package not numbered: %+v %v, %d published", out, err, len(store.published)) } - m, f, _ = packageMachine(t, tgz) - fakeRegistry(t, m, map[string][]byte{"0.1.15": tgzOf(t, sdkPackage, "another")}) - _, err = m.build(context.Background(), args) - if err == nil || !strings.HasPrefix(err.Error(), delivery.VersionTaken+"@novox/mesh-sdk@0.1.15") || len(publishes(f)) != 0 { - t.Errorf("another file under the version: %v, %d publishes", err, len(publishes(f))) + m, _, _ = packageMachine(t, tgz) + withStore(t, m).fail = errors.New("refused: the builder did not number it") + if _, err = m.build(context.Background(), args); err == nil || !strings.HasPrefix(err.Error(), delivery.NotStored+"lib: ") { + t.Errorf("a refused publish: %v", err) } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/push.go b/modules/mesh-build-node/cmd/mesh-build-node/push.go deleted file mode 100644 index fb468a033..000000000 --- a/modules/mesh-build-node/cmd/mesh-build-node/push.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -// The push of a done build's archive (novox/hq ADR 0498 §1 §2; issue 1587): from this module's own process, after the -// build step ended, with mesh-sdk's one client (buildstep.PushBlob), to the artifact store whose address the manifest -// binds into RegistryFile. No build step reaches the store: a run step has no network (ADR 0448). - -import ( - "context" - "fmt" - "net/http" - "os" - "strings" - "time" - - "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" -) - -// registry is the artifact store's address, host:port, as the manifest bound it into RegistryFile. -func (m *Module) registry() (string, error) { - raw, err := os.ReadFile(m.RegistryFile) - if err != nil { - return "", fmt.Errorf("the artifact store's address cannot be read: %w", err) - } - addr := strings.TrimSpace(string(raw)) - if addr == "" || strings.Contains(addr, "${") || strings.HasPrefix(addr, ":") || strings.HasSuffix(addr, ":") { - return "", fmt.Errorf("the artifact store's address in %s is %q, not host:port", m.RegistryFile, addr) - } - return addr, nil -} - -// push puts body into the artifact store as a blob of repository (/) by its digest: Push when a test -// set it, else buildstep.PushBlob. -func (m *Module) push(ctx context.Context, repository, digest string, body []byte) error { - addr, err := m.registry() - if err != nil { - return err - } - if m.Push != nil { - return m.Push(ctx, addr, repository, digest, body) - } - return buildstep.PushBlob(ctx, &http.Client{Timeout: 10 * time.Minute}, addr, repository, digest, body) -} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go index 50cbe7e97..df343ce8c 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go @@ -3,7 +3,9 @@ package main import ( "context" "crypto/sha256" + "encoding/base64" "encoding/hex" + "encoding/json" "errors" "os" "path/filepath" @@ -11,54 +13,87 @@ import ( "sync" "testing" + registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" "git.novox.be/novox/mesh-sdk/go/delivery" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" ) -// pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back. +// pushed are the blobs the fake holder took, by digest, for the tests that read an archive back. var pushed sync.Map -// fakeStore is a store of the test's own: its address in RegistryFile, each blob kept by repository and digest; fail, -// when set, is the store's refusal. +// fakeStore is the registry seats' holder as m's publish meets it (publish.go; CAT-BN3): publish opens a transfer and +// answers the part it lacks, publish-part takes the chunks in sequence, and the last one keeps the whole part when it +// hashes to its digest. Each file is kept by name and digest; fail, when set, is the holder's refusal at the last chunk. type fakeStore struct { - mu sync.Mutex - blobs map[string][]byte // "@" - fail error + mu sync.Mutex + blobs map[string][]byte // "@" + fail error + open map[string]registry.PublishInput + chunks map[string][]byte + published []registry.PublishInput + seats []string } func withStore(t *testing.T, m *Module) *fakeStore { t.Helper() - s := &fakeStore{blobs: map[string][]byte{}} - m.RegistryFile = filepath.Join(t.TempDir(), "registry") - if err := os.WriteFile(m.RegistryFile, []byte("store.test:5000\n"), 0o644); err != nil { - t.Fatal(err) - } - m.Push = func(ctx context.Context, registry, repository, digest string, body []byte) error { + s := &fakeStore{blobs: map[string][]byte{}, open: map[string]registry.PublishInput{}, chunks: map[string][]byte{}} + m.Call = func(key string, body any) (json.RawMessage, error) { s.mu.Lock() defer s.mu.Unlock() - if s.fail != nil { - return s.fail + raw, _ := json.Marshal(body) + seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".") + if seat != buildstep.ArtifactSeat && seat != buildstep.NpmSeat { + return nil, errors.New("nats: no responders available for request") } - if sum := sha256.Sum256(body); "sha256:"+hex.EncodeToString(sum[:]) != digest || registry != "store.test:5000" { - t.Errorf("pushed %s to %s with bytes of another digest", digest, registry) + switch verb { + case "publish": + var in registry.PublishInput + _ = json.Unmarshal(raw, &in) + if (seat == buildstep.ArtifactSeat) != (in.Kind == "bundle") || in.Transfer == "" || strings.ContainsAny(in.Transfer, "/. *>") { + t.Errorf("publish on %s of kind %q, transfer %q", seat, in.Kind, in.Transfer) + } + s.open[in.Transfer] = in + s.seats = append(s.seats, seat) + return json.Marshal(registry.PublishAnswer{Transfer: in.Transfer, Lacks: []string{in.Digest}}) + case "publish-part": + var c registry.PublishPartInput + _ = json.Unmarshal(raw, &c) + data, _ := base64.StdEncoding.DecodeString(c.Data) + s.chunks[c.Transfer] = append(s.chunks[c.Transfer], data...) + if !c.Last { + return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part}) + } + in, whole := s.open[c.Transfer], s.chunks[c.Transfer] + delete(s.chunks, c.Transfer) + if s.fail != nil { + return nil, s.fail + } + if sum := sha256.Sum256(whole); "sha256:"+hex.EncodeToString(sum[:]) != in.Digest { + t.Errorf("published %s with bytes of another digest", in.Digest) + } + s.blobs[in.Name+"@"+in.Digest] = whole + s.published = append(s.published, in) + pushed.Store(in.Digest, whole) + return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part, Complete: true}) } - s.blobs[repository+"@"+digest] = append([]byte(nil), body...) - pushed.Store(digest, s.blobs[repository+"@"+digest]) - return nil + return nil, errors.New("nats: no responders available for request") } return s } -// archiveOf is the archive a build pushed, by its digest. +// archiveOf is the archive a build published, by its digest. func archiveOf(t *testing.T, r BuildResult) []byte { t.Helper() raw, ok := pushed.Load(r.Digest) if !ok { - t.Fatalf("nothing was pushed under %s", r.Digest) + t.Fatalf("nothing was published under %s", r.Digest) } return raw.([]byte) } -// 1587-S6 (ADR 0498 §1): a done bundle answers the reference of the archive it pushed, under the manifest's module. +// 1587-S6 (ADR 0498 §1; CAT-BN3): a done bundle is published through oci-artifact-registry and answers the seat's +// reference of its archive, under the manifest's module. func TestADoneBundleAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) { m, _, _ := builtMachine(t, ``) store := withStore(t, m) @@ -67,23 +102,33 @@ func TestADoneBundleAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) { t.Fatal(err) } r := out.(BuildResult) - if want := "artifact-store://m/tools/blobs/" + r.Digest; r.Reference != want { + if want := "oci-artifact-registry/m/tools@" + r.Digest; r.Reference != want { t.Errorf("reference %q, want %q", r.Reference, want) } - if store.blobs["m/tools@"+r.Digest] == nil { - t.Errorf("the store holds %d blobs, not m/tools@%s", len(store.blobs), r.Digest) + if store.blobs["m/tools@"+r.Digest] == nil || len(store.seats) != 1 || store.seats[0] != buildstep.ArtifactSeat { + t.Errorf("the holder holds %d files through %v, not m/tools@%s through %s", len(store.blobs), store.seats, r.Digest, buildstep.ArtifactSeat) } } -// 1587-S7 and S8 (ADR 0498 §2, the lead's decision L9): a push the store refuses fails the step `not stored: -// : `, and whatever the push answered, no packed file is left on the node. +// 1587-S7 and S8 (ADR 0498 §2, the lead's decision L9): a publish the holder refuses fails the step `not stored: +// : `, a seat nothing holds too, and whatever it answered, no packed file is left. func TestARefusedPushFailsNotStoredAndNoPackedFileStays(t *testing.T) { - for _, fail := range []error{nil, errors.New("store.test:5000 refused the blob: 507 Insufficient Storage")} { + for _, c := range []struct { + fail error + noSeat bool + words string + }{{nil, false, ""}, {errors.New("refused: the store did not take it: 507 Insufficient Storage"), false, "507"}, + {nil, true, "no responders"}} { m, _, work := builtMachine(t, ``) - withStore(t, m).fail = fail + withStore(t, m).fail = c.fail + if c.noSeat { + m.Call = func(string, any) (json.RawMessage, error) { + return nil, errors.New("nats: no responders available for request") + } + } _, err := m.build(context.Background(), buildArgs()) - if fail != nil && (err == nil || !strings.HasPrefix(err.Error(), delivery.NotStored+"tools: ") || !strings.Contains(err.Error(), "507")) { - t.Errorf("a refused push answered %v", err) + if c.words != "" && (err == nil || !strings.HasPrefix(err.Error(), delivery.NotStored+"tools: ") || !strings.Contains(err.Error(), c.words)) { + t.Errorf("a refused publish answered %v", err) } var archives []string filepath.WalkDir(work, func(p string, d os.DirEntry, err error) error { @@ -93,7 +138,7 @@ func TestARefusedPushFailsNotStoredAndNoPackedFileStays(t *testing.T) { return nil }) if len(archives) != 0 { - t.Errorf("push answered %v: packed files left on the node: %v", fail, archives) + t.Errorf("publish answered %v: packed files left on the node: %v", c.fail, archives) } } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/registry.go b/modules/mesh-build-node/cmd/mesh-build-node/registry.go index e7b8e8597..fa5a0720c 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/registry.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/registry.go @@ -3,16 +3,13 @@ package main // How the toolchain's install reaches the mesh's package registry (novox/hq ADR 0405 §5, ADR 0336; issue 1488). // The lock names @novox/mesh-sdk at the registry's public address, and npm installs a locked tarball only from the // registry it is configured with, so the install is told that registry for the @novox scope and nothing else: every -// other package resolves from the public one. This is how today's build agent names it (mesh-controller -// internal/builder/dependencies.go: the scope's registry from the npm-package-registry binding, the account and -// secret the mesh sealed), read here from the module's own binding and secret files (module.json `binds` and -// `secrets`), the same files the lab module reads. +// other package resolves from the public one. The address is read from the module's npm-package-registry binding +// (module.json `binds`). Reads are anonymous (ADR 0346; the provision's form none, CAT-BN3): this module holds no +// credential for the registry, and publishes only through the seat's verbs (publish.go). import ( - "encoding/base64" "encoding/json" "fmt" - "net/http" "os" "strings" ) @@ -22,19 +19,18 @@ const Scope = "@novox" // Registry is the mesh's package registry as the binding names it. type Registry struct { - URL, Username, Password string + URL string } -// ReadRegistry reads the npm-package-registry binding and its sealed secret. The registry is the binding's +// ReadRegistry reads the npm-package-registry binding. The registry is the binding's // `public-root` when it has one (the address the lock holds), and otherwise scheme://at:port plus `npm-path`. -func ReadRegistry(bindingPath, secretPath string) (Registry, error) { +func ReadRegistry(bindingPath string) (Registry, error) { raw, err := os.ReadFile(bindingPath) if err != nil { return Registry{}, fmt.Errorf("no package registry is bound to this module: %w", err) } var b struct { At string `json:"at"` - As string `json:"as"` Serves struct { Scheme string `json:"scheme"` Port int `json:"port"` @@ -59,31 +55,10 @@ func ReadRegistry(bindingPath, secretPath string) (Registry, error) { if !strings.HasSuffix(url, "/") { url += "/" } - secret, err := os.ReadFile(secretPath) - if err != nil { - return Registry{}, fmt.Errorf("the package registry's secret cannot be read: %w", err) - } - return Registry{URL: url, Username: b.As, Password: strings.TrimSpace(string(secret))}, nil + return Registry{URL: url}, nil } -// Npmrc is the npm configuration text for the install: the @novox scope at the registry, with the account's -// credential for it when there is one. +// Npmrc is the npm configuration text for the install: the @novox scope at the registry, read anonymously. func (r Registry) Npmrc() string { - var sb strings.Builder - fmt.Fprintf(&sb, "%s:registry=%s\n", Scope, r.URL) - if r.Username != "" && r.Password != "" { - hostPath := r.URL - if i := strings.Index(hostPath, "://"); i >= 0 { - hostPath = hostPath[i+1:] - } - fmt.Fprintf(&sb, "%s:_auth=%s\n", hostPath, base64.StdEncoding.EncodeToString([]byte(r.Username+":"+r.Password))) - } - return sb.String() -} - -// authorize gives a request to the registry the account's credential, when there is one. -func (r Registry) authorize(req *http.Request) { - if r.Username != "" && r.Password != "" { - req.SetBasicAuth(r.Username, r.Password) - } + return fmt.Sprintf("%s:registry=%s\n", Scope, r.URL) } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/store_integration_test.go b/modules/mesh-build-node/cmd/mesh-build-node/store_integration_test.go index 1ad3d262e..11dd4e952 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/store_integration_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/store_integration_test.go @@ -3,14 +3,16 @@ package main import ( + "bytes" "context" "crypto/sha256" "encoding/hex" + "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" - "os" - "path/filepath" + "net/url" "strings" "testing" @@ -18,12 +20,15 @@ import ( "github.com/distribution/distribution/v3/registry/handlers" _ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" + registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) // realStore starts the artifact store's own program (the distribution registry, modules/distribution) inside this -// test, with its in-memory storage, on a loopback port, and points m at it with the real push client. It is imported -// by this integration test alone: the module's program does not carry it. +// test, with its in-memory storage, on a loopback port, and plays the artifact seat's holder storing into it: the fake +// holder (push_test.go) takes the publish, and each file it took is uploaded into the store as distribution's holder +// does. It is imported by this integration test alone: the module's program does not carry it. func realStore(t *testing.T, m *Module) string { t.Helper() config := &configuration.Configuration{Storage: configuration.Storage{"inmemory": configuration.Parameters{}}} @@ -33,14 +38,57 @@ func realStore(t *testing.T, m *Module) string { srv := httptest.NewServer(handlers.NewApp(context.Background(), config)) t.Cleanup(srv.Close) addr := strings.TrimPrefix(srv.URL, "http://") - m.RegistryFile = filepath.Join(t.TempDir(), "registry") - if err := os.WriteFile(m.RegistryFile, []byte(addr+"\n"), 0o644); err != nil { - t.Fatal(err) + s := withStore(t, m) + inner := m.Call + m.Call = func(key string, body any) (json.RawMessage, error) { + raw, err := inner(key, body) + var a registry.PublishPartAnswer + if err != nil || !strings.HasSuffix(key, ".publish-part") || json.Unmarshal(raw, &a) != nil || !a.Complete { + return raw, err + } + s.mu.Lock() + defer s.mu.Unlock() + for k, b := range s.blobs { + name, digest, _ := strings.Cut(k, "@") + if err := uploadBlob(srv.URL, name, digest, b); err != nil { + return nil, err + } + delete(s.blobs, k) + } + return raw, nil } - m.Push = nil return addr } +// uploadBlob is the store's monolithic blob upload, a POST opening it and a PUT with the digest, as the holder does. +func uploadBlob(base, name, digest string, body []byte) error { + resp, err := http.Post(base+"/v2/"+name+"/blobs/uploads/", "", nil) + if err != nil { + return err + } + resp.Body.Close() + loc, err := url.Parse(resp.Header.Get("Location")) + if err != nil || resp.StatusCode != http.StatusAccepted { + return fmt.Errorf("the store opened no upload of %s: %s %v", digest, resp.Status, err) + } + b, _ := url.Parse(base) + u := b.ResolveReference(loc) + q := u.Query() + q.Set("digest", digest) + u.RawQuery = q.Encode() + req, _ := http.NewRequest(http.MethodPut, u.String(), bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/octet-stream") + put, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + put.Body.Close() + if put.StatusCode != http.StatusCreated { + return fmt.Errorf("the store did not take %s: %s", digest, put.Status) + } + return nil +} + // 1587-S10 (ADR 0498's check): a done bundle names a reference whose tail is its digest, and that reference resolves in // a real artifact store to bytes of that digest. The build is the real tsc's and esbuild's, shut off. func TestADoneBundleResolvesInARealArtifactStore(t *testing.T) { @@ -57,11 +105,11 @@ func TestADoneBundleResolvesInARealArtifactStore(t *testing.T) { if _, err := delivery.EncodeBuilderBuild(done); err != nil { t.Errorf("the done key the builder would write is refused: %v", err) } - path, ok := strings.CutPrefix(r.Reference, "artifact-store://") - if !ok || !strings.HasSuffix(r.Reference, "/blobs/"+r.Digest) { - t.Fatalf("the reference %q", r.Reference) + if _, _, ok := catalogue.InRegistrySeat(r.Reference); !ok || !strings.HasSuffix(r.Reference, "@"+r.Digest) { + t.Fatalf("the reference %q is not in the form a registry seat answers", r.Reference) } - resp, err := http.Get("http://" + addr + "/v2/" + path) + // Read as a node-engine reads it: the seat's reference routed to the store's address. + resp, err := http.Get(catalogue.Routed(r.Reference, addr)) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/toolchain.go b/modules/mesh-build-node/cmd/mesh-build-node/toolchain.go index 76d685218..08091c421 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/toolchain.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/toolchain.go @@ -25,7 +25,7 @@ const installedMark = ".installed" // Paths are the module's places on its machine. type Paths struct { // Home is the account's home (the state directory): where the mesh writes toolchain/package.json and - // toolchain/package-lock.json (module.json resources) and the registry binding and secret. + // toolchain/package-lock.json (module.json resources) and the registry binding. Home string // Cache is the cache directory, outside the home (ADR 0466 §1). Cache string diff --git a/modules/mesh-build-node/cmd/mesh-build-node/toolchain_test.go b/modules/mesh-build-node/cmd/mesh-build-node/toolchain_test.go index ccf06d4cf..8b208b60d 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/toolchain_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/toolchain_test.go @@ -49,7 +49,7 @@ func toolchainIn(t *testing.T, fake *fakeNpm) Toolchain { os.WriteFile(filepath.Join(home, "toolchain", "package-lock.json"), []byte(`{"lock":1}`), 0o600) return Toolchain{Paths: Paths{Home: home, Cache: cache}, Run: fake.run, Registry: func() (Registry, error) { - return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "pw"}, nil + return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil }} } @@ -106,13 +106,10 @@ func TestTheInstallNamesTheRegistryInAConfigFileNotOnTheCommandLine(t *testing.T if err := tc.Ensure(context.Background(), "v1"); err != nil { t.Fatal(err) } - if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/\n") || - !strings.Contains(npmrc, "//git.novox.be/api/packages/novox/npm/:_auth=") { + // Reads are anonymous (ADR 0346, CAT-BN3): the scope's registry and no credential. + if npmrc != "@novox:registry=https://git.novox.be/api/packages/novox/npm/\n" { t.Errorf("npmrc = %q", npmrc) } - if strings.Contains(strings.Join(fake.calls[0].args, " "), "pw") { - t.Errorf("the credential is on the command line: %v", fake.calls[0].args) - } } // 1488/AC4: an install that fails leaves no directory taken for a toolchain, says why, and the next call tries again. @@ -158,18 +155,17 @@ func TestPrepareInstallsOnlyBehindAPassingGuard(t *testing.T) { func TestRegistryFromTheBinding(t *testing.T) { dir := t.TempDir() - b, s := filepath.Join(dir, "b.json"), filepath.Join(dir, "s") - os.WriteFile(s, []byte("secret\n"), 0o600) + b := filepath.Join(dir, "b.json") os.WriteFile(b, []byte(`{"provision":"npm-package-registry","at":"10.42.0.1","as":"mesh-build-node-ace","serves":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/"}}`), 0o600) - r, err := ReadRegistry(b, s) - if err != nil || r.URL != "http://10.42.0.1:3000/api/packages/novox/npm/" || r.Username != "mesh-build-node-ace" || r.Password != "secret" { + r, err := ReadRegistry(b) + if err != nil || r.URL != "http://10.42.0.1:3000/api/packages/novox/npm/" || r.Npmrc() != "@novox:registry=http://10.42.0.1:3000/api/packages/novox/npm/\n" { t.Fatalf("= %+v, %v", r, err) } os.WriteFile(b, []byte(`{"at":"h","as":"a","serves":{"port":1,"npm-path":"/p/","public-root":"https://git.novox.be/p"}}`), 0o600) - if r, _ := ReadRegistry(b, s); r.URL != "https://git.novox.be/p/" { + if r, _ := ReadRegistry(b); r.URL != "https://git.novox.be/p/" { t.Errorf("the public root is not preferred: %q", r.URL) } - if _, err := ReadRegistry(filepath.Join(dir, "none"), s); err == nil { + if _, err := ReadRegistry(filepath.Join(dir, "none")); err == nil { t.Errorf("no binding was accepted") } } diff --git a/modules/mesh-build-node/files/registry b/modules/mesh-build-node/files/registry deleted file mode 100644 index 1bcadab26..000000000 --- a/modules/mesh-build-node/files/registry +++ /dev/null @@ -1 +0,0 @@ -${seat:mesh-artifact-store:reach} diff --git a/modules/mesh-build-node/go.work.sum b/modules/mesh-build-node/go.work.sum new file mode 100644 index 000000000..d2da19167 --- /dev/null +++ b/modules/mesh-build-node/go.work.sum @@ -0,0 +1,27 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM= diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index 25f8611b4..a15225b60 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -63,20 +63,17 @@ "forge-account", "npm-package-registry", "node-toolchain", - "artifact-store", "node-tools" ], - "contributes": { - "npm-package-registry": { - "access": "write" - } - }, "binds": { "npm-package-registry": "${dir:state}/package-registry.json" }, - "secrets": { - "npm-package-registry": "${dir:state}/package-registry.secret" - }, + "invokes": [ + "seat:oci-artifact-registry.publish", + "seat:oci-artifact-registry.publish-part", + "seat:npm-package-registry.publish", + "seat:npm-package-registry.publish-part" + ], "own-secrets": { "broker": "${dir:state}/broker" }, @@ -164,14 +161,6 @@ "mode": "0644", "template": "files/gitconfig" }, - { - "id": "registry", - "type": "file", - "path": "${dir:state}/registry", - "owner": "mesh-build-node", - "mode": "0644", - "template": "files/registry" - }, { "id": "toolchain-dir", "type": "archive",