CAT-BN3: mesh-build-node publishes through the registry seats and holds no registry credential

Bundles go through oci-artifact-registry and answer the seat's reference;
a trunk package is packed under the builder's version and published through
npm-package-registry, so the holder can check it with the builder. Reads are
anonymous, so the write pair, the sealed secret and the store's address go.
The one-part sender moves into buildstep for both build modules, so the move
to the registry clients is made in one place.

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
2026-11-04 15:30:06 +01:00
parent 39a8d8d7ee
commit af280050d6
21 changed files with 410 additions and 380 deletions
@@ -0,0 +1,46 @@
package buildstep
// The publish of a done build's file through a registry seat (novox/hq design 51 *Publishing*, ADR 0611; CAT-BG1,
// CAT-BN3): from the build module's own process, after the build step ended, through the seat's verbs publish and
// publish-part, which each build module's manifest `invokes`, in chunks, only what the holder lacks. The holder checks
// the build with the builder and says published or refused on the seat's events. One place for every build module, so
// the move to the registry clients is made here once.
import (
"bytes"
"context"
"encoding/json"
"io"
"strings"
reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry"
"git.novox.be/novox/mesh-sdk/go/publish"
)
// The registry seats a build module publishes on.
const (
ArtifactSeat = "oci-artifact-registry"
GoSeat = "go-package-registry"
NpmSeat = "npm-package-registry"
)
// SeatCall asks one verb of a seat, `seat:<seat>.<verb>`, with a body, and answers its JSON (the bundle's stdio.Ask).
type SeatCall = func(key string, body any) (json.RawMessage, error)
// TransferName is a transfer's name, the build module's own: the name, a qualifier and the digest's head, safe as a
// transfer id.
func TransferName(name, qualifier, digest string) string {
head := strings.TrimPrefix(digest, "sha256:")
if len(head) > 16 {
head = head[:16]
}
return strings.NewReplacer("/", "-", ".", "-", " ", "-", "*", "-", ">", "-", "@", "-").Replace(name + "-" + qualifier + "-" + head)
}
// PublishOne publishes body, of digest, as name (at version, when it has one) of kind on seat, in one part.
func PublishOne(ctx context.Context, call SeatCall, seat, name, kind, version, digest, transfer string, body []byte) error {
in := reg.PublishInput{Name: name, Kind: kind, Version: version, Digest: digest, Transfer: transfer}
_, err := publish.Publish(ctx, publish.Call(call), seat, in, []publish.Part{{Digest: digest, Size: int64(len(body)),
Open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(body)), nil }}})
return err
}
@@ -2,32 +2,25 @@ package main
// The publish of a done build's archive (novox/hq design 51 *Publishing*, ADR 0611; CAT-BG1): from this module's own
// process, after the build step ended, through the seat oci-artifact-registry's verbs publish and publish-part, which
// the manifest `invokes`, with mesh-sdk's one sender (publish.Publish): the archive in chunks, only what the holder
// the manifest `invokes`, with the build modules' one sender (buildstep.PublishOne): the archive in chunks, only what the holder
// lacks. No build step reaches the store, and this module no longer writes into it directly (push.go is gone): the
// holder checks the build with the builder and says published or refused on the seat's events.
import (
"bytes"
"context"
"encoding/json"
"io"
"strings"
registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/publish"
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
)
// ArtifactSeat is the registry seat a bundle's or an archive's archive is published on.
const ArtifactSeat = "oci-artifact-registry"
const ArtifactSeat = buildstep.ArtifactSeat
// transferName is the transfer's name, this module's own: the artifact and its digest's head, safe as a transfer id.
// transferName is the transfer's name, this module's own (buildstep.TransferName).
func transferName(module, artifact, digest string) string {
head := strings.TrimPrefix(digest, "sha256:")
if len(head) > 16 {
head = head[:16]
}
return strings.NewReplacer("/", "-", ".", "-", " ", "-", "*", "-", ">", "-").Replace(module + "-" + artifact + "-" + head)
return buildstep.TransferName(module, artifact, digest)
}
// publishArchive publishes body, of digest, as <module>/<artifact> of kind (bundle or archive) on the artifact seat.
@@ -35,17 +28,14 @@ func (m *Module) publishArchive(ctx context.Context, module, artifact, kind, dig
return m.publishOn(ctx, ArtifactSeat, module+"/"+artifact, kind, "", digest, transferName(module, artifact, digest), body)
}
// publishOn publishes body, of digest, as name (at version, when it has one) of kind on seat, in one part: through Call
// when a test set it, else the runtime's request over the bus (stdio.Ask).
// publishOn publishes body through buildstep.PublishOne: through Call when a test set it, else the runtime's request
// over the bus (stdio.Ask).
func (m *Module) publishOn(ctx context.Context, seat, name, kind, version, digest, transfer string, body []byte) error {
call := m.Call
if call == nil {
call = stdio.Ask
}
in := registry.PublishInput{Name: name, Kind: kind, Version: version, Digest: digest, Transfer: transfer}
_, err := publish.Publish(ctx, publish.Call(call), seat, in, []publish.Part{{Digest: digest, Size: int64(len(body)),
Open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(body)), nil }}})
return err
return buildstep.PublishOne(ctx, call, seat, name, kind, version, digest, transfer, body)
}
// callFunc is the runtime's request and reply over the bus, as stdio.Ask makes it.
+13 -11
View File
@@ -24,9 +24,9 @@ the module runs `npm ci --ignore-scripts` there once (a directory without its `.
**How the registry is reached** (the choice of issue 1488): the lock holds @novox/mesh-sdk at the forge's public
address, and npm 12 installs a locked tarball only from the registry it is configured with. The module therefore
requires `npm-package-registry` (read access, provided by gitea) and, like the lab module, takes its binding and sealed
secret as files in its home. The install gets a private npm config file with the `@novox` scope at the binding's
`public-root` (else scheme, address, port and npm path) and the bound account's credential, and removes it afterwards;
requires `npm-package-registry` (provided by gitea, form none: reads are anonymous) and, like the lab module, takes its
binding as a file in its home. The install gets a private npm config file with the `@novox` scope at the binding's
`public-root` (else scheme, address, port and npm path), with no credential, and removes it afterwards;
every other package resolves from the public registry. This is the way today's build agent names it.
The runtime tree for bundles with externals (the pruned sdk and nats today in `/app/runtime`) is not installed
@@ -83,14 +83,16 @@ toolchain install: a long step answers at once with its id and ends on `<node>.<
toolchain's packages (not typescript or esbuild) copied to `stage-<artifact>/node_modules`: the build agent's depth
(`/app/modules/module`, `/app/node_modules`), so esbuild's `../../node_modules/...` path comments, and the bundle's
digest, equal its. Copies, not links, because the mesh makes no symlink. A bundle that keeps an external gets them in its own `node_modules`.
Once the build ended, this process pushes the packed archive to the artifact store (novox/hq ADR 0498 §1 §2, mesh-sdk's
`buildstep.PushBlob`, the address in `${dir:state}/registry`) under `<module>/<artifact>` by its digest, a candidate's as a
trunk build's, and answers `reference: artifact-store://<module>/<artifact>/blobs/<digest>`; a push that fails fails the
step `not stored: <artifact>: <the store's words>`. The packed bytes never stay on the node. tsc and esbuild, like the
Once the build ended, this process publishes the packed archive through the seat `oci-artifact-registry` (design 51
*Publishing*; buildstep's one sender, `buildstep.PublishOne`, the manifest `invokes` publish and publish-part) under
`<module>/<artifact>` by its digest, a candidate's as a trunk build's, and answers the seat's reference
`oci-artifact-registry/<module>/<artifact>@<digest>`; a publish the holder refuses, or a seat nothing holds, fails the
step `not stored: <artifact>: <the holder's words>`. The packed bytes never stay on the node. tsc and esbuild, like the
type-check and the test script, run shut off in bubblewrap (ADR 0448): no network, none of the account, the toolchain
read-only, the npm cache beneath a throwaway layer unless the step is told `publish` (a trunk build).
A `package` artifact (ADR 0498 §5) is built by its own `build` script and packed with `npm pack --ignore-scripts`, both
shut off; its digest is the packed file's. Only with `publish` is it published, by this process with the registry's
write credential (never in a workspace or a step's mounts): a version the registry lacks is published, the same
packed file already there is done without a publish, another file under that version fails `version taken:
<name>@<version>`; the reference `<name>@<version>` is answered only then. A candidate answers its digest alone.
shut off; its digest is the packed file's. A trunk build the builder gave a version packs it under that version (its
package.json's `version` set for the pack, then put back) and publishes it through `npm-package-registry`; the holder
asks the builder, answers held for the same file and refuses another under that version. The reference
`<name>@<version>` is answered only then. A candidate, or a trunk build not numbered, answers its digest alone.
The module holds no credential for the registry: reads are anonymous (ADR 0346) and writes go through the seat.
@@ -19,6 +19,7 @@ import (
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
sdkstep "git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -88,7 +89,9 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
return nil, err
}
if a.Kind == "package" {
return m.buildPackage(ctx, workspace, module, root, dir, a, publish)
version, _ := args["version"].(string)
seat, _ := args["seat"].(string)
return m.buildPackage(ctx, workspace, module, root, dir, a, publish, version, seat)
}
if why, err := lockProblem(dir, module); err != nil || why != "" {
if err == nil {
@@ -139,9 +142,9 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
if err != nil {
return nil, fmt.Errorf("packing %s: %w", a.Name, err)
}
// Pushed from this process once the build ended, a candidate's as a trunk build's (ADR 0498 §1 §2): the step answers
// the reference, or fails not stored. The packed bytes are never written to the node (the lead's decision L9).
if err := m.push(ctx, a.module+"/"+a.Name, digest, archive); err != nil {
// Published from this process once the build ended, a candidate's as a trunk build's (ADR 0498 §1 §2; CAT-BN3): the
// step answers the seat's reference, or fails not stored. The packed bytes are never written to the node (L9).
if err := m.publishOn(ctx, buildstep.ArtifactSeat, a.module+"/"+a.Name, "bundle", "", digest, archive); err != nil {
return nil, fmt.Errorf("%s%s: %v", delivery.NotStored, a.Name, err)
}
files := []string{}
@@ -154,7 +157,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
})
sort.Strings(files)
return BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: digest, Bytes: len(archive),
Reference: buildstep.BlobReference(a.module, a.Name, digest), Files: files}, nil
Reference: catalogue.SeatReference(buildstep.ArtifactSeat, a.module, a.Name, digest), Files: files}, nil
}
// bundle makes every entrypoint and every launcher ONE file, with esbuild (builder.go:1971-2039): an entrypoint is
@@ -192,7 +192,7 @@ func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *test
files["modules/m/package-lock.json"] = `{}`
m, f := fetchMachine(t, files, map[string]string{"@novox/other/index.js": "o", "lodash/index.js": "l"})
m.Toolchain.Registry = func() (Registry, error) {
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "s3cret-pw!"}, nil
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil
}
var npmrc string
inner := f.effect
@@ -209,20 +209,17 @@ func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *test
if err != nil {
t.Fatal(err)
}
if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/") || !strings.Contains(npmrc, ":_auth=") {
// Reads are anonymous (ADR 0346, CAT-BN3): the config names the scope's registry and no credential.
if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/") || strings.Contains(npmrc, "_auth") {
t.Errorf("npm ci was given the config %q", npmrc)
}
// The password holds characters a workspace's random name never does, so a name like "mvpwby…" is no false match.
if strings.Contains(strings.Join(npmCalls(f)[0].args, " "), "s3cret-pw!") {
t.Errorf("the credential is on npm's command line")
}
r := out.(FetchResult)
if _, err := os.Stat(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "node_modules", "@novox", "other", "index.js")); err != nil {
t.Errorf("@novox/other was not installed: %v", err)
}
for _, e := range mustReadDir(t, filepath.Join(m.Work, r.Workspace)) {
if strings.HasPrefix(e, "npm-") {
t.Errorf("the scratch install (and its credential file) is left: %s", e)
t.Errorf("the scratch install is left: %s", e)
}
}
}
@@ -2,7 +2,7 @@ package main
// The run steps shut off (novox/hq ADR 0448; issue 1309, lead decision L2 of R4e): the type-check, the package's test
// script, tsc and esbuild run the change's code, so each runs through mesh-sdk's buildstep.Isolated, in bubblewrap: no
// network, none of the account (its home, its ssh key, its registry secret, its bus login, the runtime directory), the
// network, none of the account (its home, its ssh key, its bus login, the runtime directory), the
// workspace read-write, the toolchain read-only, the shared npm cache read-only beneath a throwaway writable layer, or
// read-write for a trunk build alone (publish). The fetch step (git, npm ci of the module's own packages) holds the key
// and the network and runs none of the change's code, so it runs as before.
@@ -36,7 +36,7 @@ func main() {
}
paths := Paths{Home: home, Cache: CacheDir}
registry := func() (Registry, error) {
return ReadRegistry(filepath.Join(home, "package-registry.json"), filepath.Join(home, "package-registry.secret"))
return ReadRegistry(filepath.Join(home, "package-registry.json"))
}
m := &Module{
Node: strings.TrimSpace(os.Getenv("MESH_NODE")),
@@ -55,7 +55,6 @@ func main() {
Ctx: ctx,
Say: logf,
Bubblewrap: ProbeBubblewrap,
RegistryFile: filepath.Join(home, "registry"),
}
m.Floor = floorOf(m.Node)
m.Settings = filepath.Join(home, "settings.json")
@@ -213,37 +213,45 @@ func TestTheModuleDeclaresBubblewrapAndTheCapabilityItNeeds(t *testing.T) {
}
}
// 1587-S5 (ADR 0498 §2): the module requires the artifact store and is told its address in a file of its home, which no
// run step is given (a run step has no network either, ADR 0448).
func TestTheModuleRequiresTheArtifactStoreAndIsToldItsAddress(t *testing.T) {
// CAT-BN3 (design 51 *Publishing*, ADR 0346): the module publishes only through the registry seats' verbs, which it
// invokes; it requires no store, places no store's address, and holds no credential for the package registry: reads
// are anonymous (the provision's form none), so it contributes nothing to it and is sealed no secret.
func TestTheModuleInvokesTheRegistrySeatsAndHoldsNoWriteCredential(t *testing.T) {
m := readManifest(t)
if !slices.Contains(m.Requires, "artifact-store") {
t.Errorf("requires %v lack artifact-store", m.Requires)
for _, seat := range []string{"oci-artifact-registry", "npm-package-registry"} {
for _, verb := range []string{"publish", "publish-part"} {
if !slices.Contains(m.Invokes, "seat:"+seat+"."+verb) {
t.Errorf("invokes %v lack the seat %s's %s", m.Invokes, seat, verb)
}
}
}
// Decoding leaves the template (files/registry) unread in the resource, so its text is read here.
r := resource(t, m, "registry")
content, err := os.ReadFile(filepath.Join(moduleDir, "files", "registry"))
if err != nil || r["type"] != "file" || r["path"] != "${dir:state}/registry" || r["template"] != "files/registry" ||
!strings.Contains(string(content), "${seat:mesh-artifact-store:reach}") {
t.Errorf("the registry resource is %v (template %q, %v)", r, content, err)
if slices.Contains(m.Requires, "artifact-store") {
t.Errorf("requires %v still name artifact-store", m.Requires)
}
for _, r := range m.Resources {
if r["id"] == "registry" {
t.Errorf("the store's address is still placed: %v", r)
}
}
if _, err := os.Stat(filepath.Join(moduleDir, "files", "registry")); err == nil {
t.Error("files/registry is still shipped")
}
}
// 1587-S17 (ADR 0498 §5): the module publishes a trunk package itself, so its contribution to the package registry asks
// for write access; the credential stays with the module's own process (a run step has neither the home nor a network).
func TestTheModuleContributesWriteAccessToThePackageRegistry(t *testing.T) {
raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json"))
if err != nil {
t.Fatal(err)
}
var mj struct {
Contributes map[string]map[string]any `json:"contributes"`
Contributes map[string]any `json:"contributes"`
Secrets map[string]string `json:"secrets"`
}
if err := json.Unmarshal(raw, &mj); err != nil {
t.Fatal(err)
}
if got := mj.Contributes["npm-package-registry"]["access"]; got != "write" {
t.Errorf("npm-package-registry access = %v, want write", got)
if _, ok := mj.Contributes["npm-package-registry"]; ok {
t.Errorf("contributes %v still ask the package registry for access", mj.Contributes)
}
if _, ok := mj.Secrets["npm-package-registry"]; ok {
t.Errorf("secrets %v still seal the package registry's credential", mj.Secrets)
}
}
@@ -57,10 +57,9 @@ type Module struct {
Say func(format string, a ...any)
// HTTP reads and publishes to the package registry; a client of its own when nil. A test serves its own.
HTTP *http.Client
// RegistryFile holds the artifact store's address (host:port), placed by the manifest; Push puts a blob there
// (buildstep.PushBlob when nil; a test's fake store).
RegistryFile string
Push func(ctx context.Context, registry, repository, digest string, body []byte) error
// Call asks a registry seat's verb over the bus, the publish of a done build (publish.go): stdio.Ask when nil; a
// test's fake holder.
Call buildstep.SeatCall
// Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is
// unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing.
Bubblewrap func(ctx context.Context) (string, error)
@@ -97,9 +97,8 @@ func (m *Module) installDev(ctx context.Context, root, dir, rel string) error {
return writeShims(filepath.Join(dir, "node_modules"), shimDir(root, rel))
}
// npmrcIn writes the npm configuration for the mesh's registry, with its credential, for one command of this process;
// the caller removes it. It lies in this process's temporary directory, never in a workspace: a run step is given the
// workspace and a /tmp of its own, so no step's mounts hold the credential (ADR 0498 §5, ADR 0500 §4).
// npmrcIn writes the npm configuration for the mesh's registry, read anonymously, for one command of this process; the
// caller removes it. It lies in this process's temporary directory, never in a workspace (ADR 0498 §5, ADR 0500 §4).
func (m *Module) npmrcIn(root string) (string, error) {
if m.Toolchain == nil || m.Toolchain.Registry == nil {
return "", fmt.Errorf("the module has no package registry")
@@ -188,7 +187,14 @@ func writeShims(nm, into string) error {
// buildAndPack builds the package in dir with its own build script, if it has one, and packs it with npm pack, both
// shut off (write: a trunk build's, which alone writes the shared cache). The packed file is read and removed: it never
// stays on the node.
func (m *Module) buildAndPack(ctx context.Context, root, dir, rel string, write bool) (packed, error) {
func (m *Module) buildAndPack(ctx context.Context, root, dir, rel string, write bool, version string) (packed, error) {
if version != "" {
restore, err := setPackageVersion(dir, version)
if err != nil {
return packed{}, err
}
defer restore()
}
p, err := readPackageJSON(dir)
if err != nil {
return packed{}, err
@@ -259,7 +265,7 @@ func (m *Module) packGroup(ctx context.Context, root string, run forgekey.Runner
if err := m.installDev(ctx, root, dir, "package"); err != nil {
return packed{}, err
}
return m.buildAndPack(ctx, root, dir, "package", false)
return m.buildAndPack(ctx, root, dir, "package", false, "")
}
// sdkLocked is what a module's lock names of the SDK: its version, its integrity and where it was resolved from; ok is
@@ -356,7 +362,6 @@ func (m *Module) registryVersion(ctx context.Context, name, version string) (tar
if err != nil {
return "", "", false, err
}
reg.authorize(req)
resp, err := m.httpClient().Do(req)
if err != nil {
return "", "", false, fmt.Errorf("asking the registry for %s: %w", name, err)
@@ -386,15 +391,10 @@ func (m *Module) registryPackage(ctx context.Context, name, version string) ([]b
if err != nil || !held {
return nil, held, err
}
reg, err := m.Toolchain.Registry()
if err != nil {
return nil, false, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, tarball, nil)
if err != nil {
return nil, false, err
}
reg.authorize(req)
resp, err := m.httpClient().Do(req)
if err != nil {
return nil, false, fmt.Errorf("downloading %s@%s: %w", name, version, err)
@@ -465,63 +465,60 @@ func fetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit,
return buildstep.FetchCommit(ctx, run, dir, remote, commit, ref, m.FetchOptions...)
}
// buildPackage builds and packs a package artifact (ADR 0498 §5): its digest is the packed file's. Only a trunk build
// (publish) publishes it, from this module's own process with the registry's write credential, which no run step is
// given: a version the registry lacks is published; the same packed file already there is done without a publish;
// another packed file under that version fails `version taken:`. The reference is <name>@<version>, answered only when
// the registry holds this build's file; a candidate answers its digest alone.
func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir string, a artifact, publish bool) (any, error) {
// buildPackage builds and packs a package artifact (ADR 0498 §5): its digest is the packed file's. A trunk build
// (publish) the builder gave a version packs it under that version and publishes it through npm-package-registry
// (design 51 *Publishing*, CAT-BN3), from this module's own process, which holds no write credential: the holder asks
// the builder, answers held for the same file and refuses another under that version. The reference is
// <name>@<version>, answered only when the holder took it; a candidate, or a trunk build not numbered, answers its
// digest alone.
func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir string, a artifact, publish bool, version, seat string) (any, error) {
rel := module
if rel == "" {
rel = "."
}
p, err := m.buildAndPack(ctx, root, dir, rel, publish)
if !publish {
version = ""
}
if seat == "" {
seat = buildstep.NpmSeat
}
if version != "" && seat != buildstep.NpmSeat {
return nil, fmt.Errorf("the package %s is published through %s, not %s: this module publishes npm packages", a.Name, buildstep.NpmSeat, seat)
}
p, err := m.buildAndPack(ctx, root, dir, rel, publish, version)
if err != nil {
return nil, err
}
res := BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: p.Digest, Bytes: len(p.tgz)}
if publish {
if res.Reference, err = m.publish(ctx, root, p); err != nil {
return nil, err
}
if version == "" {
return res, nil
}
if err := m.publishOn(ctx, seat, p.Name, "package", p.Version, p.Digest, p.tgz); err != nil {
return nil, fmt.Errorf("%s%s: %v", delivery.NotStored, a.Name, err)
}
res.Reference = p.Name + "@" + p.Version
return res, nil
}
// publish puts a packed package into the registry unless it holds that version, and answers <name>@<version>.
func (m *Module) publish(ctx context.Context, root string, p packed) (string, error) {
ref := p.Name + "@" + p.Version
_, integrity, held, err := m.registryVersion(ctx, p.Name, p.Version)
// setPackageVersion writes version as dir's package.json `version` (the builder's, design 51 *The versions*), so npm
// packs the package under it; restore puts the file back as it was.
func setPackageVersion(dir, version string) (restore func(), err error) {
path := filepath.Join(dir, "package.json")
raw, err := os.ReadFile(path)
if err != nil {
return "", err
return nil, fmt.Errorf("the package's package.json: %w", err)
}
if held {
if integrity == p.Integrity {
return ref, nil // this very file, published before: done without a publish
}
return "", fmt.Errorf("%s%s: the registry holds it as %s, and this build packs %s", delivery.VersionTaken, ref, integrity, p.Integrity)
var doc map[string]json.RawMessage
if err := json.Unmarshal(raw, &doc); err != nil {
return nil, fmt.Errorf("the package's package.json does not parse: %w", err)
}
file, err := os.CreateTemp(root, "publish-*.tgz")
doc["version"], _ = json.Marshal(version)
out, err := json.MarshalIndent(doc, "", " ")
if err != nil {
return "", err
return nil, err
}
defer os.Remove(file.Name()) // the packed file leaves the node once its publish answered
if _, err := file.Write(p.tgz); err != nil {
file.Close()
return "", err
if err := os.WriteFile(path, append(out, '\n'), 0o644); err != nil {
return nil, err
}
if err := file.Close(); err != nil {
return "", err
}
npmrc, err := m.npmrcIn(root)
if err != nil {
return "", err
}
defer os.Remove(npmrc)
env := append(m.baseEnv(), "npm_config_cache="+m.npmCache())
r := m.Run(ctx, env, "npm", "publish", file.Name(), "--ignore-scripts", "--userconfig", npmrc)
if why := ranFailed(r); why != "" {
return "", fmt.Errorf("publishing %s: %s", ref, why)
}
return ref, nil
return func() { _ = os.WriteFile(path, raw, 0o644) }, nil
}
@@ -4,82 +4,18 @@ package main
import (
"context"
"encoding/base64"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// npmRegistry is a registry of the test's own speaking npm's publish protocol: a PUT of the package's document with the
// packed file attached, then GET of the document and the file; it asks for the account's credential.
type npmRegistry struct {
mu sync.Mutex
versions map[string]map[string]any
files map[string][]byte
publishes int
}
func (r *npmRegistry) serve(t *testing.T) *httptest.Server {
var srv *httptest.Server
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, q *http.Request) {
r.mu.Lock()
defer r.mu.Unlock()
if u, p, ok := q.BasicAuth(); !ok || u != "builder" || p != "the write credential" {
w.WriteHeader(http.StatusUnauthorized)
return
}
doc := strings.TrimPrefix(q.URL.EscapedPath(), "/npm/")
switch {
case q.Method == http.MethodPut && (doc == "@novox%2fmesh-sdk" || doc == "@novox%2Fmesh-sdk"):
var body struct {
Versions map[string]map[string]any `json:"versions"`
Attachments map[string]struct {
Data string `json:"data"`
} `json:"_attachments"`
}
raw, _ := io.ReadAll(q.Body)
if err := json.Unmarshal(raw, &body); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
for v, meta := range body.Versions {
for name, a := range body.Attachments {
data, _ := base64.StdEncoding.DecodeString(a.Data)
r.files[name] = data
meta["dist"] = map[string]any{"tarball": srv.URL + "/files/" + name, "integrity": integrityOf(data)}
}
r.versions[v] = meta
}
r.publishes++
w.WriteHeader(http.StatusCreated)
w.Write([]byte(`{"ok":true}`))
case q.Method == http.MethodGet && (doc == "@novox%2fmesh-sdk" || doc == "@novox%2Fmesh-sdk"):
if len(r.versions) == 0 {
http.NotFound(w, q)
return
}
json.NewEncoder(w).Encode(map[string]any{"name": "@novox/mesh-sdk", "versions": r.versions})
case strings.HasPrefix(q.URL.Path, "/files/"):
w.Write(r.files[strings.TrimPrefix(q.URL.Path, "/files/")])
default:
http.NotFound(w, q)
}
}))
t.Cleanup(srv.Close)
return srv
}
// 1587-S20 (ADR 0498 §5): the real npm publishes a trunk build's package to a registry speaking npm's protocol, with
// the module's credential, once: a second trunk build of the same package finds its file there and publishes nothing;
// a candidate publishes nothing at all.
// 1587-S20 (ADR 0498 §5), CAT-BN3 (design 51 *Publishing*): the real npm packs a trunk build's package under the
// version the builder gave it, and the module publishes that packed file through npm-package-registry's verbs, with no
// credential and no npm publish; the workspace's package.json is put back. A candidate publishes nothing at all.
func TestARealTrunkPackageIsPublishedOnce(t *testing.T) {
needBubblewrap(t)
for _, tool := range []string{"node", "npm"} {
@@ -89,34 +25,39 @@ func TestARealTrunkPackageIsPublishedOnce(t *testing.T) {
}
m, _, work := newMachine(t)
m.Run = forgekey.ExecRunner
reg := &npmRegistry{versions: map[string]map[string]any{}, files: map[string][]byte{}}
srv := reg.serve(t)
m.Toolchain.Registry = func() (Registry, error) {
return Registry{URL: srv.URL + "/npm/", Username: "builder", Password: "the write credential"}, nil
}
m.HTTP = srv.Client()
store := withStore(t, m)
files := map[string]string{}
for n, c := range sdkFixture {
files[n] = c
}
workspaceWith(t, work, files)
dir := filepath.Join(work, "w1", "tree")
before, _ := os.ReadFile(filepath.Join(dir, "package.json"))
if err := m.installDev(context.Background(), filepath.Join(work, "w1"), dir, "."); err != nil {
t.Fatal(err)
}
args := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib"}
if out, err := m.build(context.Background(), args); err != nil || out.(BuildResult).Reference != "" || reg.publishes != 0 {
t.Fatalf("a candidate: %+v %v, %d publishes", out, err, reg.publishes)
if out, err := m.build(context.Background(), args); err != nil || out.(BuildResult).Reference != "" || len(store.published) != 0 {
t.Fatalf("a candidate: %+v %v, %d published", out, err, len(store.published))
}
args["publish"] = true
args["publish"], args["version"], args["seat"] = true, "9.9.10", "npm-package-registry"
out, err := m.build(context.Background(), args)
if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@9.9.9" || reg.publishes != 1 {
t.Fatalf("a trunk build: %+v %v, %d publishes", out, err, reg.publishes)
if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@9.9.10" || len(store.published) != 1 {
t.Fatalf("a trunk build: %+v %v, %d published", out, err, len(store.published))
}
if got := digestOf(reg.files["@novox/mesh-sdk-9.9.9.tgz"]); got != out.(BuildResult).Digest {
t.Errorf("the registry holds %s, the build packed %s (files %d)", got, out.(BuildResult).Digest, len(reg.files))
p := store.published[0]
tgz := store.blobs[p.Name+"@"+p.Digest]
if p.Name != "@novox/mesh-sdk" || p.Version != "9.9.10" || p.Kind != "package" || digestOf(tgz) != out.(BuildResult).Digest {
t.Errorf("published %+v (%d bytes), the build packed %s", p, len(tgz), out.(BuildResult).Digest)
}
if again, err := m.build(context.Background(), args); err != nil || again.(BuildResult).Reference != "@novox/mesh-sdk@9.9.9" || reg.publishes != 1 {
t.Errorf("the same package again: %+v %v, %d publishes", again, err, reg.publishes)
into := t.TempDir()
if err := unpackPackage(tgz, into); err != nil {
t.Fatal(err)
}
if raw, _ := os.ReadFile(filepath.Join(into, "package.json")); !strings.Contains(string(raw), `"version": "9.9.10"`) {
t.Errorf("the packed package.json is %s, want the builder's version", raw)
}
if after, _ := os.ReadFile(filepath.Join(dir, "package.json")); string(after) != string(before) {
t.Errorf("the workspace's package.json was not put back: %s", after)
}
}
@@ -2,9 +2,9 @@ package main
import (
"context"
"errors"
"os"
"path/filepath"
"slices"
"strings"
"testing"
@@ -69,26 +69,37 @@ func TestACandidatesPackageIsBuiltAndPackedAndNeverPublished(t *testing.T) {
}
}
// ADR 0498 §5: a trunk build (publish) publishes a version the registry lacks and answers <name>@<version>; the same
// packed file already there is done without a publish; another file under that version fails `version taken:`. The
// packed file is gone from the node afterwards.
// ADR 0498 §5, CAT-BN3 (design 51 *Publishing*): a trunk build the builder gave a version packs the package under that
// version and publishes it through npm-package-registry, never by npm publish nor with a credential, and answers
// <name>@<version>; its package.json is put back. A trunk build not numbered publishes nothing; a refusal fails it
// not stored. The packed file is gone from the node afterwards.
func TestATrunkPackageIsPublishedOnceAndAVersionTakenIsRefused(t *testing.T) {
tgz := tgzOf(t, sdkPackage, "0.1.15")
args := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib", "publish": true}
tgz := tgzOf(t, sdkPackage, "0.1.16")
args := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib", "publish": true, "version": "0.1.16", "seat": "npm-package-registry"}
m, f, work := packageMachine(t, tgz)
fakeRegistry(t, m, nil)
out, err := m.build(context.Background(), args)
if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@0.1.15" || len(publishes(f)) != 1 {
t.Fatalf("an absent version: %+v %v, %d publishes", out, err, len(publishes(f)))
}
if p := publishes(f)[0]; !slices.Contains(p.args, "--ignore-scripts") || !strings.HasSuffix(p.args[1], ".tgz") {
t.Errorf("published with %v", p.args)
}
// ADR 0500 §4: the registry's credential is never in a workspace, which a run step is given.
for i, a := range publishes(f)[0].args {
if a == "--userconfig" && strings.HasPrefix(publishes(f)[0].args[i+1], work) {
t.Errorf("the credential was written into the workspace: %s", publishes(f)[0].args[i+1])
var packedAs string
inner := f.effect
f.effect = func(c call) {
if c.name == "npm" && len(c.args) > 0 && c.args[0] == "pack" {
raw, _ := os.ReadFile(filepath.Join(work, "w1", "tree", "package.json"))
packedAs = string(raw)
}
inner(c)
}
store := withStore(t, m)
out, err := m.build(context.Background(), args)
if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@0.1.16" || len(publishes(f)) != 0 {
t.Fatalf("a numbered trunk package: %+v %v, %d npm publishes", out, err, len(publishes(f)))
}
if !strings.Contains(packedAs, `"version": "0.1.16"`) {
t.Errorf("packed with package.json %s, want the builder's version", packedAs)
}
if raw, _ := os.ReadFile(filepath.Join(work, "w1", "tree", "package.json")); !strings.Contains(string(raw), `"version":"0.1.15"`) {
t.Errorf("package.json was not put back: %s", raw)
}
if len(store.published) != 1 || store.published[0].Name != "@novox/mesh-sdk" || store.published[0].Version != "0.1.16" ||
store.published[0].Kind != "package" || store.seats[0] != "npm-package-registry" || store.published[0].Digest != digestOf(tgz) {
t.Errorf("published %+v through %v", store.published, store.seats)
}
var left []string
filepath.WalkDir(work, func(p string, d os.DirEntry, err error) error {
@@ -102,15 +113,15 @@ func TestATrunkPackageIsPublishedOnceAndAVersionTakenIsRefused(t *testing.T) {
}
m, f, _ = packageMachine(t, tgz)
fakeRegistry(t, m, map[string][]byte{"0.1.15": tgz})
if out, err := m.build(context.Background(), args); err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@0.1.15" || len(publishes(f)) != 0 {
t.Errorf("the same file published before: %+v %v, %d publishes", out, err, len(publishes(f)))
store = withStore(t, m)
unnumbered := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib", "publish": true}
if out, err := m.build(context.Background(), unnumbered); err != nil || out.(BuildResult).Reference != "" || len(store.published) != 0 || len(publishes(f)) != 0 {
t.Errorf("a trunk package not numbered: %+v %v, %d published", out, err, len(store.published))
}
m, f, _ = packageMachine(t, tgz)
fakeRegistry(t, m, map[string][]byte{"0.1.15": tgzOf(t, sdkPackage, "another")})
_, err = m.build(context.Background(), args)
if err == nil || !strings.HasPrefix(err.Error(), delivery.VersionTaken+"@novox/mesh-sdk@0.1.15") || len(publishes(f)) != 0 {
t.Errorf("another file under the version: %v, %d publishes", err, len(publishes(f)))
m, _, _ = packageMachine(t, tgz)
withStore(t, m).fail = errors.New("refused: the builder did not number it")
if _, err = m.build(context.Background(), args); err == nil || !strings.HasPrefix(err.Error(), delivery.NotStored+"lib: ") {
t.Errorf("a refused publish: %v", err)
}
}
@@ -1,42 +0,0 @@
package main
// The push of a done build's archive (novox/hq ADR 0498 §1 §2; issue 1587): from this module's own process, after the
// build step ended, with mesh-sdk's one client (buildstep.PushBlob), to the artifact store whose address the manifest
// binds into RegistryFile. No build step reaches the store: a run step has no network (ADR 0448).
import (
"context"
"fmt"
"net/http"
"os"
"strings"
"time"
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
)
// registry is the artifact store's address, host:port, as the manifest bound it into RegistryFile.
func (m *Module) registry() (string, error) {
raw, err := os.ReadFile(m.RegistryFile)
if err != nil {
return "", fmt.Errorf("the artifact store's address cannot be read: %w", err)
}
addr := strings.TrimSpace(string(raw))
if addr == "" || strings.Contains(addr, "${") || strings.HasPrefix(addr, ":") || strings.HasSuffix(addr, ":") {
return "", fmt.Errorf("the artifact store's address in %s is %q, not host:port", m.RegistryFile, addr)
}
return addr, nil
}
// push puts body into the artifact store as a blob of repository (<module>/<artifact>) by its digest: Push when a test
// set it, else buildstep.PushBlob.
func (m *Module) push(ctx context.Context, repository, digest string, body []byte) error {
addr, err := m.registry()
if err != nil {
return err
}
if m.Push != nil {
return m.Push(ctx, addr, repository, digest, body)
}
return buildstep.PushBlob(ctx, &http.Client{Timeout: 10 * time.Minute}, addr, repository, digest, body)
}
@@ -3,7 +3,9 @@ package main
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"os"
"path/filepath"
@@ -11,54 +13,87 @@ import (
"sync"
"testing"
registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
)
// pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back.
// pushed are the blobs the fake holder took, by digest, for the tests that read an archive back.
var pushed sync.Map
// fakeStore is a store of the test's own: its address in RegistryFile, each blob kept by repository and digest; fail,
// when set, is the store's refusal.
// fakeStore is the registry seats' holder as m's publish meets it (publish.go; CAT-BN3): publish opens a transfer and
// answers the part it lacks, publish-part takes the chunks in sequence, and the last one keeps the whole part when it
// hashes to its digest. Each file is kept by name and digest; fail, when set, is the holder's refusal at the last chunk.
type fakeStore struct {
mu sync.Mutex
blobs map[string][]byte // "<repository>@<digest>"
fail error
mu sync.Mutex
blobs map[string][]byte // "<name>@<digest>"
fail error
open map[string]registry.PublishInput
chunks map[string][]byte
published []registry.PublishInput
seats []string
}
func withStore(t *testing.T, m *Module) *fakeStore {
t.Helper()
s := &fakeStore{blobs: map[string][]byte{}}
m.RegistryFile = filepath.Join(t.TempDir(), "registry")
if err := os.WriteFile(m.RegistryFile, []byte("store.test:5000\n"), 0o644); err != nil {
t.Fatal(err)
}
m.Push = func(ctx context.Context, registry, repository, digest string, body []byte) error {
s := &fakeStore{blobs: map[string][]byte{}, open: map[string]registry.PublishInput{}, chunks: map[string][]byte{}}
m.Call = func(key string, body any) (json.RawMessage, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.fail != nil {
return s.fail
raw, _ := json.Marshal(body)
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
if seat != buildstep.ArtifactSeat && seat != buildstep.NpmSeat {
return nil, errors.New("nats: no responders available for request")
}
if sum := sha256.Sum256(body); "sha256:"+hex.EncodeToString(sum[:]) != digest || registry != "store.test:5000" {
t.Errorf("pushed %s to %s with bytes of another digest", digest, registry)
switch verb {
case "publish":
var in registry.PublishInput
_ = json.Unmarshal(raw, &in)
if (seat == buildstep.ArtifactSeat) != (in.Kind == "bundle") || in.Transfer == "" || strings.ContainsAny(in.Transfer, "/. *>") {
t.Errorf("publish on %s of kind %q, transfer %q", seat, in.Kind, in.Transfer)
}
s.open[in.Transfer] = in
s.seats = append(s.seats, seat)
return json.Marshal(registry.PublishAnswer{Transfer: in.Transfer, Lacks: []string{in.Digest}})
case "publish-part":
var c registry.PublishPartInput
_ = json.Unmarshal(raw, &c)
data, _ := base64.StdEncoding.DecodeString(c.Data)
s.chunks[c.Transfer] = append(s.chunks[c.Transfer], data...)
if !c.Last {
return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part})
}
in, whole := s.open[c.Transfer], s.chunks[c.Transfer]
delete(s.chunks, c.Transfer)
if s.fail != nil {
return nil, s.fail
}
if sum := sha256.Sum256(whole); "sha256:"+hex.EncodeToString(sum[:]) != in.Digest {
t.Errorf("published %s with bytes of another digest", in.Digest)
}
s.blobs[in.Name+"@"+in.Digest] = whole
s.published = append(s.published, in)
pushed.Store(in.Digest, whole)
return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part, Complete: true})
}
s.blobs[repository+"@"+digest] = append([]byte(nil), body...)
pushed.Store(digest, s.blobs[repository+"@"+digest])
return nil
return nil, errors.New("nats: no responders available for request")
}
return s
}
// archiveOf is the archive a build pushed, by its digest.
// archiveOf is the archive a build published, by its digest.
func archiveOf(t *testing.T, r BuildResult) []byte {
t.Helper()
raw, ok := pushed.Load(r.Digest)
if !ok {
t.Fatalf("nothing was pushed under %s", r.Digest)
t.Fatalf("nothing was published under %s", r.Digest)
}
return raw.([]byte)
}
// 1587-S6 (ADR 0498 §1): a done bundle answers the reference of the archive it pushed, under the manifest's module.
// 1587-S6 (ADR 0498 §1; CAT-BN3): a done bundle is published through oci-artifact-registry and answers the seat's
// reference of its archive, under the manifest's module.
func TestADoneBundleAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) {
m, _, _ := builtMachine(t, ``)
store := withStore(t, m)
@@ -67,23 +102,33 @@ func TestADoneBundleAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) {
t.Fatal(err)
}
r := out.(BuildResult)
if want := "artifact-store://m/tools/blobs/" + r.Digest; r.Reference != want {
if want := "oci-artifact-registry/m/tools@" + r.Digest; r.Reference != want {
t.Errorf("reference %q, want %q", r.Reference, want)
}
if store.blobs["m/tools@"+r.Digest] == nil {
t.Errorf("the store holds %d blobs, not m/tools@%s", len(store.blobs), r.Digest)
if store.blobs["m/tools@"+r.Digest] == nil || len(store.seats) != 1 || store.seats[0] != buildstep.ArtifactSeat {
t.Errorf("the holder holds %d files through %v, not m/tools@%s through %s", len(store.blobs), store.seats, r.Digest, buildstep.ArtifactSeat)
}
}
// 1587-S7 and S8 (ADR 0498 §2, the lead's decision L9): a push the store refuses fails the step `not stored:
// <artifact>: <the store's words>`, and whatever the push answered, no packed file is left on the node.
// 1587-S7 and S8 (ADR 0498 §2, the lead's decision L9): a publish the holder refuses fails the step `not stored:
// <artifact>: <the holder's words>`, a seat nothing holds too, and whatever it answered, no packed file is left.
func TestARefusedPushFailsNotStoredAndNoPackedFileStays(t *testing.T) {
for _, fail := range []error{nil, errors.New("store.test:5000 refused the blob: 507 Insufficient Storage")} {
for _, c := range []struct {
fail error
noSeat bool
words string
}{{nil, false, ""}, {errors.New("refused: the store did not take it: 507 Insufficient Storage"), false, "507"},
{nil, true, "no responders"}} {
m, _, work := builtMachine(t, ``)
withStore(t, m).fail = fail
withStore(t, m).fail = c.fail
if c.noSeat {
m.Call = func(string, any) (json.RawMessage, error) {
return nil, errors.New("nats: no responders available for request")
}
}
_, err := m.build(context.Background(), buildArgs())
if fail != nil && (err == nil || !strings.HasPrefix(err.Error(), delivery.NotStored+"tools: ") || !strings.Contains(err.Error(), "507")) {
t.Errorf("a refused push answered %v", err)
if c.words != "" && (err == nil || !strings.HasPrefix(err.Error(), delivery.NotStored+"tools: ") || !strings.Contains(err.Error(), c.words)) {
t.Errorf("a refused publish answered %v", err)
}
var archives []string
filepath.WalkDir(work, func(p string, d os.DirEntry, err error) error {
@@ -93,7 +138,7 @@ func TestARefusedPushFailsNotStoredAndNoPackedFileStays(t *testing.T) {
return nil
})
if len(archives) != 0 {
t.Errorf("push answered %v: packed files left on the node: %v", fail, archives)
t.Errorf("publish answered %v: packed files left on the node: %v", c.fail, archives)
}
}
}
@@ -3,16 +3,13 @@ package main
// How the toolchain's install reaches the mesh's package registry (novox/hq ADR 0405 §5, ADR 0336; issue 1488).
// The lock names @novox/mesh-sdk at the registry's public address, and npm installs a locked tarball only from the
// registry it is configured with, so the install is told that registry for the @novox scope and nothing else: every
// other package resolves from the public one. This is how today's build agent names it (mesh-controller
// internal/builder/dependencies.go: the scope's registry from the npm-package-registry binding, the account and
// secret the mesh sealed), read here from the module's own binding and secret files (module.json `binds` and
// `secrets`), the same files the lab module reads.
// other package resolves from the public one. The address is read from the module's npm-package-registry binding
// (module.json `binds`). Reads are anonymous (ADR 0346; the provision's form none, CAT-BN3): this module holds no
// credential for the registry, and publishes only through the seat's verbs (publish.go).
import (
"encoding/base64"
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
)
@@ -22,19 +19,18 @@ const Scope = "@novox"
// Registry is the mesh's package registry as the binding names it.
type Registry struct {
URL, Username, Password string
URL string
}
// ReadRegistry reads the npm-package-registry binding and its sealed secret. The registry is the binding's
// ReadRegistry reads the npm-package-registry binding. The registry is the binding's
// `public-root` when it has one (the address the lock holds), and otherwise scheme://at:port plus `npm-path`.
func ReadRegistry(bindingPath, secretPath string) (Registry, error) {
func ReadRegistry(bindingPath string) (Registry, error) {
raw, err := os.ReadFile(bindingPath)
if err != nil {
return Registry{}, fmt.Errorf("no package registry is bound to this module: %w", err)
}
var b struct {
At string `json:"at"`
As string `json:"as"`
Serves struct {
Scheme string `json:"scheme"`
Port int `json:"port"`
@@ -59,31 +55,10 @@ func ReadRegistry(bindingPath, secretPath string) (Registry, error) {
if !strings.HasSuffix(url, "/") {
url += "/"
}
secret, err := os.ReadFile(secretPath)
if err != nil {
return Registry{}, fmt.Errorf("the package registry's secret cannot be read: %w", err)
}
return Registry{URL: url, Username: b.As, Password: strings.TrimSpace(string(secret))}, nil
return Registry{URL: url}, nil
}
// Npmrc is the npm configuration text for the install: the @novox scope at the registry, with the account's
// credential for it when there is one.
// Npmrc is the npm configuration text for the install: the @novox scope at the registry, read anonymously.
func (r Registry) Npmrc() string {
var sb strings.Builder
fmt.Fprintf(&sb, "%s:registry=%s\n", Scope, r.URL)
if r.Username != "" && r.Password != "" {
hostPath := r.URL
if i := strings.Index(hostPath, "://"); i >= 0 {
hostPath = hostPath[i+1:]
}
fmt.Fprintf(&sb, "%s:_auth=%s\n", hostPath, base64.StdEncoding.EncodeToString([]byte(r.Username+":"+r.Password)))
}
return sb.String()
}
// authorize gives a request to the registry the account's credential, when there is one.
func (r Registry) authorize(req *http.Request) {
if r.Username != "" && r.Password != "" {
req.SetBasicAuth(r.Username, r.Password)
}
return fmt.Sprintf("%s:registry=%s\n", Scope, r.URL)
}
@@ -3,14 +3,16 @@
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"net/url"
"strings"
"testing"
@@ -18,12 +20,15 @@ import (
"github.com/distribution/distribution/v3/registry/handlers"
_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// realStore starts the artifact store's own program (the distribution registry, modules/distribution) inside this
// test, with its in-memory storage, on a loopback port, and points m at it with the real push client. It is imported
// by this integration test alone: the module's program does not carry it.
// test, with its in-memory storage, on a loopback port, and plays the artifact seat's holder storing into it: the fake
// holder (push_test.go) takes the publish, and each file it took is uploaded into the store as distribution's holder
// does. It is imported by this integration test alone: the module's program does not carry it.
func realStore(t *testing.T, m *Module) string {
t.Helper()
config := &configuration.Configuration{Storage: configuration.Storage{"inmemory": configuration.Parameters{}}}
@@ -33,14 +38,57 @@ func realStore(t *testing.T, m *Module) string {
srv := httptest.NewServer(handlers.NewApp(context.Background(), config))
t.Cleanup(srv.Close)
addr := strings.TrimPrefix(srv.URL, "http://")
m.RegistryFile = filepath.Join(t.TempDir(), "registry")
if err := os.WriteFile(m.RegistryFile, []byte(addr+"\n"), 0o644); err != nil {
t.Fatal(err)
s := withStore(t, m)
inner := m.Call
m.Call = func(key string, body any) (json.RawMessage, error) {
raw, err := inner(key, body)
var a registry.PublishPartAnswer
if err != nil || !strings.HasSuffix(key, ".publish-part") || json.Unmarshal(raw, &a) != nil || !a.Complete {
return raw, err
}
s.mu.Lock()
defer s.mu.Unlock()
for k, b := range s.blobs {
name, digest, _ := strings.Cut(k, "@")
if err := uploadBlob(srv.URL, name, digest, b); err != nil {
return nil, err
}
delete(s.blobs, k)
}
return raw, nil
}
m.Push = nil
return addr
}
// uploadBlob is the store's monolithic blob upload, a POST opening it and a PUT with the digest, as the holder does.
func uploadBlob(base, name, digest string, body []byte) error {
resp, err := http.Post(base+"/v2/"+name+"/blobs/uploads/", "", nil)
if err != nil {
return err
}
resp.Body.Close()
loc, err := url.Parse(resp.Header.Get("Location"))
if err != nil || resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("the store opened no upload of %s: %s %v", digest, resp.Status, err)
}
b, _ := url.Parse(base)
u := b.ResolveReference(loc)
q := u.Query()
q.Set("digest", digest)
u.RawQuery = q.Encode()
req, _ := http.NewRequest(http.MethodPut, u.String(), bytes.NewReader(body))
req.Header.Set("Content-Type", "application/octet-stream")
put, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
put.Body.Close()
if put.StatusCode != http.StatusCreated {
return fmt.Errorf("the store did not take %s: %s", digest, put.Status)
}
return nil
}
// 1587-S10 (ADR 0498's check): a done bundle names a reference whose tail is its digest, and that reference resolves in
// a real artifact store to bytes of that digest. The build is the real tsc's and esbuild's, shut off.
func TestADoneBundleResolvesInARealArtifactStore(t *testing.T) {
@@ -57,11 +105,11 @@ func TestADoneBundleResolvesInARealArtifactStore(t *testing.T) {
if _, err := delivery.EncodeBuilderBuild(done); err != nil {
t.Errorf("the done key the builder would write is refused: %v", err)
}
path, ok := strings.CutPrefix(r.Reference, "artifact-store://")
if !ok || !strings.HasSuffix(r.Reference, "/blobs/"+r.Digest) {
t.Fatalf("the reference %q", r.Reference)
if _, _, ok := catalogue.InRegistrySeat(r.Reference); !ok || !strings.HasSuffix(r.Reference, "@"+r.Digest) {
t.Fatalf("the reference %q is not in the form a registry seat answers", r.Reference)
}
resp, err := http.Get("http://" + addr + "/v2/" + path)
// Read as a node-engine reads it: the seat's reference routed to the store's address.
resp, err := http.Get(catalogue.Routed(r.Reference, addr))
if err != nil {
t.Fatal(err)
}
@@ -25,7 +25,7 @@ const installedMark = ".installed"
// Paths are the module's places on its machine.
type Paths struct {
// Home is the account's home (the state directory): where the mesh writes toolchain/package.json and
// toolchain/package-lock.json (module.json resources) and the registry binding and secret.
// toolchain/package-lock.json (module.json resources) and the registry binding.
Home string
// Cache is the cache directory, outside the home (ADR 0466 §1).
Cache string
@@ -49,7 +49,7 @@ func toolchainIn(t *testing.T, fake *fakeNpm) Toolchain {
os.WriteFile(filepath.Join(home, "toolchain", "package-lock.json"), []byte(`{"lock":1}`), 0o600)
return Toolchain{Paths: Paths{Home: home, Cache: cache}, Run: fake.run,
Registry: func() (Registry, error) {
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "pw"}, nil
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil
}}
}
@@ -106,13 +106,10 @@ func TestTheInstallNamesTheRegistryInAConfigFileNotOnTheCommandLine(t *testing.T
if err := tc.Ensure(context.Background(), "v1"); err != nil {
t.Fatal(err)
}
if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/\n") ||
!strings.Contains(npmrc, "//git.novox.be/api/packages/novox/npm/:_auth=") {
// Reads are anonymous (ADR 0346, CAT-BN3): the scope's registry and no credential.
if npmrc != "@novox:registry=https://git.novox.be/api/packages/novox/npm/\n" {
t.Errorf("npmrc = %q", npmrc)
}
if strings.Contains(strings.Join(fake.calls[0].args, " "), "pw") {
t.Errorf("the credential is on the command line: %v", fake.calls[0].args)
}
}
// 1488/AC4: an install that fails leaves no directory taken for a toolchain, says why, and the next call tries again.
@@ -158,18 +155,17 @@ func TestPrepareInstallsOnlyBehindAPassingGuard(t *testing.T) {
func TestRegistryFromTheBinding(t *testing.T) {
dir := t.TempDir()
b, s := filepath.Join(dir, "b.json"), filepath.Join(dir, "s")
os.WriteFile(s, []byte("secret\n"), 0o600)
b := filepath.Join(dir, "b.json")
os.WriteFile(b, []byte(`{"provision":"npm-package-registry","at":"10.42.0.1","as":"mesh-build-node-ace","serves":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/"}}`), 0o600)
r, err := ReadRegistry(b, s)
if err != nil || r.URL != "http://10.42.0.1:3000/api/packages/novox/npm/" || r.Username != "mesh-build-node-ace" || r.Password != "secret" {
r, err := ReadRegistry(b)
if err != nil || r.URL != "http://10.42.0.1:3000/api/packages/novox/npm/" || r.Npmrc() != "@novox:registry=http://10.42.0.1:3000/api/packages/novox/npm/\n" {
t.Fatalf("= %+v, %v", r, err)
}
os.WriteFile(b, []byte(`{"at":"h","as":"a","serves":{"port":1,"npm-path":"/p/","public-root":"https://git.novox.be/p"}}`), 0o600)
if r, _ := ReadRegistry(b, s); r.URL != "https://git.novox.be/p/" {
if r, _ := ReadRegistry(b); r.URL != "https://git.novox.be/p/" {
t.Errorf("the public root is not preferred: %q", r.URL)
}
if _, err := ReadRegistry(filepath.Join(dir, "none"), s); err == nil {
if _, err := ReadRegistry(filepath.Join(dir, "none")); err == nil {
t.Errorf("no binding was accepted")
}
}
-1
View File
@@ -1 +0,0 @@
${seat:mesh-artifact-store:reach}
+27
View File
@@ -0,0 +1,27 @@
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ=
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
+6 -17
View File
@@ -63,20 +63,17 @@
"forge-account",
"npm-package-registry",
"node-toolchain",
"artifact-store",
"node-tools"
],
"contributes": {
"npm-package-registry": {
"access": "write"
}
},
"binds": {
"npm-package-registry": "${dir:state}/package-registry.json"
},
"secrets": {
"npm-package-registry": "${dir:state}/package-registry.secret"
},
"invokes": [
"seat:oci-artifact-registry.publish",
"seat:oci-artifact-registry.publish-part",
"seat:npm-package-registry.publish",
"seat:npm-package-registry.publish-part"
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
@@ -164,14 +161,6 @@
"mode": "0644",
"template": "files/gitconfig"
},
{
"id": "registry",
"type": "file",
"path": "${dir:state}/registry",
"owner": "mesh-build-node",
"mode": "0644",
"template": "files/registry"
},
{
"id": "toolchain-dir",
"type": "archive",