Join: merge A3 (wip/registry-a3)

Conflicts, both sides kept: artifact-store-tools' invokes (the assigner's
and indexer's read tools, followed through views, beside the registry
seats' hold, release and remove), its main (the views reader and the
seat asker), its manifest row; gitea's invokes (the indexer's read tool,
the hand act and the builder's versions) and its consumes of the
indexer's views. gitea's account test takes the packages account's
names; repoCheckFacts, removed on main, is not imported. No catalogue
module reads the indexer's state.

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
2026-11-04 16:38:20 +01:00
186 changed files with 12804 additions and 3650 deletions
@@ -2,6 +2,7 @@ package main
import (
"context"
"path/filepath"
"time"
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
@@ -29,6 +30,7 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps {
}
return buildstep.AskSteps{
Fetch: byName["build_node_fetch"], Build: byName["build_node_build"], Test: byName["build_node_test"],
Compare: buildstep.CompareStep(buildstep.GeneratedRootTS, func(ws string) string { return filepath.Join(m.Work, ws, "tree") }),
Toolchain: func(ctx context.Context) (delivery.ToolchainUsed, error) {
installed, err := m.Probe(ctx)
if err != nil {
@@ -19,6 +19,7 @@ import (
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
sdkstep "git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -88,7 +89,9 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
return nil, err
}
if a.Kind == "package" {
return m.buildPackage(ctx, workspace, module, root, dir, a, publish)
version, _ := args["version"].(string)
seat, _ := args["seat"].(string)
return m.buildPackage(ctx, workspace, module, root, dir, a, publish, version, seat)
}
if why, err := lockProblem(dir, module); err != nil || why != "" {
if err == nil {
@@ -139,9 +142,9 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
if err != nil {
return nil, fmt.Errorf("packing %s: %w", a.Name, err)
}
// Pushed from this process once the build ended, a candidate's as a trunk build's (ADR 0498 §1 §2): the step answers
// the reference, or fails not stored. The packed bytes are never written to the node (the lead's decision L9).
if err := m.push(ctx, a.module+"/"+a.Name, digest, archive); err != nil {
// Published from this process once the build ended, a candidate's as a trunk build's (ADR 0498 §1 §2; CAT-BN3): the
// step answers the seat's reference, or fails not stored. The packed bytes are never written to the node (L9).
if err := m.publishOn(ctx, buildstep.ArtifactSeat, a.module+"/"+a.Name, "bundle", "", digest, archive); err != nil {
return nil, fmt.Errorf("%s%s: %v", delivery.NotStored, a.Name, err)
}
files := []string{}
@@ -154,7 +157,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
})
sort.Strings(files)
return BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: digest, Bytes: len(archive),
Reference: buildstep.BlobReference(a.module, a.Name, digest), Files: files}, nil
Reference: catalogue.SeatReference(buildstep.ArtifactSeat, a.module, a.Name, digest), Files: files}, nil
}
// bundle makes every entrypoint and every launcher ONE file, with esbuild (builder.go:1971-2039): an entrypoint is
@@ -59,6 +59,9 @@ type TestResult struct {
// By is the build module and the SDK it was built with (sdkstep.By): the checker names it when a failure hides
// its cause.
By string `json:"by,omitempty"`
// OutOfStep are the generated TypeScript files of a contracts module out of step with the seat generator (CAT-BN2,
// buildstep.OutOfStep): the unit level fails on any, Why naming each. Absent when none is.
OutOfStep []string `json:"out-of-step,omitempty"`
}
// Failure is the SDK's failure contract (sdkstep.Failure, issue 1901): a failing test of the test script, read from
@@ -89,6 +92,29 @@ func (m *Module) testTool() StepTool {
}
func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
res, err := m.testLevel(ctx, args)
if err != nil || res.Level != "unit" {
return res, err
}
// CAT-BN2: a contracts module's generated TypeScript is regenerated and compared; a file out of step fails the level.
_, dir, err := m.moduleDirOf(res.Workspace, res.Module)
if err != nil {
return nil, err
}
if res.OutOfStep, err = buildstep.OutOfStep(dir, buildstep.GeneratedRootTS); err != nil {
return nil, err
}
if len(res.OutOfStep) > 0 {
why := "generated files out of step with the seat generator (regenerate them): " + strings.Join(res.OutOfStep, ", ")
if res.Why != "" && !res.NoTests {
why = res.Why + "; " + why
}
res.Passed, res.NoTests, res.Why = false, false, why
}
return res, nil
}
func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResult, error) {
workspace, _ := args["workspace"].(string)
module, _ := args["module"].(string)
level, _ := args["level"].(string)
@@ -79,6 +79,29 @@ func TestAPackageWithNeitherScriptNorTypeScriptHasNoTests(t *testing.T) {
if len(f.calls) != 0 {
t.Errorf("%d commands ran for a package with no tests", len(f.calls))
}
// CAT-BN2: a contracts module (a package artifact naming generated-from) with no tests of its own still fails its
// unit level on a generated TypeScript file out of step with the seat generator, each named; its Go files are
// mesh-build-go's to compare.
m, _, work = newMachine(t)
workspaceWith(t, work, map[string]string{
"module.json": `{"module":"c","build":{"artifacts":[{"name":"ts","kind":"package","language":"typescript","paths":["ts"],"generated-from":"."}]}}`,
"seats/index.json": `{"format":1,"seats":["one-store"]}`,
"seats/one-store/seat.json": `{"format":1,"seat":"one-store","scope":"mesh","grain":"seat-list","decision":"ADR 1","verbs":[{"name":"query","input":{"type":"object"}}]}`,
"go/go.mod": "module example.org/c/go\n", "go/stray.go": "package stray\n",
"ts/package.json": `{"name":"@example/c"}`, "ts/stray.ts.txt": "not written by the generator\n",
})
out, err := m.test(context.Background(), testArgs("unit"))
r := out.(*TestResult)
stray := false
for _, p := range r.OutOfStep {
stray = stray || p == "ts/stray.ts.txt"
if !strings.HasPrefix(p, "ts/") {
t.Errorf("%s is not under the TypeScript root", p)
}
}
if err != nil || r.Passed || r.NoTests || !stray || !strings.Contains(r.Why, "ts/stray.ts.txt") {
t.Errorf("a contracts module out of step = %+v, %v", r, err)
}
}
// 1491/AC4: with test:integration present the integration level runs it, and reports the setup (the type-check) and
@@ -192,7 +192,7 @@ func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *test
files["modules/m/package-lock.json"] = `{}`
m, f := fetchMachine(t, files, map[string]string{"@novox/other/index.js": "o", "lodash/index.js": "l"})
m.Toolchain.Registry = func() (Registry, error) {
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "s3cret-pw!"}, nil
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil
}
var npmrc string
inner := f.effect
@@ -209,20 +209,17 @@ func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *test
if err != nil {
t.Fatal(err)
}
if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/") || !strings.Contains(npmrc, ":_auth=") {
// Reads are anonymous (ADR 0346, CAT-BN3): the config names the scope's registry and no credential.
if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/") || strings.Contains(npmrc, "_auth") {
t.Errorf("npm ci was given the config %q", npmrc)
}
// The password holds characters a workspace's random name never does, so a name like "mvpwby…" is no false match.
if strings.Contains(strings.Join(npmCalls(f)[0].args, " "), "s3cret-pw!") {
t.Errorf("the credential is on npm's command line")
}
r := out.(FetchResult)
if _, err := os.Stat(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "node_modules", "@novox", "other", "index.js")); err != nil {
t.Errorf("@novox/other was not installed: %v", err)
}
for _, e := range mustReadDir(t, filepath.Join(m.Work, r.Workspace)) {
if strings.HasPrefix(e, "npm-") {
t.Errorf("the scratch install (and its credential file) is left: %s", e)
t.Errorf("the scratch install is left: %s", e)
}
}
}
@@ -2,7 +2,7 @@ package main
// The run steps shut off (novox/hq ADR 0448; issue 1309, lead decision L2 of R4e): the type-check, the package's test
// script, tsc and esbuild run the change's code, so each runs through mesh-sdk's buildstep.Isolated, in bubblewrap: no
// network, none of the account (its home, its ssh key, its registry secret, its bus login, the runtime directory), the
// network, none of the account (its home, its ssh key, its bus login, the runtime directory), the
// workspace read-write, the toolchain read-only, the shared npm cache read-only beneath a throwaway writable layer, or
// read-write for a trunk build alone (publish). The fetch step (git, npm ci of the module's own packages) holds the key
// and the network and runs none of the change's code, so it runs as before.
@@ -36,7 +36,7 @@ func main() {
}
paths := Paths{Home: home, Cache: CacheDir}
registry := func() (Registry, error) {
return ReadRegistry(filepath.Join(home, "package-registry.json"), filepath.Join(home, "package-registry.secret"))
return ReadRegistry(filepath.Join(home, "package-registry.json"))
}
m := &Module{
Node: strings.TrimSpace(os.Getenv("MESH_NODE")),
@@ -55,7 +55,6 @@ func main() {
Ctx: ctx,
Say: logf,
Bubblewrap: ProbeBubblewrap,
RegistryFile: filepath.Join(home, "registry"),
}
m.Floor = floorOf(m.Node)
m.Settings = filepath.Join(home, "settings.json")
@@ -213,37 +213,45 @@ func TestTheModuleDeclaresBubblewrapAndTheCapabilityItNeeds(t *testing.T) {
}
}
// 1587-S5 (ADR 0498 §2): the module requires the artifact store and is told its address in a file of its home, which no
// run step is given (a run step has no network either, ADR 0448).
func TestTheModuleRequiresTheArtifactStoreAndIsToldItsAddress(t *testing.T) {
// CAT-BN3 (design 51 *Publishing*, ADR 0346): the module publishes only through the registry seats' verbs, which it
// invokes; it requires no store, places no store's address, and holds no credential for the package registry: reads
// are anonymous (the provision's form none), so it contributes nothing to it and is sealed no secret.
func TestTheModuleInvokesTheRegistrySeatsAndHoldsNoWriteCredential(t *testing.T) {
m := readManifest(t)
if !slices.Contains(m.Requires, "artifact-store") {
t.Errorf("requires %v lack artifact-store", m.Requires)
for _, seat := range []string{"oci-artifact-registry", "npm-package-registry"} {
for _, verb := range []string{"publish", "publish-part"} {
if !slices.Contains(m.Invokes, "seat:"+seat+"."+verb) {
t.Errorf("invokes %v lack the seat %s's %s", m.Invokes, seat, verb)
}
}
}
// Decoding leaves the template (files/registry) unread in the resource, so its text is read here.
r := resource(t, m, "registry")
content, err := os.ReadFile(filepath.Join(moduleDir, "files", "registry"))
if err != nil || r["type"] != "file" || r["path"] != "${dir:state}/registry" || r["template"] != "files/registry" ||
!strings.Contains(string(content), "${seat:mesh-artifact-store:reach}") {
t.Errorf("the registry resource is %v (template %q, %v)", r, content, err)
if slices.Contains(m.Requires, "artifact-store") {
t.Errorf("requires %v still name artifact-store", m.Requires)
}
for _, r := range m.Resources {
if r["id"] == "registry" {
t.Errorf("the store's address is still placed: %v", r)
}
}
if _, err := os.Stat(filepath.Join(moduleDir, "files", "registry")); err == nil {
t.Error("files/registry is still shipped")
}
}
// 1587-S17 (ADR 0498 §5): the module publishes a trunk package itself, so its contribution to the package registry asks
// for write access; the credential stays with the module's own process (a run step has neither the home nor a network).
func TestTheModuleContributesWriteAccessToThePackageRegistry(t *testing.T) {
raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json"))
if err != nil {
t.Fatal(err)
}
var mj struct {
Contributes map[string]map[string]any `json:"contributes"`
Contributes map[string]any `json:"contributes"`
Secrets map[string]string `json:"secrets"`
}
if err := json.Unmarshal(raw, &mj); err != nil {
t.Fatal(err)
}
if got := mj.Contributes["npm-package-registry"]["access"]; got != "write" {
t.Errorf("npm-package-registry access = %v, want write", got)
if _, ok := mj.Contributes["npm-package-registry"]; ok {
t.Errorf("contributes %v still ask the package registry for access", mj.Contributes)
}
if _, ok := mj.Secrets["npm-package-registry"]; ok {
t.Errorf("secrets %v still seal the package registry's credential", mj.Secrets)
}
}
@@ -57,10 +57,9 @@ type Module struct {
Say func(format string, a ...any)
// HTTP reads and publishes to the package registry; a client of its own when nil. A test serves its own.
HTTP *http.Client
// RegistryFile holds the artifact store's address (host:port), placed by the manifest; Push puts a blob there
// (buildstep.PushBlob when nil; a test's fake store).
RegistryFile string
Push func(ctx context.Context, registry, repository, digest string, body []byte) error
// Call asks a registry seat's verb over the bus, the publish of a done build (publish.go): stdio.Ask when nil; a
// test's fake holder.
Call buildstep.SeatCall
// Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is
// unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing.
Bubblewrap func(ctx context.Context) (string, error)
@@ -98,9 +98,8 @@ func (m *Module) installDev(ctx context.Context, root, dir, rel string) error {
return writeShims(filepath.Join(dir, "node_modules"), shimDir(root, rel))
}
// npmrcIn writes the npm configuration for the mesh's registry, with its credential, for one command of this process;
// the caller removes it. It lies in this process's temporary directory, never in a workspace: a run step is given the
// workspace and a /tmp of its own, so no step's mounts hold the credential (ADR 0498 §5, ADR 0500 §4).
// npmrcIn writes the npm configuration for the mesh's registry, read anonymously, for one command of this process; the
// caller removes it. It lies in this process's temporary directory, never in a workspace (ADR 0498 §5, ADR 0500 §4).
func (m *Module) npmrcIn(root string) (string, error) {
if m.Toolchain == nil || m.Toolchain.Registry == nil {
return "", fmt.Errorf("the module has no package registry")
@@ -189,7 +188,14 @@ func writeShims(nm, into string) error {
// buildAndPack builds the package in dir with its own build script, if it has one, and packs it with npm pack, both
// shut off (write: a trunk build's, which alone writes the shared cache). The packed file is read and removed: it never
// stays on the node.
func (m *Module) buildAndPack(ctx context.Context, root, dir, rel string, write bool) (packed, error) {
func (m *Module) buildAndPack(ctx context.Context, root, dir, rel string, write bool, version string) (packed, error) {
if version != "" {
restore, err := setPackageVersion(dir, version)
if err != nil {
return packed{}, err
}
defer restore()
}
p, err := readPackageJSON(dir)
if err != nil {
return packed{}, err
@@ -281,7 +287,7 @@ func (m *Module) packGroup(ctx context.Context, root string, run forgekey.Runner
if err := m.installDev(ctx, root, dir, "package"); err != nil {
return packed{}, err
}
return m.buildAndPack(ctx, root, dir, "package", false)
return m.buildAndPack(ctx, root, dir, "package", false, "")
}
// sdkLocked is what a module's lock names of the SDK: its version, its integrity and where it was resolved from; ok is
@@ -412,7 +418,6 @@ func (m *Module) registryVersion(ctx context.Context, name, version string) (tar
if err != nil {
return "", "", false, err
}
reg.authorize(req)
resp, err := m.httpClient().Do(req)
if err != nil {
return "", "", false, fmt.Errorf("asking the registry for %s: %w", name, err)
@@ -442,15 +447,10 @@ func (m *Module) registryPackage(ctx context.Context, name, version string) ([]b
if err != nil || !held {
return nil, held, err
}
reg, err := m.Toolchain.Registry()
if err != nil {
return nil, false, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, tarball, nil)
if err != nil {
return nil, false, err
}
reg.authorize(req)
resp, err := m.httpClient().Do(req)
if err != nil {
return nil, false, fmt.Errorf("downloading %s@%s: %w", name, version, err)
@@ -521,63 +521,60 @@ func fetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit,
return buildstep.FetchCommit(ctx, run, dir, remote, commit, ref, m.FetchOptions...)
}
// buildPackage builds and packs a package artifact (ADR 0498 §5): its digest is the packed file's. Only a trunk build
// (publish) publishes it, from this module's own process with the registry's write credential, which no run step is
// given: a version the registry lacks is published; the same packed file already there is done without a publish;
// another packed file under that version fails `version taken:`. The reference is <name>@<version>, answered only when
// the registry holds this build's file; a candidate answers its digest alone.
func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir string, a artifact, publish bool) (any, error) {
// buildPackage builds and packs a package artifact (ADR 0498 §5): its digest is the packed file's. A trunk build
// (publish) the builder gave a version packs it under that version and publishes it through npm-package-registry
// (design 51 *Publishing*, CAT-BN3), from this module's own process, which holds no write credential: the holder asks
// the builder, answers held for the same file and refuses another under that version. The reference is
// <name>@<version>, answered only when the holder took it; a candidate, or a trunk build not numbered, answers its
// digest alone.
func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir string, a artifact, publish bool, version, seat string) (any, error) {
rel := module
if rel == "" {
rel = "."
}
p, err := m.buildAndPack(ctx, root, dir, rel, publish)
if !publish {
version = ""
}
if seat == "" {
seat = buildstep.NpmSeat
}
if version != "" && seat != buildstep.NpmSeat {
return nil, fmt.Errorf("the package %s is published through %s, not %s: this module publishes npm packages", a.Name, buildstep.NpmSeat, seat)
}
p, err := m.buildAndPack(ctx, root, dir, rel, publish, version)
if err != nil {
return nil, err
}
res := BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: p.Digest, Bytes: len(p.tgz)}
if publish {
if res.Reference, err = m.publish(ctx, root, p); err != nil {
return nil, err
}
if version == "" {
return res, nil
}
if err := m.publishOn(ctx, seat, p.Name, "package", p.Version, p.Digest, p.tgz); err != nil {
return nil, fmt.Errorf("%s%s: %v", delivery.NotStored, a.Name, err)
}
res.Reference = p.Name + "@" + p.Version
return res, nil
}
// publish puts a packed package into the registry unless it holds that version, and answers <name>@<version>.
func (m *Module) publish(ctx context.Context, root string, p packed) (string, error) {
ref := p.Name + "@" + p.Version
_, integrity, held, err := m.registryVersion(ctx, p.Name, p.Version)
// setPackageVersion writes version as dir's package.json `version` (the builder's, design 51 *The versions*), so npm
// packs the package under it; restore puts the file back as it was.
func setPackageVersion(dir, version string) (restore func(), err error) {
path := filepath.Join(dir, "package.json")
raw, err := os.ReadFile(path)
if err != nil {
return "", err
return nil, fmt.Errorf("the package's package.json: %w", err)
}
if held {
if integrity == p.Integrity {
return ref, nil // this very file, published before: done without a publish
}
return "", fmt.Errorf("%s%s: the registry holds it as %s, and this build packs %s", delivery.VersionTaken, ref, integrity, p.Integrity)
var doc map[string]json.RawMessage
if err := json.Unmarshal(raw, &doc); err != nil {
return nil, fmt.Errorf("the package's package.json does not parse: %w", err)
}
file, err := os.CreateTemp(root, "publish-*.tgz")
doc["version"], _ = json.Marshal(version)
out, err := json.MarshalIndent(doc, "", " ")
if err != nil {
return "", err
return nil, err
}
defer os.Remove(file.Name()) // the packed file leaves the node once its publish answered
if _, err := file.Write(p.tgz); err != nil {
file.Close()
return "", err
if err := os.WriteFile(path, append(out, '\n'), 0o644); err != nil {
return nil, err
}
if err := file.Close(); err != nil {
return "", err
}
npmrc, err := m.npmrcIn(root)
if err != nil {
return "", err
}
defer os.Remove(npmrc)
env := append(m.baseEnv(), "npm_config_cache="+m.npmCache())
r := m.Run(ctx, env, "npm", "publish", file.Name(), "--ignore-scripts", "--userconfig", npmrc)
if why := ranFailed(r); why != "" {
return "", fmt.Errorf("publishing %s: %s", ref, why)
}
return ref, nil
return func() { _ = os.WriteFile(path, raw, 0o644) }, nil
}
@@ -0,0 +1,29 @@
package main
// The publish of a done build's file (novox/hq design 51 *Publishing*, ADR 0611; CAT-BN3): from this module's own
// process, after the build step ended, through a registry seat's verbs publish and publish-part, which the manifest
// `invokes`, with the build modules' one sender (buildstep.PublishOne): a bundle's archive on oci-artifact-registry, a
// trunk package on npm-package-registry under the version the builder gave it. No build step reaches a store, and this
// module holds no write credential: the holder checks the build with the builder and says published or refused.
import (
"context"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
)
// publishOn publishes body, of digest, as name (at version, when it has one) of kind on seat: through Call when a
// test set it, else the runtime's request over the bus (stdio.Ask).
func (m *Module) publishOn(ctx context.Context, seat, name, kind, version, digest string, body []byte) error {
call := m.Call
if call == nil {
call = stdio.Ask
}
q := kind
if version != "" {
q = "v" + version
}
return buildstep.PublishOne(ctx, call, seat, name, kind, version, digest, buildstep.TransferName(name, q, digest), body)
}
@@ -4,82 +4,18 @@ package main
import (
"context"
"encoding/base64"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// npmRegistry is a registry of the test's own speaking npm's publish protocol: a PUT of the package's document with the
// packed file attached, then GET of the document and the file; it asks for the account's credential.
type npmRegistry struct {
mu sync.Mutex
versions map[string]map[string]any
files map[string][]byte
publishes int
}
func (r *npmRegistry) serve(t *testing.T) *httptest.Server {
var srv *httptest.Server
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, q *http.Request) {
r.mu.Lock()
defer r.mu.Unlock()
if u, p, ok := q.BasicAuth(); !ok || u != "builder" || p != "the write credential" {
w.WriteHeader(http.StatusUnauthorized)
return
}
doc := strings.TrimPrefix(q.URL.EscapedPath(), "/npm/")
switch {
case q.Method == http.MethodPut && (doc == "@novox%2fmesh-sdk" || doc == "@novox%2Fmesh-sdk"):
var body struct {
Versions map[string]map[string]any `json:"versions"`
Attachments map[string]struct {
Data string `json:"data"`
} `json:"_attachments"`
}
raw, _ := io.ReadAll(q.Body)
if err := json.Unmarshal(raw, &body); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
for v, meta := range body.Versions {
for name, a := range body.Attachments {
data, _ := base64.StdEncoding.DecodeString(a.Data)
r.files[name] = data
meta["dist"] = map[string]any{"tarball": srv.URL + "/files/" + name, "integrity": integrityOf(data)}
}
r.versions[v] = meta
}
r.publishes++
w.WriteHeader(http.StatusCreated)
w.Write([]byte(`{"ok":true}`))
case q.Method == http.MethodGet && (doc == "@novox%2fmesh-sdk" || doc == "@novox%2Fmesh-sdk"):
if len(r.versions) == 0 {
http.NotFound(w, q)
return
}
json.NewEncoder(w).Encode(map[string]any{"name": "@novox/mesh-sdk", "versions": r.versions})
case strings.HasPrefix(q.URL.Path, "/files/"):
w.Write(r.files[strings.TrimPrefix(q.URL.Path, "/files/")])
default:
http.NotFound(w, q)
}
}))
t.Cleanup(srv.Close)
return srv
}
// 1587-S20 (ADR 0498 §5): the real npm publishes a trunk build's package to a registry speaking npm's protocol, with
// the module's credential, once: a second trunk build of the same package finds its file there and publishes nothing;
// a candidate publishes nothing at all.
// 1587-S20 (ADR 0498 §5), CAT-BN3 (design 51 *Publishing*): the real npm packs a trunk build's package under the
// version the builder gave it, and the module publishes that packed file through npm-package-registry's verbs, with no
// credential and no npm publish; the workspace's package.json is put back. A candidate publishes nothing at all.
func TestARealTrunkPackageIsPublishedOnce(t *testing.T) {
needBubblewrap(t)
for _, tool := range []string{"node", "npm"} {
@@ -89,34 +25,39 @@ func TestARealTrunkPackageIsPublishedOnce(t *testing.T) {
}
m, _, work := newMachine(t)
m.Run = forgekey.ExecRunner
reg := &npmRegistry{versions: map[string]map[string]any{}, files: map[string][]byte{}}
srv := reg.serve(t)
m.Toolchain.Registry = func() (Registry, error) {
return Registry{URL: srv.URL + "/npm/", Username: "builder", Password: "the write credential"}, nil
}
m.HTTP = srv.Client()
store := withStore(t, m)
files := map[string]string{}
for n, c := range sdkFixture {
files[n] = c
}
workspaceWith(t, work, files)
dir := filepath.Join(work, "w1", "tree")
before, _ := os.ReadFile(filepath.Join(dir, "package.json"))
if err := m.installDev(context.Background(), filepath.Join(work, "w1"), dir, "."); err != nil {
t.Fatal(err)
}
args := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib"}
if out, err := m.build(context.Background(), args); err != nil || out.(BuildResult).Reference != "" || reg.publishes != 0 {
t.Fatalf("a candidate: %+v %v, %d publishes", out, err, reg.publishes)
if out, err := m.build(context.Background(), args); err != nil || out.(BuildResult).Reference != "" || len(store.published) != 0 {
t.Fatalf("a candidate: %+v %v, %d published", out, err, len(store.published))
}
args["publish"] = true
args["publish"], args["version"], args["seat"] = true, "9.9.10", "npm-package-registry"
out, err := m.build(context.Background(), args)
if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@9.9.9" || reg.publishes != 1 {
t.Fatalf("a trunk build: %+v %v, %d publishes", out, err, reg.publishes)
if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@9.9.10" || len(store.published) != 1 {
t.Fatalf("a trunk build: %+v %v, %d published", out, err, len(store.published))
}
if got := digestOf(reg.files["@novox/mesh-sdk-9.9.9.tgz"]); got != out.(BuildResult).Digest {
t.Errorf("the registry holds %s, the build packed %s (files %d)", got, out.(BuildResult).Digest, len(reg.files))
p := store.published[0]
tgz := store.blobs[p.Name+"@"+p.Digest]
if p.Name != "@novox/mesh-sdk" || p.Version != "9.9.10" || p.Kind != "package" || digestOf(tgz) != out.(BuildResult).Digest {
t.Errorf("published %+v (%d bytes), the build packed %s", p, len(tgz), out.(BuildResult).Digest)
}
if again, err := m.build(context.Background(), args); err != nil || again.(BuildResult).Reference != "@novox/mesh-sdk@9.9.9" || reg.publishes != 1 {
t.Errorf("the same package again: %+v %v, %d publishes", again, err, reg.publishes)
into := t.TempDir()
if err := unpackPackage(tgz, into); err != nil {
t.Fatal(err)
}
if raw, _ := os.ReadFile(filepath.Join(into, "package.json")); !strings.Contains(string(raw), `"version": "9.9.10"`) {
t.Errorf("the packed package.json is %s, want the builder's version", raw)
}
if after, _ := os.ReadFile(filepath.Join(dir, "package.json")); string(after) != string(before) {
t.Errorf("the workspace's package.json was not put back: %s", after)
}
}
@@ -2,9 +2,9 @@ package main
import (
"context"
"errors"
"os"
"path/filepath"
"slices"
"strings"
"testing"
@@ -69,26 +69,37 @@ func TestACandidatesPackageIsBuiltAndPackedAndNeverPublished(t *testing.T) {
}
}
// ADR 0498 §5: a trunk build (publish) publishes a version the registry lacks and answers <name>@<version>; the same
// packed file already there is done without a publish; another file under that version fails `version taken:`. The
// packed file is gone from the node afterwards.
// ADR 0498 §5, CAT-BN3 (design 51 *Publishing*): a trunk build the builder gave a version packs the package under that
// version and publishes it through npm-package-registry, never by npm publish nor with a credential, and answers
// <name>@<version>; its package.json is put back. A trunk build not numbered publishes nothing; a refusal fails it
// not stored. The packed file is gone from the node afterwards.
func TestATrunkPackageIsPublishedOnceAndAVersionTakenIsRefused(t *testing.T) {
tgz := tgzOf(t, sdkPackage, "0.1.15")
args := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib", "publish": true}
tgz := tgzOf(t, sdkPackage, "0.1.16")
args := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib", "publish": true, "version": "0.1.16", "seat": "npm-package-registry"}
m, f, work := packageMachine(t, tgz)
fakeRegistry(t, m, nil)
out, err := m.build(context.Background(), args)
if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@0.1.15" || len(publishes(f)) != 1 {
t.Fatalf("an absent version: %+v %v, %d publishes", out, err, len(publishes(f)))
}
if p := publishes(f)[0]; !slices.Contains(p.args, "--ignore-scripts") || !strings.HasSuffix(p.args[1], ".tgz") {
t.Errorf("published with %v", p.args)
}
// ADR 0500 §4: the registry's credential is never in a workspace, which a run step is given.
for i, a := range publishes(f)[0].args {
if a == "--userconfig" && strings.HasPrefix(publishes(f)[0].args[i+1], work) {
t.Errorf("the credential was written into the workspace: %s", publishes(f)[0].args[i+1])
var packedAs string
inner := f.effect
f.effect = func(c call) {
if c.name == "npm" && len(c.args) > 0 && c.args[0] == "pack" {
raw, _ := os.ReadFile(filepath.Join(work, "w1", "tree", "package.json"))
packedAs = string(raw)
}
inner(c)
}
store := withStore(t, m)
out, err := m.build(context.Background(), args)
if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@0.1.16" || len(publishes(f)) != 0 {
t.Fatalf("a numbered trunk package: %+v %v, %d npm publishes", out, err, len(publishes(f)))
}
if !strings.Contains(packedAs, `"version": "0.1.16"`) {
t.Errorf("packed with package.json %s, want the builder's version", packedAs)
}
if raw, _ := os.ReadFile(filepath.Join(work, "w1", "tree", "package.json")); !strings.Contains(string(raw), `"version":"0.1.15"`) {
t.Errorf("package.json was not put back: %s", raw)
}
if len(store.published) != 1 || store.published[0].Name != "@novox/mesh-sdk" || store.published[0].Version != "0.1.16" ||
store.published[0].Kind != "package" || store.seats[0] != "npm-package-registry" || store.published[0].Digest != digestOf(tgz) {
t.Errorf("published %+v through %v", store.published, store.seats)
}
var left []string
filepath.WalkDir(work, func(p string, d os.DirEntry, err error) error {
@@ -102,15 +113,15 @@ func TestATrunkPackageIsPublishedOnceAndAVersionTakenIsRefused(t *testing.T) {
}
m, f, _ = packageMachine(t, tgz)
fakeRegistry(t, m, map[string][]byte{"0.1.15": tgz})
if out, err := m.build(context.Background(), args); err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@0.1.15" || len(publishes(f)) != 0 {
t.Errorf("the same file published before: %+v %v, %d publishes", out, err, len(publishes(f)))
store = withStore(t, m)
unnumbered := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib", "publish": true}
if out, err := m.build(context.Background(), unnumbered); err != nil || out.(BuildResult).Reference != "" || len(store.published) != 0 || len(publishes(f)) != 0 {
t.Errorf("a trunk package not numbered: %+v %v, %d published", out, err, len(store.published))
}
m, f, _ = packageMachine(t, tgz)
fakeRegistry(t, m, map[string][]byte{"0.1.15": tgzOf(t, sdkPackage, "another")})
_, err = m.build(context.Background(), args)
if err == nil || !strings.HasPrefix(err.Error(), delivery.VersionTaken+"@novox/mesh-sdk@0.1.15") || len(publishes(f)) != 0 {
t.Errorf("another file under the version: %v, %d publishes", err, len(publishes(f)))
m, _, _ = packageMachine(t, tgz)
withStore(t, m).fail = errors.New("refused: the builder did not number it")
if _, err = m.build(context.Background(), args); err == nil || !strings.HasPrefix(err.Error(), delivery.NotStored+"lib: ") {
t.Errorf("a refused publish: %v", err)
}
}
@@ -1,42 +0,0 @@
package main
// The push of a done build's archive (novox/hq ADR 0498 §1 §2; issue 1587): from this module's own process, after the
// build step ended, with mesh-sdk's one client (buildstep.PushBlob), to the artifact store whose address the manifest
// binds into RegistryFile. No build step reaches the store: a run step has no network (ADR 0448).
import (
"context"
"fmt"
"net/http"
"os"
"strings"
"time"
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
)
// registry is the artifact store's address, host:port, as the manifest bound it into RegistryFile.
func (m *Module) registry() (string, error) {
raw, err := os.ReadFile(m.RegistryFile)
if err != nil {
return "", fmt.Errorf("the artifact store's address cannot be read: %w", err)
}
addr := strings.TrimSpace(string(raw))
if addr == "" || strings.Contains(addr, "${") || strings.HasPrefix(addr, ":") || strings.HasSuffix(addr, ":") {
return "", fmt.Errorf("the artifact store's address in %s is %q, not host:port", m.RegistryFile, addr)
}
return addr, nil
}
// push puts body into the artifact store as a blob of repository (<module>/<artifact>) by its digest: Push when a test
// set it, else buildstep.PushBlob.
func (m *Module) push(ctx context.Context, repository, digest string, body []byte) error {
addr, err := m.registry()
if err != nil {
return err
}
if m.Push != nil {
return m.Push(ctx, addr, repository, digest, body)
}
return buildstep.PushBlob(ctx, &http.Client{Timeout: 10 * time.Minute}, addr, repository, digest, body)
}
@@ -3,7 +3,9 @@ package main
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"os"
"path/filepath"
@@ -11,54 +13,87 @@ import (
"sync"
"testing"
registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
)
// pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back.
// pushed are the blobs the fake holder took, by digest, for the tests that read an archive back.
var pushed sync.Map
// fakeStore is a store of the test's own: its address in RegistryFile, each blob kept by repository and digest; fail,
// when set, is the store's refusal.
// fakeStore is the registry seats' holder as m's publish meets it (publish.go; CAT-BN3): publish opens a transfer and
// answers the part it lacks, publish-part takes the chunks in sequence, and the last one keeps the whole part when it
// hashes to its digest. Each file is kept by name and digest; fail, when set, is the holder's refusal at the last chunk.
type fakeStore struct {
mu sync.Mutex
blobs map[string][]byte // "<repository>@<digest>"
fail error
mu sync.Mutex
blobs map[string][]byte // "<name>@<digest>"
fail error
open map[string]registry.PublishInput
chunks map[string][]byte
published []registry.PublishInput
seats []string
}
func withStore(t *testing.T, m *Module) *fakeStore {
t.Helper()
s := &fakeStore{blobs: map[string][]byte{}}
m.RegistryFile = filepath.Join(t.TempDir(), "registry")
if err := os.WriteFile(m.RegistryFile, []byte("store.test:5000\n"), 0o644); err != nil {
t.Fatal(err)
}
m.Push = func(ctx context.Context, registry, repository, digest string, body []byte) error {
s := &fakeStore{blobs: map[string][]byte{}, open: map[string]registry.PublishInput{}, chunks: map[string][]byte{}}
m.Call = func(key string, body any) (json.RawMessage, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.fail != nil {
return s.fail
raw, _ := json.Marshal(body)
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
if seat != buildstep.ArtifactSeat && seat != buildstep.NpmSeat {
return nil, errors.New("nats: no responders available for request")
}
if sum := sha256.Sum256(body); "sha256:"+hex.EncodeToString(sum[:]) != digest || registry != "store.test:5000" {
t.Errorf("pushed %s to %s with bytes of another digest", digest, registry)
switch verb {
case "publish":
var in registry.PublishInput
_ = json.Unmarshal(raw, &in)
if (seat == buildstep.ArtifactSeat) != (in.Kind == "bundle") || in.Transfer == "" || strings.ContainsAny(in.Transfer, "/. *>") {
t.Errorf("publish on %s of kind %q, transfer %q", seat, in.Kind, in.Transfer)
}
s.open[in.Transfer] = in
s.seats = append(s.seats, seat)
return json.Marshal(registry.PublishAnswer{Transfer: in.Transfer, Lacks: []string{in.Digest}})
case "publish-part":
var c registry.PublishPartInput
_ = json.Unmarshal(raw, &c)
data, _ := base64.StdEncoding.DecodeString(c.Data)
s.chunks[c.Transfer] = append(s.chunks[c.Transfer], data...)
if !c.Last {
return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part})
}
in, whole := s.open[c.Transfer], s.chunks[c.Transfer]
delete(s.chunks, c.Transfer)
if s.fail != nil {
return nil, s.fail
}
if sum := sha256.Sum256(whole); "sha256:"+hex.EncodeToString(sum[:]) != in.Digest {
t.Errorf("published %s with bytes of another digest", in.Digest)
}
s.blobs[in.Name+"@"+in.Digest] = whole
s.published = append(s.published, in)
pushed.Store(in.Digest, whole)
return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part, Complete: true})
}
s.blobs[repository+"@"+digest] = append([]byte(nil), body...)
pushed.Store(digest, s.blobs[repository+"@"+digest])
return nil
return nil, errors.New("nats: no responders available for request")
}
return s
}
// archiveOf is the archive a build pushed, by its digest.
// archiveOf is the archive a build published, by its digest.
func archiveOf(t *testing.T, r BuildResult) []byte {
t.Helper()
raw, ok := pushed.Load(r.Digest)
if !ok {
t.Fatalf("nothing was pushed under %s", r.Digest)
t.Fatalf("nothing was published under %s", r.Digest)
}
return raw.([]byte)
}
// 1587-S6 (ADR 0498 §1): a done bundle answers the reference of the archive it pushed, under the manifest's module.
// 1587-S6 (ADR 0498 §1; CAT-BN3): a done bundle is published through oci-artifact-registry and answers the seat's
// reference of its archive, under the manifest's module.
func TestADoneBundleAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) {
m, _, _ := builtMachine(t, ``)
store := withStore(t, m)
@@ -67,23 +102,33 @@ func TestADoneBundleAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) {
t.Fatal(err)
}
r := out.(BuildResult)
if want := "artifact-store://m/tools/blobs/" + r.Digest; r.Reference != want {
if want := "oci-artifact-registry/m/tools@" + r.Digest; r.Reference != want {
t.Errorf("reference %q, want %q", r.Reference, want)
}
if store.blobs["m/tools@"+r.Digest] == nil {
t.Errorf("the store holds %d blobs, not m/tools@%s", len(store.blobs), r.Digest)
if store.blobs["m/tools@"+r.Digest] == nil || len(store.seats) != 1 || store.seats[0] != buildstep.ArtifactSeat {
t.Errorf("the holder holds %d files through %v, not m/tools@%s through %s", len(store.blobs), store.seats, r.Digest, buildstep.ArtifactSeat)
}
}
// 1587-S7 and S8 (ADR 0498 §2, the lead's decision L9): a push the store refuses fails the step `not stored:
// <artifact>: <the store's words>`, and whatever the push answered, no packed file is left on the node.
// 1587-S7 and S8 (ADR 0498 §2, the lead's decision L9): a publish the holder refuses fails the step `not stored:
// <artifact>: <the holder's words>`, a seat nothing holds too, and whatever it answered, no packed file is left.
func TestARefusedPushFailsNotStoredAndNoPackedFileStays(t *testing.T) {
for _, fail := range []error{nil, errors.New("store.test:5000 refused the blob: 507 Insufficient Storage")} {
for _, c := range []struct {
fail error
noSeat bool
words string
}{{nil, false, ""}, {errors.New("refused: the store did not take it: 507 Insufficient Storage"), false, "507"},
{nil, true, "no responders"}} {
m, _, work := builtMachine(t, ``)
withStore(t, m).fail = fail
withStore(t, m).fail = c.fail
if c.noSeat {
m.Call = func(string, any) (json.RawMessage, error) {
return nil, errors.New("nats: no responders available for request")
}
}
_, err := m.build(context.Background(), buildArgs())
if fail != nil && (err == nil || !strings.HasPrefix(err.Error(), delivery.NotStored+"tools: ") || !strings.Contains(err.Error(), "507")) {
t.Errorf("a refused push answered %v", err)
if c.words != "" && (err == nil || !strings.HasPrefix(err.Error(), delivery.NotStored+"tools: ") || !strings.Contains(err.Error(), c.words)) {
t.Errorf("a refused publish answered %v", err)
}
var archives []string
filepath.WalkDir(work, func(p string, d os.DirEntry, err error) error {
@@ -93,7 +138,7 @@ func TestARefusedPushFailsNotStoredAndNoPackedFileStays(t *testing.T) {
return nil
})
if len(archives) != 0 {
t.Errorf("push answered %v: packed files left on the node: %v", fail, archives)
t.Errorf("publish answered %v: packed files left on the node: %v", c.fail, archives)
}
}
}
@@ -3,16 +3,13 @@ package main
// How the toolchain's install reaches the mesh's package registry (novox/hq ADR 0405 §5, ADR 0336; issue 1488).
// The lock names @novox/mesh-sdk at the registry's public address, and npm installs a locked tarball only from the
// registry it is configured with, so the install is told that registry for the @novox scope and nothing else: every
// other package resolves from the public one. This is how today's build agent names it (mesh-controller
// internal/builder/dependencies.go: the scope's registry from the npm-package-registry binding, the account and
// secret the mesh sealed), read here from the module's own binding and secret files (module.json `binds` and
// `secrets`), the same files the lab module reads.
// other package resolves from the public one. The address is read from the module's npm-package-registry binding
// (module.json `binds`). Reads are anonymous (ADR 0346; the provision's form none, CAT-BN3): this module holds no
// credential for the registry, and publishes only through the seat's verbs (publish.go).
import (
"encoding/base64"
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
)
@@ -22,19 +19,18 @@ const Scope = "@novox"
// Registry is the mesh's package registry as the binding names it.
type Registry struct {
URL, Username, Password string
URL string
}
// ReadRegistry reads the npm-package-registry binding and its sealed secret. The registry is the binding's
// ReadRegistry reads the npm-package-registry binding. The registry is the binding's
// `public-root` when it has one (the address the lock holds), and otherwise scheme://at:port plus `npm-path`.
func ReadRegistry(bindingPath, secretPath string) (Registry, error) {
func ReadRegistry(bindingPath string) (Registry, error) {
raw, err := os.ReadFile(bindingPath)
if err != nil {
return Registry{}, fmt.Errorf("no package registry is bound to this module: %w", err)
}
var b struct {
At string `json:"at"`
As string `json:"as"`
Serves struct {
Scheme string `json:"scheme"`
Port int `json:"port"`
@@ -59,31 +55,10 @@ func ReadRegistry(bindingPath, secretPath string) (Registry, error) {
if !strings.HasSuffix(url, "/") {
url += "/"
}
secret, err := os.ReadFile(secretPath)
if err != nil {
return Registry{}, fmt.Errorf("the package registry's secret cannot be read: %w", err)
}
return Registry{URL: url, Username: b.As, Password: strings.TrimSpace(string(secret))}, nil
return Registry{URL: url}, nil
}
// Npmrc is the npm configuration text for the install: the @novox scope at the registry, with the account's
// credential for it when there is one.
// Npmrc is the npm configuration text for the install: the @novox scope at the registry, read anonymously.
func (r Registry) Npmrc() string {
var sb strings.Builder
fmt.Fprintf(&sb, "%s:registry=%s\n", Scope, r.URL)
if r.Username != "" && r.Password != "" {
hostPath := r.URL
if i := strings.Index(hostPath, "://"); i >= 0 {
hostPath = hostPath[i+1:]
}
fmt.Fprintf(&sb, "%s:_auth=%s\n", hostPath, base64.StdEncoding.EncodeToString([]byte(r.Username+":"+r.Password)))
}
return sb.String()
}
// authorize gives a request to the registry the account's credential, when there is one.
func (r Registry) authorize(req *http.Request) {
if r.Username != "" && r.Password != "" {
req.SetBasicAuth(r.Username, r.Password)
}
return fmt.Sprintf("%s:registry=%s\n", Scope, r.URL)
}
@@ -3,14 +3,16 @@
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"net/url"
"strings"
"testing"
@@ -18,12 +20,15 @@ import (
"github.com/distribution/distribution/v3/registry/handlers"
_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// realStore starts the artifact store's own program (the distribution registry, modules/distribution) inside this
// test, with its in-memory storage, on a loopback port, and points m at it with the real push client. It is imported
// by this integration test alone: the module's program does not carry it.
// test, with its in-memory storage, on a loopback port, and plays the artifact seat's holder storing into it: the fake
// holder (push_test.go) takes the publish, and each file it took is uploaded into the store as distribution's holder
// does. It is imported by this integration test alone: the module's program does not carry it.
func realStore(t *testing.T, m *Module) string {
t.Helper()
config := &configuration.Configuration{Storage: configuration.Storage{"inmemory": configuration.Parameters{}}}
@@ -33,14 +38,57 @@ func realStore(t *testing.T, m *Module) string {
srv := httptest.NewServer(handlers.NewApp(context.Background(), config))
t.Cleanup(srv.Close)
addr := strings.TrimPrefix(srv.URL, "http://")
m.RegistryFile = filepath.Join(t.TempDir(), "registry")
if err := os.WriteFile(m.RegistryFile, []byte(addr+"\n"), 0o644); err != nil {
t.Fatal(err)
s := withStore(t, m)
inner := m.Call
m.Call = func(key string, body any) (json.RawMessage, error) {
raw, err := inner(key, body)
var a registry.PublishPartAnswer
if err != nil || !strings.HasSuffix(key, ".publish-part") || json.Unmarshal(raw, &a) != nil || !a.Complete {
return raw, err
}
s.mu.Lock()
defer s.mu.Unlock()
for k, b := range s.blobs {
name, digest, _ := strings.Cut(k, "@")
if err := uploadBlob(srv.URL, name, digest, b); err != nil {
return nil, err
}
delete(s.blobs, k)
}
return raw, nil
}
m.Push = nil
return addr
}
// uploadBlob is the store's monolithic blob upload, a POST opening it and a PUT with the digest, as the holder does.
func uploadBlob(base, name, digest string, body []byte) error {
resp, err := http.Post(base+"/v2/"+name+"/blobs/uploads/", "", nil)
if err != nil {
return err
}
resp.Body.Close()
loc, err := url.Parse(resp.Header.Get("Location"))
if err != nil || resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("the store opened no upload of %s: %s %v", digest, resp.Status, err)
}
b, _ := url.Parse(base)
u := b.ResolveReference(loc)
q := u.Query()
q.Set("digest", digest)
u.RawQuery = q.Encode()
req, _ := http.NewRequest(http.MethodPut, u.String(), bytes.NewReader(body))
req.Header.Set("Content-Type", "application/octet-stream")
put, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
put.Body.Close()
if put.StatusCode != http.StatusCreated {
return fmt.Errorf("the store did not take %s: %s", digest, put.Status)
}
return nil
}
// 1587-S10 (ADR 0498's check): a done bundle names a reference whose tail is its digest, and that reference resolves in
// a real artifact store to bytes of that digest. The build is the real tsc's and esbuild's, shut off.
func TestADoneBundleResolvesInARealArtifactStore(t *testing.T) {
@@ -57,11 +105,11 @@ func TestADoneBundleResolvesInARealArtifactStore(t *testing.T) {
if _, err := delivery.EncodeBuilderBuild(done); err != nil {
t.Errorf("the done key the builder would write is refused: %v", err)
}
path, ok := strings.CutPrefix(r.Reference, "artifact-store://")
if !ok || !strings.HasSuffix(r.Reference, "/blobs/"+r.Digest) {
t.Fatalf("the reference %q", r.Reference)
if _, _, ok := catalogue.InRegistrySeat(r.Reference); !ok || !strings.HasSuffix(r.Reference, "@"+r.Digest) {
t.Fatalf("the reference %q is not in the form a registry seat answers", r.Reference)
}
resp, err := http.Get("http://" + addr + "/v2/" + path)
// Read as a node-engine reads it: the seat's reference routed to the store's address.
resp, err := http.Get(catalogue.Routed(r.Reference, addr))
if err != nil {
t.Fatal(err)
}
@@ -25,7 +25,7 @@ const installedMark = ".installed"
// Paths are the module's places on its machine.
type Paths struct {
// Home is the account's home (the state directory): where the mesh writes toolchain/package.json and
// toolchain/package-lock.json (module.json resources) and the registry binding and secret.
// toolchain/package-lock.json (module.json resources) and the registry binding.
Home string
// Cache is the cache directory, outside the home (ADR 0466 §1).
Cache string
@@ -49,7 +49,7 @@ func toolchainIn(t *testing.T, fake *fakeNpm) Toolchain {
os.WriteFile(filepath.Join(home, "toolchain", "package-lock.json"), []byte(`{"lock":1}`), 0o600)
return Toolchain{Paths: Paths{Home: home, Cache: cache}, Run: fake.run,
Registry: func() (Registry, error) {
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "pw"}, nil
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil
}}
}
@@ -106,13 +106,10 @@ func TestTheInstallNamesTheRegistryInAConfigFileNotOnTheCommandLine(t *testing.T
if err := tc.Ensure(context.Background(), "v1"); err != nil {
t.Fatal(err)
}
if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/\n") ||
!strings.Contains(npmrc, "//git.novox.be/api/packages/novox/npm/:_auth=") {
// Reads are anonymous (ADR 0346, CAT-BN3): the scope's registry and no credential.
if npmrc != "@novox:registry=https://git.novox.be/api/packages/novox/npm/\n" {
t.Errorf("npmrc = %q", npmrc)
}
if strings.Contains(strings.Join(fake.calls[0].args, " "), "pw") {
t.Errorf("the credential is on the command line: %v", fake.calls[0].args)
}
}
// 1488/AC4: an install that fails leaves no directory taken for a toolchain, says why, and the next call tries again.
@@ -158,18 +155,17 @@ func TestPrepareInstallsOnlyBehindAPassingGuard(t *testing.T) {
func TestRegistryFromTheBinding(t *testing.T) {
dir := t.TempDir()
b, s := filepath.Join(dir, "b.json"), filepath.Join(dir, "s")
os.WriteFile(s, []byte("secret\n"), 0o600)
b := filepath.Join(dir, "b.json")
os.WriteFile(b, []byte(`{"provision":"npm-package-registry","at":"10.42.0.1","as":"mesh-build-node-ace","serves":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/"}}`), 0o600)
r, err := ReadRegistry(b, s)
if err != nil || r.URL != "http://10.42.0.1:3000/api/packages/novox/npm/" || r.Username != "mesh-build-node-ace" || r.Password != "secret" {
r, err := ReadRegistry(b)
if err != nil || r.URL != "http://10.42.0.1:3000/api/packages/novox/npm/" || r.Npmrc() != "@novox:registry=http://10.42.0.1:3000/api/packages/novox/npm/\n" {
t.Fatalf("= %+v, %v", r, err)
}
os.WriteFile(b, []byte(`{"at":"h","as":"a","serves":{"port":1,"npm-path":"/p/","public-root":"https://git.novox.be/p"}}`), 0o600)
if r, _ := ReadRegistry(b, s); r.URL != "https://git.novox.be/p/" {
if r, _ := ReadRegistry(b); r.URL != "https://git.novox.be/p/" {
t.Errorf("the public root is not preferred: %q", r.URL)
}
if _, err := ReadRegistry(filepath.Join(dir, "none"), s); err == nil {
if _, err := ReadRegistry(filepath.Join(dir, "none")); err == nil {
t.Errorf("no binding was accepted")
}
}