Merge pull request 'Design 47 H1: a trunk build places every mesh package it names; the bus module's whole test waits for a fill pass (H-A-267)' (#728) from fix/47-H1-catalog-trunk-packages-and-nats-test into main

This commit was merged in pull request #728.
This commit is contained in:
2026-11-21 14:49:34 +00:00
5 changed files with 310 additions and 61 deletions
@@ -245,6 +245,11 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group
return nil, err
}
}
// Every other mesh package it names, at the latest release (design 47 H, H-A-267); the group's and the
// members' builds are placed over them.
if err := m.placeMeshPackages(ctx, dir, d, served, members); err != nil {
return nil, err
}
if err := placeGroup(dir, served); err != nil {
return nil, err
}
@@ -2,6 +2,8 @@ package main
import (
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"slices"
@@ -97,9 +99,17 @@ func TestAModuleNamingOnlyTheSdkAndTypesInstallsNothing(t *testing.T) {
}
// 47/G A-72: a module naming another @novox/mesh-* package beside the SDK has no package beyond the SDK, so it installs
// nothing and is not refused for want of a lock.
// nothing and is not refused for want of a lock. 47/H H-A-267 (ADR 0573 §2): a fetch with no group and no members' builds
// places that package at the latest release the registry holds, as it places the SDK, whatever the module's lock names.
func TestAMeshPackageIsNotBeyondTheSdk(t *testing.T) {
m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","@novox/mesh-tools":"^0.1.6"}}`), nil)
const contracts = "@novox/mesh-delivery-contracts-git"
files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","` + contracts + `":"^0.1.2"}}`)
files["modules/m/package-lock.json"] = `{"lockfileVersion":3,"packages":{"node_modules/` + contracts + `":{"version":"0.1.2","integrity":"sha512-old"}}}`
m, f := fetchMachine(t, files, nil)
fakeRegistryOf(t, m, map[string]map[string][]byte{
sdkPackage: {"0.1.15": tgzOf(t, sdkPackage, "0.1.15")},
contracts: {"0.1.2": tgzOf(t, contracts, "0.1.2"), "0.1.4": tgzOf(t, contracts, "0.1.4"), "0.1.5-feat.1": tgzOf(t, contracts, "0.1.5-feat.1")},
})
out, err := m.fetch(context.Background(), fetchArgs)
if err != nil {
t.Fatal(err)
@@ -108,6 +118,42 @@ func TestAMeshPackageIsNotBeyondTheSdk(t *testing.T) {
if len(npmCalls(f)) != 0 || len(r.Installed) != 0 || len(r.Refused) != 0 {
t.Errorf("result = %+v, npm calls %d, want nothing installed and nothing refused", r, len(npmCalls(f)))
}
dir := filepath.Join(m.Work, r.Workspace, "tree", "modules", "m")
got, err := readPackageJSON(filepath.Join(dir, "node_modules", filepath.FromSlash(contracts)))
if err != nil || got.Version != "0.1.4" {
t.Errorf("%s placed %q (%v), want the latest release 0.1.4", contracts, got.Version, err)
}
if v := placedSDK(dir); v != "0.1.15" || r.SDK["modules/m"] != "0.1.15" {
t.Errorf("the SDK placed %q, answered %q, want the latest release 0.1.15", v, r.SDK["modules/m"])
}
// A member's build of a mesh package takes precedence: it is laid, and the registry, holding none of it, is not
// waited on.
const member = "@novox/mesh-tools"
m, _ = fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","`+member+`":"^0.1.6"}}`), nil)
tgz := tgzOf(t, member, "0.0.0-member")
digest := digestOf(tgz)
store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v2/tools/pkg/blobs/"+digest {
http.NotFound(w, r)
return
}
w.Write(tgz)
}))
defer store.Close()
m.ArtifactStoreFile = filepath.Join(t.TempDir(), "artifact-store.json")
write(t, m.ArtifactStoreFile, `{"at":"`+strings.TrimPrefix(store.URL, "http://")+`"}`, 0o600)
args := map[string]any{"repository": fetchArgs["repository"], "commit": fetchArgs["commit"],
"builds": []any{map[string]any{"package": member, "module": "tools", "artifact": "pkg", "digest": digest}}}
out, err = m.fetch(context.Background(), args)
if err != nil {
t.Fatalf("a fetch with a member's build of %s: %v", member, err)
}
r = out.(FetchResult)
got, err = readPackageJSON(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "node_modules", filepath.FromSlash(member)))
if err != nil || got.Version != "0.0.0-member" {
t.Errorf("%s placed %q (%v), want the member's build", member, got.Version, err)
}
}
// 1489/AC2: a module naming lodash with a lock gets npm ci --omit=dev --ignore-scripts (fake runner), its install put
@@ -364,9 +364,9 @@ func placeGroup(dir string, served map[string]*packed) error {
return nil
}
// sdkLocked is what a module's lock names of the SDK: its version, its integrity and where it was resolved from; ok is
// false when the lock names none (the latest release is then placed, placeSDK).
func sdkLocked(dir string) (version, integrity string, ok bool, err error) {
// locked is what a module's lock names of the package name: its version and its integrity; ok is false when the lock
// names none (the latest release is then placed, placeLatest).
func locked(dir, name string) (version, integrity string, ok bool, err error) {
raw, err := os.ReadFile(filepath.Join(dir, "package-lock.json"))
if os.IsNotExist(err) {
return "", "", false, nil
@@ -383,7 +383,7 @@ func sdkLocked(dir string) (version, integrity string, ok bool, err error) {
if err := json.Unmarshal(raw, &lock); err != nil {
return "", "", false, fmt.Errorf("the package-lock.json is not JSON: %w", err)
}
p, ok := lock.Packages["node_modules/"+sdkPackage]
p, ok := lock.Packages["node_modules/"+name]
if !ok || p.Version == "" {
return "", "", false, nil
}
@@ -398,51 +398,197 @@ func sdkLocked(dir string) (version, integrity string, ok bool, err error) {
// none. The lock's integrity is checked when the lock names the registry's version placed; a registry holding no
// release is a wait (`waits for:`).
func (m *Module) placeSDK(ctx context.Context, dir, rel string, group *packed) error {
version, integrity, _, err := sdkLocked(dir)
if group == nil {
return m.placeLatest(ctx, dir, rel, sdkPackage)
}
version, _, _, err := locked(dir, sdkPackage)
if err != nil {
return err
}
var tgz []byte
if group != nil {
if m.Say != nil && version != "" && version != group.Version {
m.Say("%s: its lock names %s@%s; the group's %s is placed (design 47 G A-284)", rel, sdkPackage, version, group.Version)
}
tgz = group.tgz
} else {
latest, err := m.registryLatest(ctx, sdkPackage)
if err != nil {
return err
}
if latest == "" {
return fmt.Errorf("%s%s", builds.WaitsFor, sdkPackage)
}
if version != latest {
if m.Say != nil && version != "" {
m.Say("%s: its lock names %s@%s; the latest published %s is placed (ADR 0573 §2)", rel, sdkPackage, version, latest)
}
version, integrity = latest, ""
}
want := sdkPackage + "@" + version
got, registered, held, err := m.registryPackageChecked(ctx, sdkPackage, version)
if err != nil {
return err
}
if !held {
return fmt.Errorf("%s%s", builds.WaitsFor, want)
}
if integrity == "" {
integrity = registered
}
if integrity != "" && integrityOf(got) != integrity {
return fmt.Errorf("the registry's %s is %s, and the lock of %s names %s", want, integrityOf(got), rel, integrity)
}
tgz = got
if m.Say != nil && version != "" && version != group.Version {
m.Say("%s: its lock names %s@%s; the group's %s is placed (design 47 G A-284)", rel, sdkPackage, version, group.Version)
}
into := filepath.Join(dir, "node_modules", filepath.FromSlash(sdkPackage))
if err := os.RemoveAll(into); err != nil {
return err
}
return unpackPackage(tgz, into)
return unpackPackage(group.tgz, into)
}
// placeMeshPackages places in the module's node_modules each mesh package beyond the SDK (@novox/mesh-*) its
// package.json names at the latest release the registry holds, as placeSDK does for the SDK (ADR 0573 §2; design 47 H,
// H-A-267): the install leaves them out (ownDependencies). One the group serves or a member's build lays is left to
// placeGroup and layBuilds, which take precedence; the module itself is not placed.
func (m *Module) placeMeshPackages(ctx context.Context, dir, rel string, served map[string]*packed, members []storedBuild) error {
self, _ := readPackageJSON(dir)
var names []string
for name := range namedPackages(dir) {
if name == sdkPackage || !strings.HasPrefix(name, "@novox/mesh-") || name == self.Name || served[name] != nil || namesPackage(members, name) {
continue
}
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
if err := m.placeLatest(ctx, dir, rel, name); err != nil {
return err
}
}
return nil
}
// placeLatest places in the module's node_modules the latest release of the mesh package name the registry holds,
// whatever the module's lock names (ADR 0573 §2), checked against the lock's integrity when the lock names that
// version; a registry holding no release is a wait (`waits for:`).
func (m *Module) placeLatest(ctx context.Context, dir, rel, name string) error {
version, integrity, _, err := locked(dir, name)
if err != nil {
return err
}
latest, err := m.registryLatest(ctx, name)
if err != nil {
return err
}
if latest == "" {
return fmt.Errorf("%s%s", builds.WaitsFor, name)
}
if version != latest {
if m.Say != nil && version != "" {
m.Say("%s: its lock names %s@%s; the latest published %s is placed (ADR 0573 §2)", rel, name, version, latest)
}
version, integrity = latest, ""
}
if spec := namedRange(dir, name); spec != "" && m.Say != nil {
// Said, never refused: the latest release is placed whatever the range (ADR 0573 §2; design 47 H, H-A-276).
switch in, read := inRange(spec, version); {
case !read:
m.Say("%s: its package.json names %s at %q, a range not read; the latest published %s is placed (ADR 0573 §2)", rel, name, spec, version)
case !in:
m.Say("%s: the latest published %s@%s lies outside the range %q its package.json names; it is placed (ADR 0573 §2)", rel, name, version, spec)
}
}
want := name + "@" + version
got, registered, held, err := m.registryPackageChecked(ctx, name, version)
if err != nil {
return err
}
if !held {
return fmt.Errorf("%s%s", builds.WaitsFor, want)
}
if integrity == "" {
integrity = registered
}
if integrity != "" && integrityOf(got) != integrity {
return fmt.Errorf("the registry's %s is %s, and the lock of %s names %s", want, integrityOf(got), rel, integrity)
}
into := filepath.Join(dir, "node_modules", filepath.FromSlash(name))
if err := os.RemoveAll(into); err != nil {
return err
}
return unpackPackage(got, into)
}
// namedRange is the range the module's package.json names of name: its dependencies first, then its peer, optional and
// dev dependencies; "" when it names none.
func namedRange(dir, name string) string {
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
if err != nil {
return ""
}
var p map[string]any
if json.Unmarshal(raw, &p) != nil {
return ""
}
for _, k := range []string{"dependencies", "peerDependencies", "optionalDependencies", "devDependencies"} {
deps, _ := p[k].(map[string]any)
if spec, ok := deps[name].(string); ok {
return strings.TrimSpace(spec)
}
}
return ""
}
// inRange says whether the release v (major.minor.patch) lies in the npm range spec; read is false for a range of a
// form it does not read (a union, a hyphen range, a comparator, a tag other than latest), never taken as in or out. It
// reads what a package file of the mesh names: an exact version, an x-range, a caret and a tilde range, as npm does.
func inRange(spec, v string) (in, read bool) {
spec = strings.TrimSpace(spec)
if spec == "" || spec == "*" || spec == "latest" {
return true, true
}
if strings.ContainsAny(spec, " |<>") {
return false, false
}
op := ""
if spec[0] == '^' || spec[0] == '~' {
op, spec = spec[:1], spec[1:]
}
spec = strings.TrimPrefix(strings.TrimPrefix(spec, "="), "v")
if strings.Contains(spec, "-") || strings.Contains(spec, "+") {
return false, false // a pre-release or build range
}
var low [3]int
given := 0
for i, part := range strings.Split(spec, ".") {
if i > 2 {
return false, false
}
if part == "x" || part == "X" || part == "*" {
break
}
n, err := strconv.Atoi(part)
if err != nil || n < 0 {
return false, false
}
low[i] = n
given = i + 1
}
if given == 0 {
return true, true // x, x.x
}
var at [3]int
for i, part := range strings.Split(v, ".") {
if i > 2 {
return false, false
}
n, err := strconv.Atoi(part)
if err != nil {
return false, false
}
at[i] = n
}
// bump is the part of low raised by one for the exclusive upper bound, the parts after it zero.
bump := given - 1
switch op {
case "^":
switch {
case low[0] > 0 || given == 1:
bump = 0
case low[1] > 0 || given == 2:
bump = 1
default:
bump = 2
}
case "~":
bump = 0
if given > 1 {
bump = 1
}
}
high := low
high[bump]++
for i := bump + 1; i < 3; i++ {
high[i] = 0
}
less := func(a, b [3]int) bool {
for i := range a {
if a[i] != b[i] {
return a[i] < b[i]
}
}
return false
}
return !less(at, low) && less(at, high), true
}
// placedSDK is the version of the SDK placed in the module's node_modules ("" when none is).
@@ -6,6 +6,7 @@ import (
"compress/gzip"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
@@ -33,34 +34,40 @@ func tgzOf(t *testing.T, name, version string) []byte {
return buf.Bytes()
}
// fakeRegistry serves the packument and the packed files of the versions it holds; asked counts its answers.
// fakeRegistry serves the packument and the packed files of the SDK's versions it holds; asked counts its answers.
func fakeRegistry(t *testing.T, m *Module, held map[string][]byte) *atomic.Int32 {
t.Helper()
return fakeRegistryOf(t, m, map[string]map[string][]byte{sdkPackage: held})
}
// fakeRegistryOf serves the packument and the packed files of each package's versions it holds (by name); asked counts
// its answers.
func fakeRegistryOf(t *testing.T, m *Module, held map[string]map[string][]byte) *atomic.Int32 {
t.Helper()
var asked atomic.Int32
var srv *httptest.Server
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
asked.Add(1)
switch {
case r.URL.Path == "/npm/@novox%2fmesh-sdk" || r.URL.Path == "/npm/@novox/mesh-sdk":
if len(held) == 0 {
http.NotFound(w, r)
return
}
versions := map[string]any{}
for v, tgz := range held {
versions[v] = map[string]any{"dist": map[string]any{"tarball": srv.URL + "/files/" + v + ".tgz", "integrity": integrityOf(tgz)}}
}
json.NewEncoder(w).Encode(map[string]any{"versions": versions})
case strings.HasPrefix(r.URL.Path, "/files/"):
v := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/files/"), ".tgz")
if tgz, ok := held[v]; ok {
if f, ok := strings.CutPrefix(r.URL.Path, "/files/"); ok {
name, v, _ := strings.Cut(strings.TrimSuffix(f, ".tgz"), "@@")
if tgz, ok := held[strings.ReplaceAll(name, "~", "/")][v]; ok {
w.Write(tgz)
return
}
http.NotFound(w, r)
default:
http.NotFound(w, r)
return
}
name, ok := strings.CutPrefix(r.URL.Path, "/npm/")
name = strings.ReplaceAll(name, "%2f", "/")
if !ok || len(held[name]) == 0 {
http.NotFound(w, r)
return
}
versions := map[string]any{}
for v, tgz := range held[name] {
versions[v] = map[string]any{"dist": map[string]any{"tarball": srv.URL + "/files/" + strings.ReplaceAll(name, "/", "~") + "@@" + v + ".tgz", "integrity": integrityOf(tgz)}}
}
json.NewEncoder(w).Encode(map[string]any{"versions": versions})
}))
t.Cleanup(srv.Close)
m.Toolchain.Registry = func() (Registry, error) { return Registry{URL: srv.URL + "/npm/"}, nil }
@@ -183,3 +190,46 @@ func TestTheGroupPackagesArgumentIsRefusedWhenItIsNotOne(t *testing.T) {
t.Errorf("a member's root and a member's directory: %+v %v", gs, err)
}
}
// design 47 H, H-A-276 (amending H-A-267): the latest release placed outside the range the package file names is said in
// the journal, and placed all the same; one inside it is not said; a range of a form not read is said as such.
func TestALatestReleaseOutsideItsRangeIsSaid(t *testing.T) {
for _, c := range []struct {
spec, v string
in, read bool
}{
{"^0.1.0", "0.1.9", true, true}, {"^0.1.0", "0.2.0", false, true}, {"^1.2.3", "1.9.0", true, true},
{"^1.2.3", "2.0.0", false, true}, {"^0.0.3", "0.0.4", false, true}, {"^0.x", "0.9.0", true, true},
{"~0.1.2", "0.1.9", true, true}, {"~0.1.2", "0.2.0", false, true}, {"~1", "1.5.0", true, true},
{"0.1.4", "0.1.4", true, true}, {"0.1.4", "0.1.5", false, true}, {"0.1.x", "0.1.7", true, true},
{"0.1", "0.2.0", false, true}, {"*", "9.9.9", true, true}, {"^0.1.0", "0.0.9", false, true},
{">=0.1.0", "0.2.0", false, false}, {"^0.1.0 || ^0.2.0", "0.2.0", false, false}, {"file:../x", "0.1.0", false, false},
} {
if in, read := inRange(c.spec, c.v); in != c.in || read != c.read {
t.Errorf("inRange(%q, %q) = %v, %v; want %v, %v", c.spec, c.v, in, read, c.in, c.read)
}
}
m, _, _ := newMachine(t)
const name = "@novox/mesh-delivery-contracts-git"
fakeRegistryOf(t, m, map[string]map[string][]byte{name: {"0.1.9": tgzOf(t, name, "0.1.9"), "0.2.0": tgzOf(t, name, "0.2.0")}})
for _, c := range []struct{ spec, said string }{
{"^0.1.0", `the latest published ` + name + `@0.2.0 lies outside the range "^0.1.0"`},
{"^0.2.0", ""},
{">=0.1.0", `at ">=0.1.0", a range not read`},
} {
var said []string
m.Say = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
dir := t.TempDir()
write(t, filepath.Join(dir, "package.json"), `{"dependencies":{"`+name+`":"`+c.spec+`"}}`, 0o644)
if err := m.placeLatest(context.Background(), dir, "modules/m", name); err != nil {
t.Fatalf("%s: %v", c.spec, err)
}
if p, _ := readPackageJSON(filepath.Join(dir, "node_modules", filepath.FromSlash(name))); p.Version != "0.2.0" {
t.Errorf("%s: placed %q, want the latest release 0.2.0", c.spec, p.Version)
}
joined := strings.Join(said, "\n")
if c.said == "" && joined != "" || c.said != "" && !strings.Contains(joined, c.said) {
t.Errorf("%s: said %q, want %q", c.spec, joined, c.said)
}
}
}
@@ -316,7 +316,9 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
_ = Keep(kctx, deaf, func(context.Context, func(string, []byte) error) error { return errors.New("no consumer yet") }, nil, nil, t.Logf)
close(done)
}()
eventually(t, "the first view answered", func() bool { return !deaf.Status().FirstView.Answered.IsZero() })
// A first view listing no module is none yet and never marked answered (A-367): the first fill pass is waited
// for instead, then taken as the first (design 47 H, H-A-268).
eventually(t, "the first fill pass", func() bool { return !deaf.Status().Pass.IsZero() })
first := deaf.Status().Pass
eventually(t, "a fill pass while the consumer is bound again", func() bool { return deaf.Status().Pass.After(first) })
stopDeaf()