diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go index 623bec652..f53b14107 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go @@ -245,6 +245,11 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group return nil, err } } + // Every other mesh package it names, at the latest release (design 47 H, H-A-267); the group's and the + // members' builds are placed over them. + if err := m.placeMeshPackages(ctx, dir, d, served, members); err != nil { + return nil, err + } if err := placeGroup(dir, served); err != nil { return nil, err } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go index 9571a76d6..6d272a14f 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go @@ -2,6 +2,8 @@ package main import ( "context" + "net/http" + "net/http/httptest" "os" "path/filepath" "slices" @@ -97,9 +99,17 @@ func TestAModuleNamingOnlyTheSdkAndTypesInstallsNothing(t *testing.T) { } // 47/G A-72: a module naming another @novox/mesh-* package beside the SDK has no package beyond the SDK, so it installs -// nothing and is not refused for want of a lock. +// nothing and is not refused for want of a lock. 47/H H-A-267 (ADR 0573 §2): a fetch with no group and no members' builds +// places that package at the latest release the registry holds, as it places the SDK, whatever the module's lock names. func TestAMeshPackageIsNotBeyondTheSdk(t *testing.T) { - m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","@novox/mesh-tools":"^0.1.6"}}`), nil) + const contracts = "@novox/mesh-delivery-contracts-git" + files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","` + contracts + `":"^0.1.2"}}`) + files["modules/m/package-lock.json"] = `{"lockfileVersion":3,"packages":{"node_modules/` + contracts + `":{"version":"0.1.2","integrity":"sha512-old"}}}` + m, f := fetchMachine(t, files, nil) + fakeRegistryOf(t, m, map[string]map[string][]byte{ + sdkPackage: {"0.1.15": tgzOf(t, sdkPackage, "0.1.15")}, + contracts: {"0.1.2": tgzOf(t, contracts, "0.1.2"), "0.1.4": tgzOf(t, contracts, "0.1.4"), "0.1.5-feat.1": tgzOf(t, contracts, "0.1.5-feat.1")}, + }) out, err := m.fetch(context.Background(), fetchArgs) if err != nil { t.Fatal(err) @@ -108,6 +118,42 @@ func TestAMeshPackageIsNotBeyondTheSdk(t *testing.T) { if len(npmCalls(f)) != 0 || len(r.Installed) != 0 || len(r.Refused) != 0 { t.Errorf("result = %+v, npm calls %d, want nothing installed and nothing refused", r, len(npmCalls(f))) } + dir := filepath.Join(m.Work, r.Workspace, "tree", "modules", "m") + got, err := readPackageJSON(filepath.Join(dir, "node_modules", filepath.FromSlash(contracts))) + if err != nil || got.Version != "0.1.4" { + t.Errorf("%s placed %q (%v), want the latest release 0.1.4", contracts, got.Version, err) + } + if v := placedSDK(dir); v != "0.1.15" || r.SDK["modules/m"] != "0.1.15" { + t.Errorf("the SDK placed %q, answered %q, want the latest release 0.1.15", v, r.SDK["modules/m"]) + } + + // A member's build of a mesh package takes precedence: it is laid, and the registry, holding none of it, is not + // waited on. + const member = "@novox/mesh-tools" + m, _ = fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","`+member+`":"^0.1.6"}}`), nil) + tgz := tgzOf(t, member, "0.0.0-member") + digest := digestOf(tgz) + store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v2/tools/pkg/blobs/"+digest { + http.NotFound(w, r) + return + } + w.Write(tgz) + })) + defer store.Close() + m.ArtifactStoreFile = filepath.Join(t.TempDir(), "artifact-store.json") + write(t, m.ArtifactStoreFile, `{"at":"`+strings.TrimPrefix(store.URL, "http://")+`"}`, 0o600) + args := map[string]any{"repository": fetchArgs["repository"], "commit": fetchArgs["commit"], + "builds": []any{map[string]any{"package": member, "module": "tools", "artifact": "pkg", "digest": digest}}} + out, err = m.fetch(context.Background(), args) + if err != nil { + t.Fatalf("a fetch with a member's build of %s: %v", member, err) + } + r = out.(FetchResult) + got, err = readPackageJSON(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "node_modules", filepath.FromSlash(member))) + if err != nil || got.Version != "0.0.0-member" { + t.Errorf("%s placed %q (%v), want the member's build", member, got.Version, err) + } } // 1489/AC2: a module naming lodash with a lock gets npm ci --omit=dev --ignore-scripts (fake runner), its install put diff --git a/modules/mesh-build-node/cmd/mesh-build-node/package.go b/modules/mesh-build-node/cmd/mesh-build-node/package.go index f55341aa3..e926909c1 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/package.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/package.go @@ -364,9 +364,9 @@ func placeGroup(dir string, served map[string]*packed) error { return nil } -// sdkLocked is what a module's lock names of the SDK: its version, its integrity and where it was resolved from; ok is -// false when the lock names none (the latest release is then placed, placeSDK). -func sdkLocked(dir string) (version, integrity string, ok bool, err error) { +// locked is what a module's lock names of the package name: its version and its integrity; ok is false when the lock +// names none (the latest release is then placed, placeLatest). +func locked(dir, name string) (version, integrity string, ok bool, err error) { raw, err := os.ReadFile(filepath.Join(dir, "package-lock.json")) if os.IsNotExist(err) { return "", "", false, nil @@ -383,7 +383,7 @@ func sdkLocked(dir string) (version, integrity string, ok bool, err error) { if err := json.Unmarshal(raw, &lock); err != nil { return "", "", false, fmt.Errorf("the package-lock.json is not JSON: %w", err) } - p, ok := lock.Packages["node_modules/"+sdkPackage] + p, ok := lock.Packages["node_modules/"+name] if !ok || p.Version == "" { return "", "", false, nil } @@ -398,51 +398,197 @@ func sdkLocked(dir string) (version, integrity string, ok bool, err error) { // none. The lock's integrity is checked when the lock names the registry's version placed; a registry holding no // release is a wait (`waits for:`). func (m *Module) placeSDK(ctx context.Context, dir, rel string, group *packed) error { - version, integrity, _, err := sdkLocked(dir) + if group == nil { + return m.placeLatest(ctx, dir, rel, sdkPackage) + } + version, _, _, err := locked(dir, sdkPackage) if err != nil { return err } - var tgz []byte - if group != nil { - if m.Say != nil && version != "" && version != group.Version { - m.Say("%s: its lock names %s@%s; the group's %s is placed (design 47 G A-284)", rel, sdkPackage, version, group.Version) - } - tgz = group.tgz - } else { - latest, err := m.registryLatest(ctx, sdkPackage) - if err != nil { - return err - } - if latest == "" { - return fmt.Errorf("%s%s", builds.WaitsFor, sdkPackage) - } - if version != latest { - if m.Say != nil && version != "" { - m.Say("%s: its lock names %s@%s; the latest published %s is placed (ADR 0573 §2)", rel, sdkPackage, version, latest) - } - version, integrity = latest, "" - } - want := sdkPackage + "@" + version - got, registered, held, err := m.registryPackageChecked(ctx, sdkPackage, version) - if err != nil { - return err - } - if !held { - return fmt.Errorf("%s%s", builds.WaitsFor, want) - } - if integrity == "" { - integrity = registered - } - if integrity != "" && integrityOf(got) != integrity { - return fmt.Errorf("the registry's %s is %s, and the lock of %s names %s", want, integrityOf(got), rel, integrity) - } - tgz = got + if m.Say != nil && version != "" && version != group.Version { + m.Say("%s: its lock names %s@%s; the group's %s is placed (design 47 G A-284)", rel, sdkPackage, version, group.Version) } into := filepath.Join(dir, "node_modules", filepath.FromSlash(sdkPackage)) if err := os.RemoveAll(into); err != nil { return err } - return unpackPackage(tgz, into) + return unpackPackage(group.tgz, into) +} + +// placeMeshPackages places in the module's node_modules each mesh package beyond the SDK (@novox/mesh-*) its +// package.json names at the latest release the registry holds, as placeSDK does for the SDK (ADR 0573 §2; design 47 H, +// H-A-267): the install leaves them out (ownDependencies). One the group serves or a member's build lays is left to +// placeGroup and layBuilds, which take precedence; the module itself is not placed. +func (m *Module) placeMeshPackages(ctx context.Context, dir, rel string, served map[string]*packed, members []storedBuild) error { + self, _ := readPackageJSON(dir) + var names []string + for name := range namedPackages(dir) { + if name == sdkPackage || !strings.HasPrefix(name, "@novox/mesh-") || name == self.Name || served[name] != nil || namesPackage(members, name) { + continue + } + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if err := m.placeLatest(ctx, dir, rel, name); err != nil { + return err + } + } + return nil +} + +// placeLatest places in the module's node_modules the latest release of the mesh package name the registry holds, +// whatever the module's lock names (ADR 0573 §2), checked against the lock's integrity when the lock names that +// version; a registry holding no release is a wait (`waits for:`). +func (m *Module) placeLatest(ctx context.Context, dir, rel, name string) error { + version, integrity, _, err := locked(dir, name) + if err != nil { + return err + } + latest, err := m.registryLatest(ctx, name) + if err != nil { + return err + } + if latest == "" { + return fmt.Errorf("%s%s", builds.WaitsFor, name) + } + if version != latest { + if m.Say != nil && version != "" { + m.Say("%s: its lock names %s@%s; the latest published %s is placed (ADR 0573 §2)", rel, name, version, latest) + } + version, integrity = latest, "" + } + if spec := namedRange(dir, name); spec != "" && m.Say != nil { + // Said, never refused: the latest release is placed whatever the range (ADR 0573 §2; design 47 H, H-A-276). + switch in, read := inRange(spec, version); { + case !read: + m.Say("%s: its package.json names %s at %q, a range not read; the latest published %s is placed (ADR 0573 §2)", rel, name, spec, version) + case !in: + m.Say("%s: the latest published %s@%s lies outside the range %q its package.json names; it is placed (ADR 0573 §2)", rel, name, version, spec) + } + } + want := name + "@" + version + got, registered, held, err := m.registryPackageChecked(ctx, name, version) + if err != nil { + return err + } + if !held { + return fmt.Errorf("%s%s", builds.WaitsFor, want) + } + if integrity == "" { + integrity = registered + } + if integrity != "" && integrityOf(got) != integrity { + return fmt.Errorf("the registry's %s is %s, and the lock of %s names %s", want, integrityOf(got), rel, integrity) + } + into := filepath.Join(dir, "node_modules", filepath.FromSlash(name)) + if err := os.RemoveAll(into); err != nil { + return err + } + return unpackPackage(got, into) +} + +// namedRange is the range the module's package.json names of name: its dependencies first, then its peer, optional and +// dev dependencies; "" when it names none. +func namedRange(dir, name string) string { + raw, err := os.ReadFile(filepath.Join(dir, "package.json")) + if err != nil { + return "" + } + var p map[string]any + if json.Unmarshal(raw, &p) != nil { + return "" + } + for _, k := range []string{"dependencies", "peerDependencies", "optionalDependencies", "devDependencies"} { + deps, _ := p[k].(map[string]any) + if spec, ok := deps[name].(string); ok { + return strings.TrimSpace(spec) + } + } + return "" +} + +// inRange says whether the release v (major.minor.patch) lies in the npm range spec; read is false for a range of a +// form it does not read (a union, a hyphen range, a comparator, a tag other than latest), never taken as in or out. It +// reads what a package file of the mesh names: an exact version, an x-range, a caret and a tilde range, as npm does. +func inRange(spec, v string) (in, read bool) { + spec = strings.TrimSpace(spec) + if spec == "" || spec == "*" || spec == "latest" { + return true, true + } + if strings.ContainsAny(spec, " |<>") { + return false, false + } + op := "" + if spec[0] == '^' || spec[0] == '~' { + op, spec = spec[:1], spec[1:] + } + spec = strings.TrimPrefix(strings.TrimPrefix(spec, "="), "v") + if strings.Contains(spec, "-") || strings.Contains(spec, "+") { + return false, false // a pre-release or build range + } + var low [3]int + given := 0 + for i, part := range strings.Split(spec, ".") { + if i > 2 { + return false, false + } + if part == "x" || part == "X" || part == "*" { + break + } + n, err := strconv.Atoi(part) + if err != nil || n < 0 { + return false, false + } + low[i] = n + given = i + 1 + } + if given == 0 { + return true, true // x, x.x + } + var at [3]int + for i, part := range strings.Split(v, ".") { + if i > 2 { + return false, false + } + n, err := strconv.Atoi(part) + if err != nil { + return false, false + } + at[i] = n + } + // bump is the part of low raised by one for the exclusive upper bound, the parts after it zero. + bump := given - 1 + switch op { + case "^": + switch { + case low[0] > 0 || given == 1: + bump = 0 + case low[1] > 0 || given == 2: + bump = 1 + default: + bump = 2 + } + case "~": + bump = 0 + if given > 1 { + bump = 1 + } + } + high := low + high[bump]++ + for i := bump + 1; i < 3; i++ { + high[i] = 0 + } + less := func(a, b [3]int) bool { + for i := range a { + if a[i] != b[i] { + return a[i] < b[i] + } + } + return false + } + return !less(at, low) && less(at, high), true } // placedSDK is the version of the SDK placed in the module's node_modules ("" when none is). diff --git a/modules/mesh-build-node/cmd/mesh-build-node/package_test.go b/modules/mesh-build-node/cmd/mesh-build-node/package_test.go index 4eefea9fe..8c7459d52 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/package_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/package_test.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "os" @@ -33,34 +34,40 @@ func tgzOf(t *testing.T, name, version string) []byte { return buf.Bytes() } -// fakeRegistry serves the packument and the packed files of the versions it holds; asked counts its answers. +// fakeRegistry serves the packument and the packed files of the SDK's versions it holds; asked counts its answers. func fakeRegistry(t *testing.T, m *Module, held map[string][]byte) *atomic.Int32 { + t.Helper() + return fakeRegistryOf(t, m, map[string]map[string][]byte{sdkPackage: held}) +} + +// fakeRegistryOf serves the packument and the packed files of each package's versions it holds (by name); asked counts +// its answers. +func fakeRegistryOf(t *testing.T, m *Module, held map[string]map[string][]byte) *atomic.Int32 { t.Helper() var asked atomic.Int32 var srv *httptest.Server srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { asked.Add(1) - switch { - case r.URL.Path == "/npm/@novox%2fmesh-sdk" || r.URL.Path == "/npm/@novox/mesh-sdk": - if len(held) == 0 { - http.NotFound(w, r) - return - } - versions := map[string]any{} - for v, tgz := range held { - versions[v] = map[string]any{"dist": map[string]any{"tarball": srv.URL + "/files/" + v + ".tgz", "integrity": integrityOf(tgz)}} - } - json.NewEncoder(w).Encode(map[string]any{"versions": versions}) - case strings.HasPrefix(r.URL.Path, "/files/"): - v := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/files/"), ".tgz") - if tgz, ok := held[v]; ok { + if f, ok := strings.CutPrefix(r.URL.Path, "/files/"); ok { + name, v, _ := strings.Cut(strings.TrimSuffix(f, ".tgz"), "@@") + if tgz, ok := held[strings.ReplaceAll(name, "~", "/")][v]; ok { w.Write(tgz) return } http.NotFound(w, r) - default: - http.NotFound(w, r) + return } + name, ok := strings.CutPrefix(r.URL.Path, "/npm/") + name = strings.ReplaceAll(name, "%2f", "/") + if !ok || len(held[name]) == 0 { + http.NotFound(w, r) + return + } + versions := map[string]any{} + for v, tgz := range held[name] { + versions[v] = map[string]any{"dist": map[string]any{"tarball": srv.URL + "/files/" + strings.ReplaceAll(name, "/", "~") + "@@" + v + ".tgz", "integrity": integrityOf(tgz)}} + } + json.NewEncoder(w).Encode(map[string]any{"versions": versions}) })) t.Cleanup(srv.Close) m.Toolchain.Registry = func() (Registry, error) { return Registry{URL: srv.URL + "/npm/"}, nil } @@ -183,3 +190,46 @@ func TestTheGroupPackagesArgumentIsRefusedWhenItIsNotOne(t *testing.T) { t.Errorf("a member's root and a member's directory: %+v %v", gs, err) } } + +// design 47 H, H-A-276 (amending H-A-267): the latest release placed outside the range the package file names is said in +// the journal, and placed all the same; one inside it is not said; a range of a form not read is said as such. +func TestALatestReleaseOutsideItsRangeIsSaid(t *testing.T) { + for _, c := range []struct { + spec, v string + in, read bool + }{ + {"^0.1.0", "0.1.9", true, true}, {"^0.1.0", "0.2.0", false, true}, {"^1.2.3", "1.9.0", true, true}, + {"^1.2.3", "2.0.0", false, true}, {"^0.0.3", "0.0.4", false, true}, {"^0.x", "0.9.0", true, true}, + {"~0.1.2", "0.1.9", true, true}, {"~0.1.2", "0.2.0", false, true}, {"~1", "1.5.0", true, true}, + {"0.1.4", "0.1.4", true, true}, {"0.1.4", "0.1.5", false, true}, {"0.1.x", "0.1.7", true, true}, + {"0.1", "0.2.0", false, true}, {"*", "9.9.9", true, true}, {"^0.1.0", "0.0.9", false, true}, + {">=0.1.0", "0.2.0", false, false}, {"^0.1.0 || ^0.2.0", "0.2.0", false, false}, {"file:../x", "0.1.0", false, false}, + } { + if in, read := inRange(c.spec, c.v); in != c.in || read != c.read { + t.Errorf("inRange(%q, %q) = %v, %v; want %v, %v", c.spec, c.v, in, read, c.in, c.read) + } + } + m, _, _ := newMachine(t) + const name = "@novox/mesh-delivery-contracts-git" + fakeRegistryOf(t, m, map[string]map[string][]byte{name: {"0.1.9": tgzOf(t, name, "0.1.9"), "0.2.0": tgzOf(t, name, "0.2.0")}}) + for _, c := range []struct{ spec, said string }{ + {"^0.1.0", `the latest published ` + name + `@0.2.0 lies outside the range "^0.1.0"`}, + {"^0.2.0", ""}, + {">=0.1.0", `at ">=0.1.0", a range not read`}, + } { + var said []string + m.Say = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) } + dir := t.TempDir() + write(t, filepath.Join(dir, "package.json"), `{"dependencies":{"`+name+`":"`+c.spec+`"}}`, 0o644) + if err := m.placeLatest(context.Background(), dir, "modules/m", name); err != nil { + t.Fatalf("%s: %v", c.spec, err) + } + if p, _ := readPackageJSON(filepath.Join(dir, "node_modules", filepath.FromSlash(name))); p.Version != "0.2.0" { + t.Errorf("%s: placed %q, want the latest release 0.2.0", c.spec, p.Version) + } + joined := strings.Join(said, "\n") + if c.said == "" && joined != "" || c.said != "" && !strings.Contains(joined, c.said) { + t.Errorf("%s: said %q, want %q", c.spec, joined, c.said) + } + } +} diff --git a/modules/nats/cmd/nats-tools/keeper_integration_test.go b/modules/nats/cmd/nats-tools/keeper_integration_test.go index 5b070fd78..8f9ca6c45 100644 --- a/modules/nats/cmd/nats-tools/keeper_integration_test.go +++ b/modules/nats/cmd/nats-tools/keeper_integration_test.go @@ -316,7 +316,9 @@ func TestTheBusModuleRunsWhole(t *testing.T) { _ = Keep(kctx, deaf, func(context.Context, func(string, []byte) error) error { return errors.New("no consumer yet") }, nil, nil, t.Logf) close(done) }() - eventually(t, "the first view answered", func() bool { return !deaf.Status().FirstView.Answered.IsZero() }) + // A first view listing no module is none yet and never marked answered (A-367): the first fill pass is waited + // for instead, then taken as the first (design 47 H, H-A-268). + eventually(t, "the first fill pass", func() bool { return !deaf.Status().Pass.IsZero() }) first := deaf.Status().Pass eventually(t, "a fill pass while the consumer is bound again", func() bool { return deaf.Status().Pass.After(first) }) stopDeaf()