diff --git a/modules/gitea/cmd/npm-registry/main.go b/modules/gitea/cmd/npm-registry/main.go index 52d07f9e..e964d0c6 100644 --- a/modules/gitea/cmd/npm-registry/main.go +++ b/modules/gitea/cmd/npm-registry/main.go @@ -65,6 +65,7 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool { } s.Said = append(s.Said, fmt.Sprintf("%d packages, %d versions, %d kept; %d repositories read, %d unread", len(s.Packages), total, kept, s.Repositories, len(s.Unread))) + s.Said = append(s.Said, sizesSaid(s)...) return s, nil }, }, diff --git a/modules/gitea/cmd/npm-registry/refs.go b/modules/gitea/cmd/npm-registry/refs.go index 18c86cda..371d50ea 100644 --- a/modules/gitea/cmd/npm-registry/refs.go +++ b/modules/gitea/cmd/npm-registry/refs.go @@ -4,8 +4,10 @@ import ( "bufio" "bytes" "encoding/json" + "fmt" "path" "regexp" + "sort" "strings" ) @@ -89,7 +91,8 @@ func notRegistry(spec string) bool { return strings.Contains(spec, "/") && !strings.HasPrefix(spec, "npm:") } -// ReadLock is every registry package version a lockfile names. +// ReadLock is every registry package version a lockfile names. An error beside pins says what part of the +// file was not read; the pins are what was. func ReadLock(base string, content []byte, where string, known map[string]bool) ([]Pinned, error) { switch base { case "package-lock.json", "npm-shrinkwrap.json": @@ -102,30 +105,56 @@ func ReadLock(base string, content []byte, where string, known map[string]bool) return nil, nil } -type npmLockEntry struct { - Name string `json:"name"` - Version string `json:"version"` - Dependencies map[string]npmLockEntry `json:"dependencies"` +// npmPackage is one entry of a version 2 or 3 lockfile's `packages`, keyed by the path it is installed +// at. Only what says which version it is is read; every other field (`dependencies` there is a map of +// name to range, `requires`, `engines`, `bin`, ...) is left alone, so a field this does not know cannot +// make the file unreadable. +type npmPackage struct { + Name string `json:"name"` + Version string `json:"version"` + Link bool `json:"link"` + Resolved string `json:"resolved"` } +// npmDependency is one entry of a version 1 (and 2's copy) lockfile's `dependencies`: nested by name, +// with its own `dependencies` of the same shape. `requires` is a map of name to range, and not read. +type npmDependency struct { + Version string `json:"version"` + Dependencies map[string]json.RawMessage `json:"dependencies"` +} + +// readNpmLock reads package-lock.json and npm-shrinkwrap.json of every lockfileVersion: 1 (nested +// `dependencies`), 2 (both) and 3 (`packages` only). It is read entry by entry, so one entry of a shape +// it does not know is said and skipped, never the whole file: the pins it did read are answered with an +// error naming what was not. func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned, error) { var lock struct { - Packages map[string]npmLockEntry `json:"packages"` - Dependencies map[string]npmLockEntry `json:"dependencies"` + Packages map[string]json.RawMessage `json:"packages"` + Dependencies map[string]json.RawMessage `json:"dependencies"` } if err := json.Unmarshal(content, &lock); err != nil { return nil, err } var out []Pinned + var skipped []string add := func(name, version string) { if known[name] && version != "" { out = append(out, Pinned{Package: name, Version: version, Where: where}) } } - // Version 2 and 3: keyed by the path it is installed at. An alias carries the real name. - for key, e := range lock.Packages { + // Version 2 and 3: keyed by the path it is installed at; the name is what follows the last + // node_modules/. An alias carries the real name. A link is a workspace on disk, not a registry version. + for key, raw := range lock.Packages { _, name, found := cutLast(key, "node_modules/") if !found { + continue // the root, or a workspace's own directory + } + var e npmPackage + if err := json.Unmarshal(raw, &e); err != nil { + skipped = append(skipped, fmt.Sprintf("packages[%q] (%v)", key, err)) + continue + } + if e.Link { continue } if e.Name != "" { @@ -134,9 +163,14 @@ func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned, add(name, e.Version) } // Version 1 (and 2's copy): nested by name. An alias's version is `npm:@`. - var walk func(map[string]npmLockEntry) - walk = func(deps map[string]npmLockEntry) { - for name, e := range deps { + var walk func(trail string, deps map[string]json.RawMessage) + walk = func(trail string, deps map[string]json.RawMessage) { + for name, raw := range deps { + var e npmDependency + if err := json.Unmarshal(raw, &e); err != nil { + skipped = append(skipped, fmt.Sprintf("dependencies%s[%q] (%v)", trail, name, err)) + continue + } if alias, ok := strings.CutPrefix(e.Version, "npm:"); ok { if at := strings.LastIndex(alias, "@"); at > 0 { add(alias[:at], alias[at+1:]) @@ -144,10 +178,14 @@ func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned, } else { add(name, e.Version) } - walk(e.Dependencies) + walk(fmt.Sprintf("%s[%q].dependencies", trail, name), e.Dependencies) } } - walk(lock.Dependencies) + walk("", lock.Dependencies) + if len(skipped) > 0 { + sort.Strings(skipped) + return out, fmt.Errorf("%d entries not read, the rest were: %s", len(skipped), strings.Join(skipped, "; ")) + } return out, nil } diff --git a/modules/gitea/cmd/npm-registry/refs_test.go b/modules/gitea/cmd/npm-registry/refs_test.go index ad75526c..6b91f310 100644 --- a/modules/gitea/cmd/npm-registry/refs_test.go +++ b/modules/gitea/cmd/npm-registry/refs_test.go @@ -90,3 +90,109 @@ func TestOnlyTheRepositorysOwnManifestsAreRead(t *testing.T) { } } } + +// Lockfiles as npm writes them: a `packages` entry's `dependencies` is a map of name to range, which a +// reader expecting nested entries cannot read (the shape every lockfile on the forge had). + +const lockV1 = `{ + "name": "app", "version": "1.0.0", "lockfileVersion": 1, "requires": true, + "dependencies": { + "@novox/mesh-sdk": { + "version": "0.1.2", + "resolved": "https://forge.example/api/packages/novox/npm/@novox/mesh-sdk/-/0.1.2/mesh-sdk-0.1.2.tgz", + "integrity": "sha512-x", + "requires": {"@novox/ui": "^1.0.0", "left-pad": "^1.3.0"}, + "dependencies": { + "@novox/ui": {"version": "1.0.0", "resolved": "https://forge.example/x.tgz", "integrity": "sha512-y", + "requires": {"left-pad": "^1.0.0"}, + "dependencies": {"@novox/mesh-sdk": {"version": "0.1.0", "requires": {}}}} + } + }, + "left-pad": {"version": "1.3.0", "dev": true}, + "sdk": {"version": "npm:@novox/ui@2.0.1"}, + "local": {"version": "file:../local", "requires": {"@novox/ui": "*"}} + } +}` + +const lockV2 = `{ + "name": "app", "version": "1.0.0", "lockfileVersion": 2, "requires": true, + "packages": { + "": {"name": "app", "version": "1.0.0", "dependencies": {"@novox/mesh-sdk": "^0.1.0"}, + "devDependencies": {"@novox/ui": "~1.1.0"}, "engines": {"node": ">=20"}}, + "node_modules/@novox/mesh-sdk": {"version": "0.1.3", "resolved": "https://forge.example/a.tgz", + "integrity": "sha512-a", "dependencies": {"left-pad": "^1.3.0"}, "peerDependencies": {"@novox/ui": "*"}, + "optionalDependencies": {"fsevents": "^2"}, "bin": {"sdk": "bin/sdk.js"}, "engines": {"node": ">=18"}}, + "node_modules/@novox/ui": {"version": "1.1.4", "dev": true, "funding": [{"type": "x", "url": "y"}]}, + "node_modules/left-pad": {"version": "1.3.0"} + }, + "dependencies": { + "@novox/mesh-sdk": {"version": "0.1.3", "requires": {"left-pad": "^1.3.0"}}, + "@novox/ui": {"version": "1.1.4", "dev": true}, + "left-pad": {"version": "1.3.0"} + } +}` + +const lockV3 = `{ + "name": "app", "version": "1.0.0", "lockfileVersion": 3, "requires": true, + "packages": { + "": {"name": "app", "workspaces": ["packages/*"], "dependencies": {"@novox/mesh-sdk": "^0.1.0", "sdk": "npm:@novox/ui@^2.0.0"}}, + "node_modules/@novox/mesh-sdk": {"version": "0.1.5", "license": "MIT", "dependencies": {"@novox/ui": "^1.0.0"}}, + "node_modules/@novox/ui": {"version": "1.0.9", "devDependencies": {"typescript": "^5"}}, + "node_modules/a/node_modules/@novox/mesh-sdk": {"version": "0.1.1", "dependencies": {"left-pad": "1.x"}}, + "node_modules/a/node_modules/b/node_modules/@novox/ui": {"version": "1.0.2"}, + "node_modules/sdk": {"name": "@novox/ui", "version": "2.0.4", "resolved": "https://forge.example/u.tgz"}, + "node_modules/@novox/workspace-thing": {"resolved": "packages/thing", "link": true}, + "node_modules/@novox/linked": {"resolved": "packages/linked", "link": true, "version": "9.9.9"}, + "packages/thing": {"name": "@novox/workspace-thing", "version": "0.0.1", "dependencies": {"@novox/ui": "^1.0.0"}}, + "node_modules/left-pad": {"version": "1.3.0", "peerDependencies": {"x": "*"}, "optionalDependencies": {"y": "1"}} + } +}` + +var knownLocks = map[string]bool{"@novox/mesh-sdk": true, "@novox/ui": true, "@novox/workspace-thing": true, "@novox/linked": true} + +func TestALockfileVersion1NamesEveryNestedVersion(t *testing.T) { + got, err := ReadLock("package-lock.json", []byte(lockV1), "r", knownLocks) + if err != nil { + t.Fatal(err) + } + if pins(got) != "@novox/mesh-sdk@0.1.0 @novox/mesh-sdk@0.1.2 @novox/ui@1.0.0 @novox/ui@2.0.1" { + t.Fatalf("v1: %s", pins(got)) + } +} + +func TestALockfileVersion2ReadsBothHalves(t *testing.T) { + got, err := ReadLock("package-lock.json", []byte(lockV2), "r", knownLocks) + if err != nil { + t.Fatal(err) + } + // Each half names both; a version is kept once it is named, so naming it twice is harmless. + if pins(got) != "@novox/mesh-sdk@0.1.3 @novox/mesh-sdk@0.1.3 @novox/ui@1.1.4 @novox/ui@1.1.4" { + t.Fatalf("v2: %s", pins(got)) + } +} + +func TestALockfileVersion3ReadsPathsAliasesAndSkipsLinks(t *testing.T) { + got, err := ReadLock("package-lock.json", []byte(lockV3), "r", knownLocks) + if err != nil { + t.Fatal(err) + } + want := "@novox/mesh-sdk@0.1.1 @novox/mesh-sdk@0.1.5 @novox/ui@1.0.2 @novox/ui@1.0.9 @novox/ui@2.0.4" + if pins(got) != want { + t.Fatalf("v3: %s\nwant %s", pins(got), want) + } +} + +func TestAnEntryOfAnUnknownShapeIsSaidAndTheRestIsRead(t *testing.T) { + lock := `{"lockfileVersion":3,"packages":{ + "node_modules/@novox/ui":{"version":"1.0.0"}, + "node_modules/@novox/mesh-sdk":{"version":{"odd":true}}}, + "dependencies":{"@novox/ui":"not an object"}}` + got, err := ReadLock("package-lock.json", []byte(lock), "r", knownLocks) + if pins(got) != "@novox/ui@1.0.0" { + t.Fatalf("read %s", pins(got)) + } + if err == nil || !strings.Contains(err.Error(), "2 entries not read") || + !strings.Contains(err.Error(), `packages["node_modules/@novox/mesh-sdk"]`) { + t.Fatalf("got %v", err) + } +} diff --git a/modules/gitea/cmd/npm-registry/retention.go b/modules/gitea/cmd/npm-registry/retention.go index 886e2706..8331bdcb 100644 --- a/modules/gitea/cmd/npm-registry/retention.go +++ b/modules/gitea/cmd/npm-registry/retention.go @@ -187,11 +187,10 @@ func readRepositories(ctx context.Context, f *Forge, known map[string]bool, s *S continue } got, err := ReadLock(base, content, where, known) + ps = append(ps, got...) if err != nil { notRead = append(notRead, fmt.Sprintf("%s: not readable (%v)", where, err)) - continue } - ps = append(ps, got...) } mu.Lock() defer mu.Unlock() @@ -356,6 +355,7 @@ type Candidate struct { Version string `json:"version"` Published time.Time `json:"published"` Bytes int64 `json:"bytes"` + SizeError string `json:"size_error,omitempty"` } // Plan is what retention would delete, and — for a real run — what it did. @@ -365,8 +365,10 @@ type Plan struct { Why string `json:"why,omitempty"` Candidates []Candidate `json:"would_delete"` Bytes int64 `json:"bytes"` - Deleted []string `json:"deleted,omitempty"` - Refused []string `json:"refused,omitempty"` + // RegistryBytes is every version's size together, deleted or kept: what retention is measured against. + RegistryBytes int64 `json:"registry_bytes"` + Deleted []string `json:"deleted,omitempty"` + Refused []string `json:"refused,omitempty"` } // retention works out the plan, and carries it out when dryRun is false. @@ -374,21 +376,28 @@ func retention(ctx context.Context, f *Forge, only string, keep int, dryRun bool if !dryRun && strings.TrimSpace(why) == "" { return nil, errors.New("a real run deletes versions from the package registry and needs why; nothing was done") } - s, err := survey(ctx, f, only, keep, func(i Item) bool { return !i.Kept }) + // Every version is measured, kept or not, so no size in the answer is a 0 nobody asked for. + s, err := survey(ctx, f, only, keep, func(Item) bool { return true }) if err != nil { return nil, err } p := &Plan{Survey: s, DryRun: dryRun, Why: strings.TrimSpace(why), Candidates: []Candidate{}} for _, pkg := range s.Packages { for _, v := range pkg.Versions { + p.RegistryBytes += v.Bytes if !v.Kept { - p.Candidates = append(p.Candidates, Candidate{pkg.Name, v.Version, v.Published, v.Bytes}) + p.Candidates = append(p.Candidates, Candidate{pkg.Name, v.Version, v.Published, v.Bytes, v.SizeError}) p.Bytes += v.Bytes } } } - s.Said = append(s.Said, fmt.Sprintf("%d repositories read; %d versions in %d packages; %d would be deleted, %s", - s.Repositories, countVersions(s), len(s.Packages), len(p.Candidates), mib(p.Bytes))) + s.Said = append(s.Said, fmt.Sprintf("%d repositories read; %d versions in %d packages, %s; %d would be deleted, %s", + s.Repositories, countVersions(s), len(s.Packages), mib(p.RegistryBytes), len(p.Candidates), mib(p.Bytes))) + s.Said = append(s.Said, sizesSaid(s)...) + if len(s.NotRead) > 0 { + s.Said = append(s.Said, fmt.Sprintf("%d files were not read whole (files_not_read): a version only they name is "+ + "kept by nothing here", len(s.NotRead))) + } if len(s.Unread) > 0 { s.Said = append(s.Said, fmt.Sprintf("%d repositories or files could not be read: what they depend on is not "+ "known, so a real run deletes nothing", len(s.Unread))) @@ -422,4 +431,22 @@ func countVersions(s *Survey) int { return n } +// sizesSaid says, when any version's size could not be read, how many and the first reason: those versions +// count 0 bytes, so every total is at least, not exactly. +func sizesSaid(s *Survey) []string { + var failed []string + for _, p := range s.Packages { + for _, v := range p.Versions { + if v.SizeError != "" { + failed = append(failed, fmt.Sprintf("%s@%s: %s", p.Name, v.Version, v.SizeError)) + } + } + } + if len(failed) == 0 { + return nil + } + return []string{fmt.Sprintf("the size of %d versions could not be read, so the bytes said are at least, not exactly "+ + "(first: %s)", len(failed), failed[0])} +} + func mib(b int64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20)) } diff --git a/modules/gitea/cmd/npm-registry/retention_test.go b/modules/gitea/cmd/npm-registry/retention_test.go index d81e3c4a..545d5c2d 100644 --- a/modules/gitea/cmd/npm-registry/retention_test.go +++ b/modules/gitea/cmd/npm-registry/retention_test.go @@ -24,6 +24,8 @@ type fakeForge struct { tags map[string]map[string]string // package -> dist-tags repos map[string]map[string]string // repository -> path -> content broken map[string]bool // repositories whose tree answers 500 + files map[string][]int64 // package@version -> its files' sizes (default 1 MiB + 1 KiB) + noSize map[string]bool // package@version whose files answer 500 mu sync.Mutex deleted []string password string @@ -74,7 +76,19 @@ func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) { name, _ := url.PathUnescape(m[1]) version, _ := url.PathUnescape(m[2]) if m[3] != "" { - writeJSON(w, []map[string]any{{"name": "a.tgz", "size": 1 << 20}, {"name": "b", "size": 1024}}) + if f.noSize[name+"@"+version] { + http.Error(w, "storage unavailable", http.StatusInternalServerError) + return + } + sizes, has := f.files[name+"@"+version] + if !has { + sizes = []int64{1 << 20, 1024} + } + var files []map[string]any + for i, n := range sizes { + files = append(files, map[string]any{"id": i + 1, "size": n, "name": fmt.Sprintf("f%d", i), "md5": "x"}) + } + writeJSON(w, files) return } if r.Method != http.MethodDelete { @@ -391,3 +405,63 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { t.Error("ran without a forge") } } + +func TestEveryVersionIsMeasuredAndASizeNotReadIsSaid(t *testing.T) { + f := aRegistry(t) + f.files = map[string][]int64{"@acme/sdk@0.1.1": {1000, 234}, "@acme/sdk@0.3.0": {5000}} + f.noSize = map[string]bool{"@acme/sdk@0.2.0": true} + p, err := retention(context.Background(), f.serve(), "@acme/sdk", 3, true, "") + if err != nil { + t.Fatal(err) + } + d := decided(p.Survey) + // A kept version is measured too: no 0 that nobody asked for. + if got := d["@acme/sdk@0.3.0"]; !got.Kept || got.Bytes != 5000 || got.SizeError != "" { + t.Errorf("kept 0.3.0: %+v", got) + } + if got := d["@acme/sdk@0.1.1"]; got.Bytes != 1234 { + t.Errorf("0.1.1: %+v", got) + } + broken := d["@acme/sdk@0.2.0"] + if broken.Bytes != 0 || !strings.Contains(broken.SizeError, "500") { + t.Errorf("0.2.0: %+v", broken) + } + var candidate *Candidate + for i := range p.Candidates { + if p.Candidates[i].Version == "0.2.0" { + candidate = &p.Candidates[i] + } + } + if candidate == nil || candidate.SizeError == "" { + t.Errorf("the candidate does not carry its size error: %+v", p.Candidates) + } + // Would delete 0.1.1 (1234), 0.1.2 and 0.2.1 (1 MiB + 1 KiB each) and 0.2.0 (unknown). + if want := int64(1234 + 2*(1<<20+1024)); p.Bytes != want { + t.Errorf("bytes %d, want %d", p.Bytes, want) + } + if p.RegistryBytes <= p.Bytes { + t.Errorf("registry bytes %d not above %d", p.RegistryBytes, p.Bytes) + } + said := strings.Join(p.Said, "\n") + if !strings.Contains(said, "the size of 1 versions could not be read") || !strings.Contains(said, "@acme/sdk@0.2.0") { + t.Errorf("said %q", said) + } +} + +func TestALockfileOfEveryShapeKeepsWhatItPins(t *testing.T) { + f := aRegistry(t) + f.repos["acme/v2"] = map[string]string{"package-lock.json": `{"lockfileVersion":2,"packages":{ + "":{"name":"v2","dependencies":{"@acme/sdk":"^0.2.0"}}, + "node_modules/@acme/sdk":{"version":"0.2.0","dependencies":{"left-pad":"^1.0.0"}}}, + "dependencies":{"@acme/sdk":{"version":"0.2.0","requires":{"left-pad":"^1.0.0"}}}}`} + s, err := survey(context.Background(), f.serve(), "@acme/sdk", 1, nil) + if err != nil { + t.Fatal(err) + } + if len(s.NotRead) != 0 { + t.Fatalf("not read %v", s.NotRead) + } + if got := decided(s)["@acme/sdk@0.2.0"]; !contains(got.Why, "a lockfile names it: acme/v2:package-lock.json") { + t.Fatalf("0.2.0: %+v", got) + } +}