gitea: read npm lockfiles of every version and measure every version's size

Retention read no package-lock.json on the forge: in lockfile v2 and v3 a packages entry's dependencies is a map of name to range, which the reader decoded as nested entries, so all 32 files failed whole and the versions they pin were kept by nothing. Entries are now read one by one, tolerating unknown fields and skipping workspace links; an entry of an unknown shape is said, not the whole file. A dry run measured only the versions it would delete, so every kept version answered 0 bytes. Every version is now measured, the registry's total is answered, and a size that cannot be read is said (hq ADR 0251 §4).
2026-10-08 13:48:12 +02:00
parent c78b5fc9f5
commit a9960d7a1b
5 changed files with 269 additions and 23 deletions
@@ -65,6 +65,7 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool {
 				}
 				s.Said = append(s.Said, fmt.Sprintf("%d packages, %d versions, %d kept; %d repositories read, %d unread",
 					len(s.Packages), total, kept, s.Repositories, len(s.Unread)))
+				s.Said = append(s.Said, sizesSaid(s)...)
 				return s, nil
 			},
 		},
@@ -4,8 +4,10 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"path"
 	"regexp"
+	"sort"
 	"strings"
 )

@@ -89,7 +91,8 @@ func notRegistry(spec string) bool {
 	return strings.Contains(spec, "/") && !strings.HasPrefix(spec, "npm:")
 }

-// ReadLock is every registry package version a lockfile names.
+// ReadLock is every registry package version a lockfile names. An error beside pins says what part of the
+// file was not read; the pins are what was.
 func ReadLock(base string, content []byte, where string, known map[string]bool) ([]Pinned, error) {
 	switch base {
 	case "package-lock.json", "npm-shrinkwrap.json":
@@ -102,30 +105,56 @@ func ReadLock(base string, content []byte, where string, known map[string]bool)
 	return nil, nil
 }

-type npmLockEntry struct {
-	Name         string                  `json:"name"`
-	Version      string                  `json:"version"`
-	Dependencies map[string]npmLockEntry `json:"dependencies"`
+// npmPackage is one entry of a version 2 or 3 lockfile's `packages`, keyed by the path it is installed
+// at. Only what says which version it is is read; every other field (`dependencies` there is a map of
+// name to range, `requires`, `engines`, `bin`, ...) is left alone, so a field this does not know cannot
+// make the file unreadable.
+type npmPackage struct {
+	Name     string `json:"name"`
+	Version  string `json:"version"`
+	Link     bool   `json:"link"`
+	Resolved string `json:"resolved"`
 }

+// npmDependency is one entry of a version 1 (and 2's copy) lockfile's `dependencies`: nested by name,
+// with its own `dependencies` of the same shape. `requires` is a map of name to range, and not read.
+type npmDependency struct {
+	Version      string                     `json:"version"`
+	Dependencies map[string]json.RawMessage `json:"dependencies"`
+}
+
+// readNpmLock reads package-lock.json and npm-shrinkwrap.json of every lockfileVersion: 1 (nested
+// `dependencies`), 2 (both) and 3 (`packages` only). It is read entry by entry, so one entry of a shape
+// it does not know is said and skipped, never the whole file: the pins it did read are answered with an
+// error naming what was not.
 func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned, error) {
 	var lock struct {
-		Packages     map[string]npmLockEntry `json:"packages"`
-		Dependencies map[string]npmLockEntry `json:"dependencies"`
+		Packages     map[string]json.RawMessage `json:"packages"`
+		Dependencies map[string]json.RawMessage `json:"dependencies"`
 	}
 	if err := json.Unmarshal(content, &lock); err != nil {
 		return nil, err
 	}
 	var out []Pinned
+	var skipped []string
 	add := func(name, version string) {
 		if known[name] && version != "" {
 			out = append(out, Pinned{Package: name, Version: version, Where: where})
 		}
 	}
-	// Version 2 and 3: keyed by the path it is installed at. An alias carries the real name.
-	for key, e := range lock.Packages {
+	// Version 2 and 3: keyed by the path it is installed at; the name is what follows the last
+	// node_modules/. An alias carries the real name. A link is a workspace on disk, not a registry version.
+	for key, raw := range lock.Packages {
 		_, name, found := cutLast(key, "node_modules/")
 		if !found {
+			continue // the root, or a workspace's own directory
+		}
+		var e npmPackage
+		if err := json.Unmarshal(raw, &e); err != nil {
+			skipped = append(skipped, fmt.Sprintf("packages[%q] (%v)", key, err))
+			continue
+		}
+		if e.Link {
 			continue
 		}
 		if e.Name != "" {
@@ -134,9 +163,14 @@ func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned,
 		add(name, e.Version)
 	}
 	// Version 1 (and 2's copy): nested by name. An alias's version is `npm:<name>@<version>`.
-	var walk func(map[string]npmLockEntry)
-	walk = func(deps map[string]npmLockEntry) {
-		for name, e := range deps {
+	var walk func(trail string, deps map[string]json.RawMessage)
+	walk = func(trail string, deps map[string]json.RawMessage) {
+		for name, raw := range deps {
+			var e npmDependency
+			if err := json.Unmarshal(raw, &e); err != nil {
+				skipped = append(skipped, fmt.Sprintf("dependencies%s[%q] (%v)", trail, name, err))
+				continue
+			}
 			if alias, ok := strings.CutPrefix(e.Version, "npm:"); ok {
 				if at := strings.LastIndex(alias, "@"); at > 0 {
 					add(alias[:at], alias[at+1:])
@@ -144,10 +178,14 @@ func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned,
 			} else {
 				add(name, e.Version)
 			}
-			walk(e.Dependencies)
+			walk(fmt.Sprintf("%s[%q].dependencies", trail, name), e.Dependencies)
 		}
 	}
-	walk(lock.Dependencies)
+	walk("", lock.Dependencies)
+	if len(skipped) > 0 {
+		sort.Strings(skipped)
+		return out, fmt.Errorf("%d entries not read, the rest were: %s", len(skipped), strings.Join(skipped, "; "))
+	}
 	return out, nil
 }

@@ -90,3 +90,109 @@ func TestOnlyTheRepositorysOwnManifestsAreRead(t *testing.T) {
 		}
 	}
 }
+
+// Lockfiles as npm writes them: a `packages` entry's `dependencies` is a map of name to range, which a
+// reader expecting nested entries cannot read (the shape every lockfile on the forge had).
+
+const lockV1 = `{
+  "name": "app", "version": "1.0.0", "lockfileVersion": 1, "requires": true,
+  "dependencies": {
+    "@novox/mesh-sdk": {
+      "version": "0.1.2",
+      "resolved": "https://forge.example/api/packages/novox/npm/@novox/mesh-sdk/-/0.1.2/mesh-sdk-0.1.2.tgz",
+      "integrity": "sha512-x",
+      "requires": {"@novox/ui": "^1.0.0", "left-pad": "^1.3.0"},
+      "dependencies": {
+        "@novox/ui": {"version": "1.0.0", "resolved": "https://forge.example/x.tgz", "integrity": "sha512-y",
+          "requires": {"left-pad": "^1.0.0"},
+          "dependencies": {"@novox/mesh-sdk": {"version": "0.1.0", "requires": {}}}}
+      }
+    },
+    "left-pad": {"version": "1.3.0", "dev": true},
+    "sdk": {"version": "npm:@novox/ui@2.0.1"},
+    "local": {"version": "file:../local", "requires": {"@novox/ui": "*"}}
+  }
+}`
+
+const lockV2 = `{
+  "name": "app", "version": "1.0.0", "lockfileVersion": 2, "requires": true,
+  "packages": {
+    "": {"name": "app", "version": "1.0.0", "dependencies": {"@novox/mesh-sdk": "^0.1.0"},
+      "devDependencies": {"@novox/ui": "~1.1.0"}, "engines": {"node": ">=20"}},
+    "node_modules/@novox/mesh-sdk": {"version": "0.1.3", "resolved": "https://forge.example/a.tgz",
+      "integrity": "sha512-a", "dependencies": {"left-pad": "^1.3.0"}, "peerDependencies": {"@novox/ui": "*"},
+      "optionalDependencies": {"fsevents": "^2"}, "bin": {"sdk": "bin/sdk.js"}, "engines": {"node": ">=18"}},
+    "node_modules/@novox/ui": {"version": "1.1.4", "dev": true, "funding": [{"type": "x", "url": "y"}]},
+    "node_modules/left-pad": {"version": "1.3.0"}
+  },
+  "dependencies": {
+    "@novox/mesh-sdk": {"version": "0.1.3", "requires": {"left-pad": "^1.3.0"}},
+    "@novox/ui": {"version": "1.1.4", "dev": true},
+    "left-pad": {"version": "1.3.0"}
+  }
+}`
+
+const lockV3 = `{
+  "name": "app", "version": "1.0.0", "lockfileVersion": 3, "requires": true,
+  "packages": {
+    "": {"name": "app", "workspaces": ["packages/*"], "dependencies": {"@novox/mesh-sdk": "^0.1.0", "sdk": "npm:@novox/ui@^2.0.0"}},
+    "node_modules/@novox/mesh-sdk": {"version": "0.1.5", "license": "MIT", "dependencies": {"@novox/ui": "^1.0.0"}},
+    "node_modules/@novox/ui": {"version": "1.0.9", "devDependencies": {"typescript": "^5"}},
+    "node_modules/a/node_modules/@novox/mesh-sdk": {"version": "0.1.1", "dependencies": {"left-pad": "1.x"}},
+    "node_modules/a/node_modules/b/node_modules/@novox/ui": {"version": "1.0.2"},
+    "node_modules/sdk": {"name": "@novox/ui", "version": "2.0.4", "resolved": "https://forge.example/u.tgz"},
+    "node_modules/@novox/workspace-thing": {"resolved": "packages/thing", "link": true},
+    "node_modules/@novox/linked": {"resolved": "packages/linked", "link": true, "version": "9.9.9"},
+    "packages/thing": {"name": "@novox/workspace-thing", "version": "0.0.1", "dependencies": {"@novox/ui": "^1.0.0"}},
+    "node_modules/left-pad": {"version": "1.3.0", "peerDependencies": {"x": "*"}, "optionalDependencies": {"y": "1"}}
+  }
+}`
+
+var knownLocks = map[string]bool{"@novox/mesh-sdk": true, "@novox/ui": true, "@novox/workspace-thing": true, "@novox/linked": true}
+
+func TestALockfileVersion1NamesEveryNestedVersion(t *testing.T) {
+	got, err := ReadLock("package-lock.json", []byte(lockV1), "r", knownLocks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if pins(got) != "@novox/mesh-sdk@0.1.0 @novox/mesh-sdk@0.1.2 @novox/ui@1.0.0 @novox/ui@2.0.1" {
+		t.Fatalf("v1: %s", pins(got))
+	}
+}
+
+func TestALockfileVersion2ReadsBothHalves(t *testing.T) {
+	got, err := ReadLock("package-lock.json", []byte(lockV2), "r", knownLocks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Each half names both; a version is kept once it is named, so naming it twice is harmless.
+	if pins(got) != "@novox/mesh-sdk@0.1.3 @novox/mesh-sdk@0.1.3 @novox/ui@1.1.4 @novox/ui@1.1.4" {
+		t.Fatalf("v2: %s", pins(got))
+	}
+}
+
+func TestALockfileVersion3ReadsPathsAliasesAndSkipsLinks(t *testing.T) {
+	got, err := ReadLock("package-lock.json", []byte(lockV3), "r", knownLocks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "@novox/mesh-sdk@0.1.1 @novox/mesh-sdk@0.1.5 @novox/ui@1.0.2 @novox/ui@1.0.9 @novox/ui@2.0.4"
+	if pins(got) != want {
+		t.Fatalf("v3: %s\nwant %s", pins(got), want)
+	}
+}
+
+func TestAnEntryOfAnUnknownShapeIsSaidAndTheRestIsRead(t *testing.T) {
+	lock := `{"lockfileVersion":3,"packages":{
+		"node_modules/@novox/ui":{"version":"1.0.0"},
+		"node_modules/@novox/mesh-sdk":{"version":{"odd":true}}},
+		"dependencies":{"@novox/ui":"not an object"}}`
+	got, err := ReadLock("package-lock.json", []byte(lock), "r", knownLocks)
+	if pins(got) != "@novox/ui@1.0.0" {
+		t.Fatalf("read %s", pins(got))
+	}
+	if err == nil || !strings.Contains(err.Error(), "2 entries not read") ||
+		!strings.Contains(err.Error(), `packages["node_modules/@novox/mesh-sdk"]`) {
+		t.Fatalf("got %v", err)
+	}
+}
@@ -187,11 +187,10 @@ func readRepositories(ctx context.Context, f *Forge, known map[string]bool, s *S
 				continue
 			}
 			got, err := ReadLock(base, content, where, known)
+			ps = append(ps, got...)
 			if err != nil {
 				notRead = append(notRead, fmt.Sprintf("%s: not readable (%v)", where, err))
-				continue
 			}
-			ps = append(ps, got...)
 		}
 		mu.Lock()
 		defer mu.Unlock()
@@ -356,6 +355,7 @@ type Candidate struct {
 	Version   string    `json:"version"`
 	Published time.Time `json:"published"`
 	Bytes     int64     `json:"bytes"`
+	SizeError string    `json:"size_error,omitempty"`
 }

 // Plan is what retention would delete, and — for a real run — what it did.
@@ -365,8 +365,10 @@ type Plan struct {
 	Why        string      `json:"why,omitempty"`
 	Candidates []Candidate `json:"would_delete"`
 	Bytes      int64       `json:"bytes"`
-	Deleted    []string    `json:"deleted,omitempty"`
-	Refused    []string    `json:"refused,omitempty"`
+	// RegistryBytes is every version's size together, deleted or kept: what retention is measured against.
+	RegistryBytes int64    `json:"registry_bytes"`
+	Deleted       []string `json:"deleted,omitempty"`
+	Refused       []string `json:"refused,omitempty"`
 }

 // retention works out the plan, and carries it out when dryRun is false.
@@ -374,21 +376,28 @@ func retention(ctx context.Context, f *Forge, only string, keep int, dryRun bool
 	if !dryRun && strings.TrimSpace(why) == "" {
 		return nil, errors.New("a real run deletes versions from the package registry and needs why; nothing was done")
 	}
-	s, err := survey(ctx, f, only, keep, func(i Item) bool { return !i.Kept })
+	// Every version is measured, kept or not, so no size in the answer is a 0 nobody asked for.
+	s, err := survey(ctx, f, only, keep, func(Item) bool { return true })
 	if err != nil {
 		return nil, err
 	}
 	p := &Plan{Survey: s, DryRun: dryRun, Why: strings.TrimSpace(why), Candidates: []Candidate{}}
 	for _, pkg := range s.Packages {
 		for _, v := range pkg.Versions {
+			p.RegistryBytes += v.Bytes
 			if !v.Kept {
-				p.Candidates = append(p.Candidates, Candidate{pkg.Name, v.Version, v.Published, v.Bytes})
+				p.Candidates = append(p.Candidates, Candidate{pkg.Name, v.Version, v.Published, v.Bytes, v.SizeError})
 				p.Bytes += v.Bytes
 			}
 		}
 	}
-	s.Said = append(s.Said, fmt.Sprintf("%d repositories read; %d versions in %d packages; %d would be deleted, %s",
-		s.Repositories, countVersions(s), len(s.Packages), len(p.Candidates), mib(p.Bytes)))
+	s.Said = append(s.Said, fmt.Sprintf("%d repositories read; %d versions in %d packages, %s; %d would be deleted, %s",
+		s.Repositories, countVersions(s), len(s.Packages), mib(p.RegistryBytes), len(p.Candidates), mib(p.Bytes)))
+	s.Said = append(s.Said, sizesSaid(s)...)
+	if len(s.NotRead) > 0 {
+		s.Said = append(s.Said, fmt.Sprintf("%d files were not read whole (files_not_read): a version only they name is "+
+			"kept by nothing here", len(s.NotRead)))
+	}
 	if len(s.Unread) > 0 {
 		s.Said = append(s.Said, fmt.Sprintf("%d repositories or files could not be read: what they depend on is not "+
 			"known, so a real run deletes nothing", len(s.Unread)))
@@ -422,4 +431,22 @@ func countVersions(s *Survey) int {
 	return n
 }

+// sizesSaid says, when any version's size could not be read, how many and the first reason: those versions
+// count 0 bytes, so every total is at least, not exactly.
+func sizesSaid(s *Survey) []string {
+	var failed []string
+	for _, p := range s.Packages {
+		for _, v := range p.Versions {
+			if v.SizeError != "" {
+				failed = append(failed, fmt.Sprintf("%s@%s: %s", p.Name, v.Version, v.SizeError))
+			}
+		}
+	}
+	if len(failed) == 0 {
+		return nil
+	}
+	return []string{fmt.Sprintf("the size of %d versions could not be read, so the bytes said are at least, not exactly "+
+		"(first: %s)", len(failed), failed[0])}
+}
+
 func mib(b int64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20)) }
@@ -24,6 +24,8 @@ type fakeForge struct {
 	tags     map[string]map[string]string // package -> dist-tags
 	repos    map[string]map[string]string // repository -> path -> content
 	broken   map[string]bool              // repositories whose tree answers 500
+	files    map[string][]int64           // package@version -> its files' sizes (default 1 MiB + 1 KiB)
+	noSize   map[string]bool              // package@version whose files answer 500
 	mu       sync.Mutex
 	deleted  []string
 	password string
@@ -74,7 +76,19 @@ func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) {
 		name, _ := url.PathUnescape(m[1])
 		version, _ := url.PathUnescape(m[2])
 		if m[3] != "" {
-			writeJSON(w, []map[string]any{{"name": "a.tgz", "size": 1 << 20}, {"name": "b", "size": 1024}})
+			if f.noSize[name+"@"+version] {
+				http.Error(w, "storage unavailable", http.StatusInternalServerError)
+				return
+			}
+			sizes, has := f.files[name+"@"+version]
+			if !has {
+				sizes = []int64{1 << 20, 1024}
+			}
+			var files []map[string]any
+			for i, n := range sizes {
+				files = append(files, map[string]any{"id": i + 1, "size": n, "name": fmt.Sprintf("f%d", i), "md5": "x"})
+			}
+			writeJSON(w, files)
 			return
 		}
 		if r.Method != http.MethodDelete {
@@ -391,3 +405,63 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) {
 		t.Error("ran without a forge")
 	}
 }
+
+func TestEveryVersionIsMeasuredAndASizeNotReadIsSaid(t *testing.T) {
+	f := aRegistry(t)
+	f.files = map[string][]int64{"@acme/sdk@0.1.1": {1000, 234}, "@acme/sdk@0.3.0": {5000}}
+	f.noSize = map[string]bool{"@acme/sdk@0.2.0": true}
+	p, err := retention(context.Background(), f.serve(), "@acme/sdk", 3, true, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	d := decided(p.Survey)
+	// A kept version is measured too: no 0 that nobody asked for.
+	if got := d["@acme/sdk@0.3.0"]; !got.Kept || got.Bytes != 5000 || got.SizeError != "" {
+		t.Errorf("kept 0.3.0: %+v", got)
+	}
+	if got := d["@acme/sdk@0.1.1"]; got.Bytes != 1234 {
+		t.Errorf("0.1.1: %+v", got)
+	}
+	broken := d["@acme/sdk@0.2.0"]
+	if broken.Bytes != 0 || !strings.Contains(broken.SizeError, "500") {
+		t.Errorf("0.2.0: %+v", broken)
+	}
+	var candidate *Candidate
+	for i := range p.Candidates {
+		if p.Candidates[i].Version == "0.2.0" {
+			candidate = &p.Candidates[i]
+		}
+	}
+	if candidate == nil || candidate.SizeError == "" {
+		t.Errorf("the candidate does not carry its size error: %+v", p.Candidates)
+	}
+	// Would delete 0.1.1 (1234), 0.1.2 and 0.2.1 (1 MiB + 1 KiB each) and 0.2.0 (unknown).
+	if want := int64(1234 + 2*(1<<20+1024)); p.Bytes != want {
+		t.Errorf("bytes %d, want %d", p.Bytes, want)
+	}
+	if p.RegistryBytes <= p.Bytes {
+		t.Errorf("registry bytes %d not above %d", p.RegistryBytes, p.Bytes)
+	}
+	said := strings.Join(p.Said, "\n")
+	if !strings.Contains(said, "the size of 1 versions could not be read") || !strings.Contains(said, "@acme/sdk@0.2.0") {
+		t.Errorf("said %q", said)
+	}
+}
+
+func TestALockfileOfEveryShapeKeepsWhatItPins(t *testing.T) {
+	f := aRegistry(t)
+	f.repos["acme/v2"] = map[string]string{"package-lock.json": `{"lockfileVersion":2,"packages":{
+		"":{"name":"v2","dependencies":{"@acme/sdk":"^0.2.0"}},
+		"node_modules/@acme/sdk":{"version":"0.2.0","dependencies":{"left-pad":"^1.0.0"}}},
+		"dependencies":{"@acme/sdk":{"version":"0.2.0","requires":{"left-pad":"^1.0.0"}}}}`}
+	s, err := survey(context.Background(), f.serve(), "@acme/sdk", 1, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(s.NotRead) != 0 {
+		t.Fatalf("not read %v", s.NotRead)
+	}
+	if got := decided(s)["@acme/sdk@0.2.0"]; !contains(got.Why, "a lockfile names it: acme/v2:package-lock.json") {
+		t.Fatalf("0.2.0: %+v", got)
+	}
+}