jochen
a9960d7a1b
Retention read no package-lock.json on the forge: in lockfile v2 and v3 a packages entry's dependencies is a map of name to range, which the reader decoded as nested entries, so all 32 files failed whole and the versions they pin were kept by nothing. Entries are now read one by one, tolerating unknown fields and skipping workspace links; an entry of an unknown shape is said, not the whole file. A dry run measured only the versions it would delete, so every kept version answered 0 bytes. Every version is now measured, the registry's total is answered, and a size that cannot be read is said (hq ADR 0251 §4).
280 lines
9.2 KiB
Go
280 lines
9.2 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"path"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// What the forge's repositories say they depend on (novox/hq ADR 0251 §4): every range a package
|
|
// manifest names, and every version a lockfile pins. Read from file contents alone, so it is tested
|
|
// without a forge.
|
|
|
|
// Wanted is one range a package manifest names, and where.
|
|
type Wanted struct {
|
|
Package string
|
|
Range string
|
|
Where string // repository:path
|
|
}
|
|
|
|
// Pinned is one version a lockfile names, and where.
|
|
type Pinned struct {
|
|
Package string
|
|
Version string
|
|
Where string
|
|
}
|
|
|
|
// Kinds of file this reads, by base name.
|
|
var (
|
|
manifestNames = map[string]bool{"package.json": true}
|
|
lockNames = map[string]bool{"package-lock.json": true, "npm-shrinkwrap.json": true, "yarn.lock": true,
|
|
"pnpm-lock.yaml": true}
|
|
)
|
|
|
|
// interesting is whether a path is a file this reads; nothing under node_modules, which is what a
|
|
// lockfile already says and is not the repository's own.
|
|
func interesting(p string) bool {
|
|
for _, part := range strings.Split(p, "/") {
|
|
if part == "node_modules" {
|
|
return false
|
|
}
|
|
}
|
|
base := path.Base(p)
|
|
return manifestNames[base] || lockNames[base]
|
|
}
|
|
|
|
var dependencyFields = []string{"dependencies", "devDependencies", "peerDependencies", "optionalDependencies"}
|
|
|
|
// ReadManifest is every range a package.json names for a registry package. An `npm:` alias names the
|
|
// package it stands for. A range that is not the registry's (a path, a URL, git, a workspace) is not
|
|
// a range on this registry, and is left out.
|
|
func ReadManifest(content []byte, where string, known map[string]bool) ([]Wanted, error) {
|
|
var m map[string]json.RawMessage
|
|
if err := json.Unmarshal(content, &m); err != nil {
|
|
return nil, err
|
|
}
|
|
var out []Wanted
|
|
for _, field := range dependencyFields {
|
|
var deps map[string]string
|
|
if raw, has := m[field]; !has || json.Unmarshal(raw, &deps) != nil {
|
|
continue
|
|
}
|
|
for name, spec := range deps {
|
|
if alias, ok := strings.CutPrefix(spec, "npm:"); ok {
|
|
at := strings.LastIndex(alias, "@")
|
|
if at <= 0 {
|
|
name, spec = alias, "*"
|
|
} else {
|
|
name, spec = alias[:at], alias[at+1:]
|
|
}
|
|
}
|
|
if !known[name] || notRegistry(spec) {
|
|
continue
|
|
}
|
|
out = append(out, Wanted{Package: name, Range: strings.TrimSpace(spec), Where: where})
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func notRegistry(spec string) bool {
|
|
for _, p := range []string{"file:", "link:", "workspace:", "git+", "git:", "github:", "http:", "https:", "portal:", "patch:"} {
|
|
if strings.HasPrefix(spec, p) {
|
|
return true
|
|
}
|
|
}
|
|
return strings.Contains(spec, "/") && !strings.HasPrefix(spec, "npm:")
|
|
}
|
|
|
|
// ReadLock is every registry package version a lockfile names. An error beside pins says what part of the
|
|
// file was not read; the pins are what was.
|
|
func ReadLock(base string, content []byte, where string, known map[string]bool) ([]Pinned, error) {
|
|
switch base {
|
|
case "package-lock.json", "npm-shrinkwrap.json":
|
|
return readNpmLock(content, where, known)
|
|
case "yarn.lock":
|
|
return readYarnLock(content, where, known), nil
|
|
case "pnpm-lock.yaml":
|
|
return readTextLock(content, where, known), nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
// npmPackage is one entry of a version 2 or 3 lockfile's `packages`, keyed by the path it is installed
|
|
// at. Only what says which version it is is read; every other field (`dependencies` there is a map of
|
|
// name to range, `requires`, `engines`, `bin`, ...) is left alone, so a field this does not know cannot
|
|
// make the file unreadable.
|
|
type npmPackage struct {
|
|
Name string `json:"name"`
|
|
Version string `json:"version"`
|
|
Link bool `json:"link"`
|
|
Resolved string `json:"resolved"`
|
|
}
|
|
|
|
// npmDependency is one entry of a version 1 (and 2's copy) lockfile's `dependencies`: nested by name,
|
|
// with its own `dependencies` of the same shape. `requires` is a map of name to range, and not read.
|
|
type npmDependency struct {
|
|
Version string `json:"version"`
|
|
Dependencies map[string]json.RawMessage `json:"dependencies"`
|
|
}
|
|
|
|
// readNpmLock reads package-lock.json and npm-shrinkwrap.json of every lockfileVersion: 1 (nested
|
|
// `dependencies`), 2 (both) and 3 (`packages` only). It is read entry by entry, so one entry of a shape
|
|
// it does not know is said and skipped, never the whole file: the pins it did read are answered with an
|
|
// error naming what was not.
|
|
func readNpmLock(content []byte, where string, known map[string]bool) ([]Pinned, error) {
|
|
var lock struct {
|
|
Packages map[string]json.RawMessage `json:"packages"`
|
|
Dependencies map[string]json.RawMessage `json:"dependencies"`
|
|
}
|
|
if err := json.Unmarshal(content, &lock); err != nil {
|
|
return nil, err
|
|
}
|
|
var out []Pinned
|
|
var skipped []string
|
|
add := func(name, version string) {
|
|
if known[name] && version != "" {
|
|
out = append(out, Pinned{Package: name, Version: version, Where: where})
|
|
}
|
|
}
|
|
// Version 2 and 3: keyed by the path it is installed at; the name is what follows the last
|
|
// node_modules/. An alias carries the real name. A link is a workspace on disk, not a registry version.
|
|
for key, raw := range lock.Packages {
|
|
_, name, found := cutLast(key, "node_modules/")
|
|
if !found {
|
|
continue // the root, or a workspace's own directory
|
|
}
|
|
var e npmPackage
|
|
if err := json.Unmarshal(raw, &e); err != nil {
|
|
skipped = append(skipped, fmt.Sprintf("packages[%q] (%v)", key, err))
|
|
continue
|
|
}
|
|
if e.Link {
|
|
continue
|
|
}
|
|
if e.Name != "" {
|
|
name = e.Name
|
|
}
|
|
add(name, e.Version)
|
|
}
|
|
// Version 1 (and 2's copy): nested by name. An alias's version is `npm:<name>@<version>`.
|
|
var walk func(trail string, deps map[string]json.RawMessage)
|
|
walk = func(trail string, deps map[string]json.RawMessage) {
|
|
for name, raw := range deps {
|
|
var e npmDependency
|
|
if err := json.Unmarshal(raw, &e); err != nil {
|
|
skipped = append(skipped, fmt.Sprintf("dependencies%s[%q] (%v)", trail, name, err))
|
|
continue
|
|
}
|
|
if alias, ok := strings.CutPrefix(e.Version, "npm:"); ok {
|
|
if at := strings.LastIndex(alias, "@"); at > 0 {
|
|
add(alias[:at], alias[at+1:])
|
|
}
|
|
} else {
|
|
add(name, e.Version)
|
|
}
|
|
walk(fmt.Sprintf("%s[%q].dependencies", trail, name), e.Dependencies)
|
|
}
|
|
}
|
|
walk("", lock.Dependencies)
|
|
if len(skipped) > 0 {
|
|
sort.Strings(skipped)
|
|
return out, fmt.Errorf("%d entries not read, the rest were: %s", len(skipped), strings.Join(skipped, "; "))
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func cutLast(s, sep string) (string, string, bool) {
|
|
i := strings.LastIndex(s, sep)
|
|
if i < 0 {
|
|
return s, "", false
|
|
}
|
|
return s[:i], s[i+len(sep):], true
|
|
}
|
|
|
|
// readYarnLock reads both yarn formats: a header of the specs it resolves (`"@a/b@^1.0.0", "@a/b@^1.1.0":`)
|
|
// and an indented `version "1.2.3"` (classic) or `version: 1.2.3` (berry) under it.
|
|
func readYarnLock(content []byte, where string, known map[string]bool) []Pinned {
|
|
var out []Pinned
|
|
var names []string
|
|
scanner := bufio.NewScanner(bytes.NewReader(content))
|
|
scanner.Buffer(make([]byte, 1<<20), 1<<20)
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
if !strings.HasPrefix(line, " ") && strings.HasSuffix(line, ":") {
|
|
names = names[:0]
|
|
for _, spec := range strings.Split(strings.TrimSuffix(line, ":"), ",") {
|
|
spec = strings.Trim(strings.TrimSpace(spec), `"`)
|
|
// `name@range`, `name@npm:range`; the name may itself start with @.
|
|
at := strings.Index(spec[min(1, len(spec)):], "@")
|
|
if at < 0 {
|
|
continue
|
|
}
|
|
names = append(names, spec[:at+1])
|
|
}
|
|
continue
|
|
}
|
|
trimmed := strings.TrimSpace(line)
|
|
if v, ok := strings.CutPrefix(trimmed, "version "); ok && strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " ") {
|
|
addYarn(&out, names, strings.Trim(v, `"`), where, known)
|
|
} else if v, ok := strings.CutPrefix(trimmed, "version: "); ok && strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " ") {
|
|
addYarn(&out, names, strings.Trim(v, `"`), where, known)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func addYarn(out *[]Pinned, names []string, version, where string, known map[string]bool) {
|
|
seen := map[string]bool{}
|
|
for _, n := range names {
|
|
if known[n] && !seen[n] {
|
|
seen[n] = true
|
|
*out = append(*out, Pinned{Package: n, Version: version, Where: where})
|
|
}
|
|
}
|
|
}
|
|
|
|
var versionText = regexp.MustCompile(`^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?`)
|
|
|
|
// readTextLock reads a lockfile whose format changes between versions (pnpm's) by what every format
|
|
// has in common: each resolved package written as `<name>@<version>` or `<name>/<version>`. Only names
|
|
// the registry holds are looked for, so another package whose name merely contains one is told apart by
|
|
// the character before it.
|
|
func readTextLock(content []byte, where string, known map[string]bool) []Pinned {
|
|
text := string(content)
|
|
var out []Pinned
|
|
seen := map[string]bool{}
|
|
for name := range known {
|
|
for start := 0; ; {
|
|
i := strings.Index(text[start:], name)
|
|
if i < 0 {
|
|
break
|
|
}
|
|
i += start
|
|
start = i + len(name)
|
|
if i > 0 && !strings.ContainsRune(" '\"/\n\t:", rune(text[i-1])) {
|
|
continue
|
|
}
|
|
rest := text[start:]
|
|
if len(rest) == 0 || (rest[0] != '@' && rest[0] != '/') {
|
|
continue
|
|
}
|
|
v := versionText.FindString(rest[1:])
|
|
if v == "" || seen[name+"@"+v] {
|
|
continue
|
|
}
|
|
seen[name+"@"+v] = true
|
|
out = append(out, Pinned{Package: name, Version: v, Where: where})
|
|
}
|
|
}
|
|
return out
|
|
}
|