From 4fa457b9bb8ae7bd3901e7eee214069b04f12c5a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 13:59:03 +0100 Subject: [PATCH] CAT-G2: the Go seat keeps what the npm seat keeps; its retention reads every go.mod go_packages and go_retention read the forge's Go registry as npm_* read npm's: a version a go.mod on a default branch requires (or replaces to) is kept, with the newest keep; a go.mod that cannot be read stops a real run before anything is deleted; each deletion says removed on the Go seat, under the module path and without Go's leading v (design 51 *Retention*). Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- modules/gitea/README.md | 2 + modules/gitea/cmd/package-registry/forge.go | 29 +++++++- modules/gitea/cmd/package-registry/main.go | 52 ++++++++++---- modules/gitea/cmd/package-registry/refs.go | 68 +++++++++++++++++-- .../gitea/cmd/package-registry/refs_test.go | 2 +- .../gitea/cmd/package-registry/retention.go | 36 ++++++++-- .../cmd/package-registry/retention_test.go | 57 ++++++++++++++-- 7 files changed, 212 insertions(+), 34 deletions(-) diff --git a/modules/gitea/README.md b/modules/gitea/README.md index 61b52b32..0de14370 100644 --- a/modules/gitea/README.md +++ b/modules/gitea/README.md @@ -201,6 +201,8 @@ deletes the versions nothing keeps. |---|---|---| | `npm_packages` | r | every package of the registry's owner, each version newest first with its publish time, its size and why it is kept | | `npm_retention` | a | the versions retention would delete, and the bytes. **A dry run unless `dry_run` is false**; a real run needs `why` | +| `go_packages` | r | every Go module of the registry's owner (the Go seat's), each version newest first with its publish time, its size and why it is kept: a `go.mod` on a default branch requires it, or it is among the newest `keep` | +| `go_retention` | a | the Go module versions retention would delete, and the bytes; as `npm_retention`, and a `go.mod` that could not be read stops a real run. Each deletion says `removed` on the Go seat | **A version is kept when:** diff --git a/modules/gitea/cmd/package-registry/forge.go b/modules/gitea/cmd/package-registry/forge.go index c90daf9e..223e336c 100644 --- a/modules/gitea/cmd/package-registry/forge.go +++ b/modules/gitea/cmd/package-registry/forge.go @@ -21,6 +21,29 @@ import ( type Forge struct { forgehttp.Client Owner string // the package registry's owner, as the seat serves it + // Type is the registry's package type retention reads: TypeNpm when empty, or TypeGo. + Type string +} + +// The package types the retention reads: npm packages (the npm seat) and Go modules (the Go seat). +const ( + TypeNpm = "npm" + TypeGo = "go" +) + +// typ is the package type this forge's retention reads. +func (f *Forge) typ() string { + if f.Type == "" { + return TypeNpm + } + return f.Type +} + +// As is the same forge, its retention reading the package type typ. +func (f *Forge) As(typ string) *Forge { + out := *f + out.Type = typ + return &out } // ForgeFromEnv is the forge as the manifest hands it to this bundle. @@ -56,12 +79,12 @@ type PackageVersion struct { const pageSize = 50 -// Packages is every npm package version of the owner, all pages. +// Packages is every package version of the owner of this forge's type, all pages. func (f *Forge) Packages(ctx context.Context) ([]PackageVersion, error) { var all []PackageVersion for page := 1; page <= 1000; page++ { var batch []PackageVersion - path := fmt.Sprintf("/api/v1/packages/%s?type=npm&page=%d&limit=%d", url.PathEscape(f.Owner), page, pageSize) + path := fmt.Sprintf("/api/v1/packages/%s?type=%s&page=%d&limit=%d", url.PathEscape(f.Owner), f.typ(), page, pageSize) if err := f.GetJSON(ctx, path, &batch); err != nil { return nil, err } @@ -74,7 +97,7 @@ func (f *Forge) Packages(ctx context.Context) ([]PackageVersion, error) { } func (f *Forge) versionPath(name, version string) string { - return fmt.Sprintf("/api/v1/packages/%s/npm/%s/%s", url.PathEscape(f.Owner), url.PathEscape(name), url.PathEscape(version)) + return f.packagePath(f.typ(), name, version) } // Size is the bytes of one version's files. diff --git a/modules/gitea/cmd/package-registry/main.go b/modules/gitea/cmd/package-registry/main.go index 4ee6b3b9..9f1ca0fc 100644 --- a/modules/gitea/cmd/package-registry/main.go +++ b/modules/gitea/cmd/package-registry/main.go @@ -21,7 +21,7 @@ import ( // ToolNames are the tools this bundle serves. The module's manifest lists no `tools`: its TypeScript // bundle's tools are served without one, and a claim without `serves` would offer a `tools` list to both // of the module's seats as their verbs. The README names these two, and a test holds the two together. -var ToolNames = []string{"npm_packages", "npm_retention"} +var ToolNames = []string{"go_packages", "go_retention", "npm_packages", "npm_retention"} func main() { forge := func() (*Forge, error) { return ForgeFromEnv() } @@ -35,17 +35,45 @@ func main() { // Tools are the bundle's tools over the forge the function gives; asked per call, so a manifest that // does not set the environment is a refusal naming what is missing, not a bundle that will not start. func Tools(forge func() (*Forge, error)) []stdio.Tool { + return append(typeTools(TypeGo, forge), typeTools(TypeNpm, forge)...) +} + +// typeTools are one package type's two tools, _packages and _retention, over the forge reading that type. +func typeTools(typ string, forgeOf func() (*Forge, error)) []stdio.Tool { + forge := func() (*Forge, error) { + f, err := forgeOf() + if err != nil { + return nil, err + } + return f.As(typ), nil + } pkg := map[string]any{"type": "string", "description": "one package, by its name (default every package)"} keep := map[string]any{"type": "integer", "description": fmt.Sprintf("how many of each package's newest versions are kept whatever names them (default %d, at least 1)", DefaultKeep)} + listed := "Every npm package in the forge's package registry, each version newest first with when it was published, " + + "its size and what keeps it: a lockfile on a repository's default branch that names it, the highest version " + + "satisfying a range a package.json there names, a dist-tag, or being among the newest `keep` of its package " + + "(novox/hq ADR 0251). Repositories and files that could not be read are named. Changes nothing. Replaces " + + "npm view and npm dist-tag ls. (r)" + deletes := "What retention would delete from the forge's package registry: every version nothing keeps — no lockfile " + + "on a repository's default branch names it, it is not the highest version satisfying any range a package.json " + + "there names, no dist-tag names it, and it is not among the newest `keep` of its package — with the bytes. " + + "A dry run unless dry_run is false; a real run needs why, and deletes nothing at all when any repository or " + + "file could not be read (novox/hq ADR 0251). Each deletion says removed on the npm seat. Replaces npm unpublish. (a)" + if typ == TypeGo { + listed = "Every Go module in the forge's package registry (the Go seat's), each version newest first with when it " + + "was published, its size and what keeps it: a go.mod on a repository's default branch that requires it, or " + + "being among the newest `keep` of its module (design 51 *Retention*). Repositories and go.mod files that " + + "could not be read are named. Changes nothing. Replaces go list -m -versions. (r)" + deletes = "What retention would delete from the forge's Go registry: every module version no go.mod on a " + + "repository's default branch requires and not among the newest `keep` of its module, with the bytes. A dry " + + "run unless dry_run is false; a real run needs why, and deletes nothing at all when any repository or go.mod " + + "could not be read. Each deletion says removed on the Go seat. (a)" + } return []stdio.Tool{ { - Name: "npm_packages", - Description: "Every npm package in the forge's package registry, each version newest first with when it was published, " + - "its size and what keeps it: a lockfile on a repository's default branch that names it, the highest version " + - "satisfying a range a package.json there names, a dist-tag, or being among the newest `keep` of its package " + - "(novox/hq ADR 0251). Repositories and files that could not be read are named. Changes nothing. Replaces " + - "npm view and npm dist-tag ls. (r)", - Input: map[string]any{"package": pkg, "keep": keep}, + Name: typ + "_packages", + Description: listed, + Input: map[string]any{"package": pkg, "keep": keep}, Run: func(args map[string]any) (any, error) { f, err := forge() if err != nil { @@ -75,12 +103,8 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool { }, }, { - Name: "npm_retention", - Description: "What retention would delete from the forge's package registry: every version nothing keeps — no lockfile " + - "on a repository's default branch names it, it is not the highest version satisfying any range a package.json " + - "there names, no dist-tag names it, and it is not among the newest `keep` of its package — with the bytes. " + - "A dry run unless dry_run is false; a real run needs why, and deletes nothing at all when any repository or " + - "file could not be read (novox/hq ADR 0251). Replaces npm unpublish. (a)", + Name: typ + "_retention", + Description: deletes, Input: map[string]any{ "package": pkg, "keep": keep, diff --git a/modules/gitea/cmd/package-registry/refs.go b/modules/gitea/cmd/package-registry/refs.go index 55fcfa45..744d9120 100644 --- a/modules/gitea/cmd/package-registry/refs.go +++ b/modules/gitea/cmd/package-registry/refs.go @@ -7,6 +7,7 @@ import ( "fmt" "path" "regexp" + "slices" "sort" "strings" ) @@ -36,18 +37,77 @@ var ( "pnpm-lock.yaml": true} ) -// interesting is whether a path is a file this reads; nothing under node_modules, which is what a -// lockfile already says and is not the repository's own. -func interesting(p string) bool { +// goModName is the file a Go module's requirements are read from. +const goModName = "go.mod" + +// interesting is whether a path is a file this reads for packages of the type typ: an npm manifest or lockfile, or +// a go.mod; nothing under node_modules or vendor, which is what a lockfile or go.mod already says and is not the +// repository's own. +func interesting(p, typ string) bool { for _, part := range strings.Split(p, "/") { - if part == "node_modules" { + if part == "node_modules" || part == "vendor" { return false } } base := path.Base(p) + if typ == TypeGo { + return base == goModName + } return manifestNames[base] || lockNames[base] } +// ReadGoMod is every registry module version a go.mod requires, or names on the right of a replace (design 51 +// *Retention*: a Go version is referenced when a go.mod on a repository's default branch requires it). A line it +// cannot read, or a block left open, is an error beside what was read: the file is then not read whole, and a real +// run stops before anything is deleted. +func ReadGoMod(content []byte, where string, known map[string]bool) ([]Pinned, error) { + var out []Pinned + pin := func(module, version string) { + if known[module] { + out = append(out, Pinned{Package: module, Version: version, Where: where}) + } + } + block := "" + for n, line := range strings.Split(string(content), "\n") { + if i := strings.Index(line, "//"); i >= 0 { + line = line[:i] + } + f := strings.Fields(line) + if len(f) == 0 { + continue + } + if block != "" { + if f[0] == ")" { + block = "" + continue + } + f = append([]string{block}, f...) + } else if len(f) == 2 && f[1] == "(" { + block = f[0] + continue + } + switch f[0] { + case "require": + if len(f) != 3 { + return out, fmt.Errorf("line %d: a require is , not %q", n+1, strings.Join(f[1:], " ")) + } + pin(f[1], f[2]) + case "replace": + arrow := slices.Index(f, "=>") + if arrow < 0 { + return out, fmt.Errorf("line %d: a replace without =>", n+1) + } + if right := f[arrow+1:]; len(right) == 2 { + pin(right[0], right[1]) + } + } + } + if block != "" { + return out, fmt.Errorf("its %s block is never closed", block) + } + return out, nil +} + var dependencyFields = []string{"dependencies", "devDependencies", "peerDependencies", "optionalDependencies"} // ReadManifest is every range a package.json names for a registry package. An `npm:` alias names the diff --git a/modules/gitea/cmd/package-registry/refs_test.go b/modules/gitea/cmd/package-registry/refs_test.go index 430e3094..d6d3c897 100644 --- a/modules/gitea/cmd/package-registry/refs_test.go +++ b/modules/gitea/cmd/package-registry/refs_test.go @@ -85,7 +85,7 @@ func TestAPnpmLockNamesWhatItPinsInEachFormat(t *testing.T) { func TestOnlyTheRepositorysOwnManifestsAreRead(t *testing.T) { for p, want := range map[string]bool{"package.json": true, "modules/x/package-lock.json": true, "yarn.lock": true, "pnpm-lock.yaml": true, "node_modules/a/package.json": false, "README.md": false, "a/package.json.bak": false} { - if interesting(p) != want { + if interesting(p, TypeNpm) != want { t.Errorf("%s: %v", p, !want) } } diff --git a/modules/gitea/cmd/package-registry/retention.go b/modules/gitea/cmd/package-registry/retention.go index 77915f22..d0a16205 100644 --- a/modules/gitea/cmd/package-registry/retention.go +++ b/modules/gitea/cmd/package-registry/retention.go @@ -50,6 +50,7 @@ type Package struct { // Survey is the registry and what the forge's repositories say of it. type Survey struct { Owner string `json:"owner"` + Type string `json:"type"` Keep int `json:"keep"` Repositories int `json:"repositories_read"` Unread []string `json:"repositories_unread,omitempty"` @@ -73,7 +74,7 @@ func survey(ctx context.Context, f *Forge, only string, keep int, sizes func(Ite } if only != "" { if _, has := byName[only]; !has { - return nil, fmt.Errorf("the registry of %s holds no npm package %q", f.Owner, only) + return nil, fmt.Errorf("the registry of %s holds no %s package %q", f.Owner, f.typ(), only) } } known := map[string]bool{} @@ -81,7 +82,7 @@ func survey(ctx context.Context, f *Forge, only string, keep int, sizes func(Ite known[name] = true } - s := &Survey{Owner: f.Owner, Keep: keep, Said: []string{}} + s := &Survey{Owner: f.Owner, Type: f.typ(), Keep: keep, Said: []string{}} wanted, pinned, err := readRepositories(ctx, f, known, s) if err != nil { return nil, err @@ -94,7 +95,10 @@ func survey(ctx context.Context, f *Forge, only string, keep int, sizes func(Ite } } sort.Strings(names) - tags := distTags(ctx, f, names) + tags := map[string]tagAnswer{} // a Go module has no dist-tags + if f.typ() == TypeNpm { + tags = distTags(ctx, f, names) + } for _, name := range names { s.Packages = append(s.Packages, decide(name, byName[name], tags[name], wanted[name], pinned[name], keep)) } @@ -169,7 +173,7 @@ func readRepositories(ctx context.Context, f *Forge, known map[string]bool, s *S var ps []Pinned var notRead, unread []string for _, p := range files { - if !interesting(p) { + if !interesting(p, f.typ()) { continue } where := r.FullName + ":" + p @@ -179,6 +183,14 @@ func readRepositories(ctx context.Context, f *Forge, known map[string]bool, s *S continue } base := path.Base(p) + if base == goModName { + got, err := ReadGoMod(content, where, known) + ps = append(ps, got...) + if err != nil { + notRead = append(notRead, fmt.Sprintf("%s: not readable (%v)", where, err)) + } + continue + } if manifestNames[base] { got, err := ReadManifest(content, where, known) if err != nil { @@ -421,16 +433,26 @@ func retention(ctx context.Context, f *Forge, only string, keep int, dryRun bool del, cancel := context.WithTimeout(context.Background(), budget) defer cancel() store := seatStore{forge: func() (*Forge, error) { return f, nil }} + seat := packageseats.NpmSeat + if f.typ() == TypeGo { + seat = packageseats.GoSeat + } for _, c := range p.Candidates { - digest, _, _ := store.Held(packageseats.NpmSeat, c.Package, c.Version) + digest, _, _ := store.Held(seat, c.Package, c.Version) if err := f.Delete(del, c.Package, c.Version); err != nil && !errors.Is(err, errNotFound) { p.Refused = append(p.Refused, fmt.Sprintf("%s@%s: %v", c.Package, c.Version, err)) continue } p.Deleted = append(p.Deleted, c.Package+"@"+c.Version) // The seat says every removal (design 51 *Retention*), so the builder's entry turns removed. - o := packageseats.Outcome{Name: c.Package, Version: c.Version, Kind: packageseats.KindPackage, Digest: digest, Reason: packageseats.ReasonRetention} - if err := say(packageseats.NpmSeat, packageseats.EventRemoved, o); err != nil { + // The seat's version is the builder's, without Go's leading v (forgeVersion adds it for the forge). + o := packageseats.Outcome{Name: c.Package, Version: strings.TrimPrefix(c.Version, "v"), Kind: packageseats.KindPackage, Digest: digest, Reason: packageseats.ReasonRetention} + if seat == packageseats.NpmSeat { + o.Version = c.Version + } else { + o.Module = c.Package + } + if err := say(seat, packageseats.EventRemoved, o); err != nil { s.Said = append(s.Said, fmt.Sprintf("%s@%s is deleted but its removed could not be said: %v", c.Package, c.Version, err)) } } diff --git a/modules/gitea/cmd/package-registry/retention_test.go b/modules/gitea/cmd/package-registry/retention_test.go index 71c81ed8..c869b663 100644 --- a/modules/gitea/cmd/package-registry/retention_test.go +++ b/modules/gitea/cmd/package-registry/retention_test.go @@ -24,6 +24,7 @@ import ( type fakeForge struct { t *testing.T versions map[string][]string // package -> versions + modules map[string][]string // Go module -> versions, listed as type go tags map[string]map[string]string // package -> dist-tags repos map[string]map[string]string // repository -> path -> content broken map[string]bool // repositories whose tree answers 500 @@ -45,7 +46,7 @@ func (f *fakeForge) serve() *Forge { } var ( - versionFiles = regexp.MustCompile(`^/api/v1/packages/acme/npm/([^/]+)/([^/]+)(/files)?$`) + versionFiles = regexp.MustCompile(`^/api/v1/packages/acme/(?:npm|go)/([^/]+)/([^/]+)(/files)?$`) treePath = regexp.MustCompile(`^/api/v1/repos/([^/]+)/([^/]+)/git/trees/`) rawPath = regexp.MustCompile(`^/api/v1/repos/([^/]+)/([^/]+)/raw/(.+)$`) ) @@ -63,10 +64,14 @@ func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) { switch { case p == "/api/v1/packages/acme" && r.Method == http.MethodGet: var all []map[string]any - names := keys(f.versions) + listed := f.versions + if q.Get("type") == TypeGo { + listed = f.modules + } + names := keys(listed) for _, n := range names { - for i, v := range f.versions[n] { - all = append(all, map[string]any{"name": n, "version": v, "type": "npm", + for i, v := range listed[n] { + all = append(all, map[string]any{"name": n, "version": v, "type": q.Get("type"), "created_at": time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC)}) } } @@ -299,6 +304,48 @@ func TestARealRunDeletesOnlyWhatNothingKeeps(t *testing.T) { } } +// TestAVersionAGoModRequiresSurvivesGoRetention: the Go seat keeps what the npm seat keeps (design 51 *Retention*): a +// version a go.mod on a default branch requires, or names on the right of a replace, and the newest keep; the others are +// deleted and said removed on the Go seat, without Go's leading v; a go.mod that cannot be read stops a real run. +func TestAVersionAGoModRequiresSurvivesGoRetention(t *testing.T) { + f := aRegistry(t) + f.modules = map[string][]string{"git.acme/sdk": {"v0.1.0", "v0.1.1", "v0.1.2", "v0.1.3", "v0.1.4"}} + f.repos["acme/svc"] = map[string]string{ + "go.mod": "module git.acme/svc\n\ngo 1.26\n\nrequire (\n\tgit.acme/sdk v0.1.0 // indirect\n\tgolang.org/x/mod v0.20.0\n)\n", + "tools/go.mod": "module git.acme/svc/tools\nrequire git.acme/other v1.0.0\nreplace git.acme/x => git.acme/sdk v0.1.2\n", + "vendor/go.mod": "module broken\nrequire (\n", + } + var removed []string + say = func(seat, event string, o packageseats.Outcome) error { + removed = append(removed, seat+" "+event+" "+o.Module+"@"+o.Version+" "+o.Reason) + return nil + } + t.Cleanup(func() { say = sayOn }) + p, err := retention(context.Background(), f.serve().As(TypeGo), "", 1, false, "tidy the Go registry") + if err != nil { + t.Fatal(err) + } + sort.Strings(f.deleted) + if strings.Join(f.deleted, " ") != "git.acme/sdk@v0.1.1 git.acme/sdk@v0.1.3" { + t.Errorf("deleted %v, want all but the required v0.1.0, the replaced-to v0.1.2 and the newest v0.1.4", f.deleted) + } + if sort.Strings(removed); strings.Join(removed, ",") != "go-package-registry removed git.acme/sdk@0.1.1 retention,go-package-registry removed git.acme/sdk@0.1.3 retention" { + t.Errorf("said %v", removed) + } + if p.Type != TypeGo || len(p.Packages) != 1 { + t.Errorf("surveyed %s %v, want the Go registry alone", p.Type, p.Packages) + } + // A go.mod outside vendor that cannot be read stops the real run before anything is deleted. + f.deleted = nil + f.repos["acme/svc"]["cmd/go.mod"] = "module git.acme/svc/cmd\nrequire (\n\tgit.acme/sdk v0.1.1\n" + if _, err := retention(context.Background(), f.serve().As(TypeGo), "", 1, false, "tidy"); err == nil || !strings.Contains(err.Error(), "not read whole") { + t.Errorf("an unreadable go.mod = %v, want the real run refused", err) + } + if len(f.deleted) != 0 { + t.Errorf("deleted %v while a go.mod was unread", f.deleted) + } +} + func TestAnUnreadRepositoryStopsARealRunBeforeAnythingIsDeleted(t *testing.T) { f := aRegistry(t) f.broken = map[string]bool{"acme/site": true} @@ -359,7 +406,7 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { for _, tl := range tools { served = append(served, tl.Name) suffix := "(r)" - if tl.Name == "npm_retention" { + if strings.HasSuffix(tl.Name, "_retention") { suffix = "(a)" } if !strings.HasSuffix(tl.Description, suffix) {