From f16efbef4f9bb218d16aca43aa0bc9128843d4ae Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 13:43:59 +0100 Subject: [PATCH] CAT-G1, G2: the forge's Go bundle serves both package seats' verbs into its package registry, for both packages Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- modules/gitea/README.md | 14 +- modules/gitea/cmd/agent-token/main.go | 2 +- modules/gitea/cmd/forge/forge.go | 2 +- modules/gitea/cmd/forge/main.go | 4 +- .../forge.go | 0 .../main.go | 9 +- .../refs.go | 0 .../refs_test.go | 0 .../retention.go | 0 .../retention_test.go | 42 +- modules/gitea/cmd/package-registry/seats.go | 366 ++++++++++++++++++ .../gitea/cmd/package-registry/seats_test.go | 157 ++++++++ .../semver.go | 0 .../semver_test.go | 0 modules/gitea/go.mod | 2 + modules/gitea/internal/forgehttp/forgehttp.go | 25 +- modules/gitea/internal/packageseats/holder.go | 6 +- modules/gitea/module.json | 58 ++- 18 files changed, 658 insertions(+), 29 deletions(-) rename modules/gitea/cmd/{npm-registry => package-registry}/forge.go (100%) rename modules/gitea/cmd/{npm-registry => package-registry}/main.go (91%) rename modules/gitea/cmd/{npm-registry => package-registry}/refs.go (100%) rename modules/gitea/cmd/{npm-registry => package-registry}/refs_test.go (100%) rename modules/gitea/cmd/{npm-registry => package-registry}/retention.go (100%) rename modules/gitea/cmd/{npm-registry => package-registry}/retention_test.go (91%) create mode 100644 modules/gitea/cmd/package-registry/seats.go create mode 100644 modules/gitea/cmd/package-registry/seats_test.go rename modules/gitea/cmd/{npm-registry => package-registry}/semver.go (100%) rename modules/gitea/cmd/{npm-registry => package-registry}/semver_test.go (100%) diff --git a/modules/gitea/README.md b/modules/gitea/README.md index c6fe87c5..61b52b32 100644 --- a/modules/gitea/README.md +++ b/modules/gitea/README.md @@ -91,7 +91,7 @@ is read once and searched in memory; a binary file is passed over, a file over 4 `skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. A search that reaches the call's deadline stops there and answers what it found, with `stopped_at_deadline`. -It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), as the +It reaches the forge through the same HTTP client as the package-registry bundle (`internal/forgehttp`), as the tools' own account (novox/hq ADR 0365 §2, issue 469): the token the TypeScript bundle mints for `mesh-tools` and keeps (`MESH_GITEA_TOKEN_FILE`), sent only when its scopes line names that account, never the admin login. One tool call takes at most 50 seconds. @@ -181,10 +181,20 @@ No tool creates in the organisation: `gitea_create_repo` creates under the tools keeps `can_create_org_repo` false and its token has no `write:organization` (ADR 0428 §8). `go test ./cmd/repositories` runs it against a fake forge. +## The two package seats (novox/hq design 51, ADR 0605, ADR 0611) + +The forge module holds `npm-package-registry` and `go-package-registry`, and the Go bundle `cmd/package-registry` +serves both seats' verbs as tools `.`: `publish` and `publish-part` (a package's one file in chunks into +the module's own object store `incoming`, its transfers in the state `transfers`; mesh-sdk's seat-neutral receiving +side), `versions` and `remove` (a tagged version, by the builder). Before anything is uploaded the holder asks the +builder whether it numbered the version with that digest; a package goes up as npm's publish sends it, a Go module's +zip to the Go upload. Every outcome is said on the seat's events (`published`, `refused`, `removed`), a Go package's +with its module path. Who may call each verb is the trusted config value `callers` (empty: design 51's list). + ## The package registry's retention (novox/hq ADR 0251 §4, to-be 51) Every publish adds a version to the registry, and until this nothing removed one. The Go bundle -`cmd/npm-registry` says which versions each package holds, what keeps each one, and — asked with a why — +`cmd/package-registry` says which versions each package holds, what keeps each one, and — asked with a why — deletes the versions nothing keeps. | tool | | what | diff --git a/modules/gitea/cmd/agent-token/main.go b/modules/gitea/cmd/agent-token/main.go index 0303b670..c755d5a2 100644 --- a/modules/gitea/cmd/agent-token/main.go +++ b/modules/gitea/cmd/agent-token/main.go @@ -17,7 +17,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" ) -// ToolNames are the tools this bundle serves. The module's manifest lists no `tools` (see the npm-registry bundle +// ToolNames are the tools this bundle serves. The module's manifest lists no `tools` (see the package-registry bundle // for why); the README names this one, and a test holds the three together. var ToolNames = []string{ToolName} diff --git a/modules/gitea/cmd/forge/forge.go b/modules/gitea/cmd/forge/forge.go index a28a915b..a9d27930 100644 --- a/modules/gitea/cmd/forge/forge.go +++ b/modules/gitea/cmd/forge/forge.go @@ -12,7 +12,7 @@ import ( ) // The forge as this bundle reaches it: its own HTTP interface on the machine, as the tools' own account (novox/hq -// ADR 0365 §2; internal/forgehttp, shared with the npm-registry bundle beside it). +// ADR 0365 §2; internal/forgehttp, shared with the package-registry bundle beside it). // Forge is the forge's HTTP interface. type Forge struct { diff --git a/modules/gitea/cmd/forge/main.go b/modules/gitea/cmd/forge/main.go index 3be08aa8..89900a64 100644 --- a/modules/gitea/cmd/forge/main.go +++ b/modules/gitea/cmd/forge/main.go @@ -1,6 +1,6 @@ // forge: the forge module's Go bundle for an agent's commit and the reads around it (novox/hq issue 375). // A process the node's runtime launches and speaks MCP over stdio to, beside the module's TypeScript bundle -// and its npm-registry bundle. gitea_commit puts one commit on a branch of its own — never the default branch, +// and its package-registry bundle. gitea_commit puts one commit on a branch of its own — never the default branch, // never one a protection rule names (novox/hq ADR 0266) — and the reads answer a commit history, a compare, a // pull request's reviews with their line comments, a directory's entries and a code search, summarised. // stdout is the protocol; it says what it says on stderr, and never the forge's credential. @@ -16,7 +16,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" ) -// ToolNames are the tools this bundle serves. The module's manifest lists no `tools` (see the npm-registry +// ToolNames are the tools this bundle serves. The module's manifest lists no `tools` (see the package-registry // bundle for why); the README names these, and a test holds the three together. var ToolNames = []string{"gitea_commit", "gitea_commits", "gitea_compare", "gitea_pull_reviews", "gitea_list_dir", "gitea_search_code"} diff --git a/modules/gitea/cmd/npm-registry/forge.go b/modules/gitea/cmd/package-registry/forge.go similarity index 100% rename from modules/gitea/cmd/npm-registry/forge.go rename to modules/gitea/cmd/package-registry/forge.go diff --git a/modules/gitea/cmd/npm-registry/main.go b/modules/gitea/cmd/package-registry/main.go similarity index 91% rename from modules/gitea/cmd/npm-registry/main.go rename to modules/gitea/cmd/package-registry/main.go index e964d0c6..4ee6b3b9 100644 --- a/modules/gitea/cmd/npm-registry/main.go +++ b/modules/gitea/cmd/package-registry/main.go @@ -1,4 +1,5 @@ -// npm-registry: the forge module's Go bundle for its package registry (novox/hq ADR 0251 §4, to-be 51). +// package-registry: the forge module's Go bundle for its package registry (novox/hq ADR 0251 §4, design 51): its +// retention tools, and the verbs of the two package seats it holds (seats.go). // A process the node's runtime launches and speaks MCP over stdio to, beside the module's TypeScript // bundle. It says which versions of each npm package the registry holds, what keeps each one, and — // asked with a why — deletes the versions nothing keeps. stdout is the protocol; it says what it says on @@ -13,6 +14,8 @@ import ( "strings" stdio "git.novox.be/novox/mesh-sdk/go" + + "gitea/internal/packageseats" ) // ToolNames are the tools this bundle serves. The module's manifest lists no `tools`: its TypeScript @@ -21,7 +24,9 @@ import ( var ToolNames = []string{"npm_packages", "npm_retention"} func main() { - if err := stdio.Serve("", Tools(func() (*Forge, error) { return ForgeFromEnv() })); err != nil { + forge := func() (*Forge, error) { return ForgeFromEnv() } + tools := append(Tools(forge), SeatTools(func() (*packageseats.Holder, error) { return busHolder(forge) })...) + if err := stdio.Serve("", tools); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } diff --git a/modules/gitea/cmd/npm-registry/refs.go b/modules/gitea/cmd/package-registry/refs.go similarity index 100% rename from modules/gitea/cmd/npm-registry/refs.go rename to modules/gitea/cmd/package-registry/refs.go diff --git a/modules/gitea/cmd/npm-registry/refs_test.go b/modules/gitea/cmd/package-registry/refs_test.go similarity index 100% rename from modules/gitea/cmd/npm-registry/refs_test.go rename to modules/gitea/cmd/package-registry/refs_test.go diff --git a/modules/gitea/cmd/npm-registry/retention.go b/modules/gitea/cmd/package-registry/retention.go similarity index 100% rename from modules/gitea/cmd/npm-registry/retention.go rename to modules/gitea/cmd/package-registry/retention.go diff --git a/modules/gitea/cmd/npm-registry/retention_test.go b/modules/gitea/cmd/package-registry/retention_test.go similarity index 91% rename from modules/gitea/cmd/npm-registry/retention_test.go rename to modules/gitea/cmd/package-registry/retention_test.go index 03917f59..a4da4f36 100644 --- a/modules/gitea/cmd/npm-registry/retention_test.go +++ b/modules/gitea/cmd/package-registry/retention_test.go @@ -17,6 +17,7 @@ import ( "time" "gitea/internal/forgehttp" + "gitea/internal/packageseats" ) // fakeForge answers the forge's interface from maps, and records every delete. @@ -373,8 +374,14 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { t.Fatal(err) } var m struct { - Tools []string `json:"tools"` - Build struct { + Tools []string `json:"tools"` + Claims []struct { + Name string `json:"name"` + Serves []string `json:"serves"` + } `json:"claims"` + Requires []any `json:"requires"` + Invokes []string `json:"invokes"` + Build struct { Artifacts []struct { From string `json:"from"` Loads []string `json:"loads"` @@ -385,12 +392,39 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } + // Both package seats are claimed serving the four verbs this bundle answers as ., over its own incoming + // and transfers, asking the builder's numbering (design 51 *Publishing*). + claimed := map[string]string{} + for _, c := range m.Claims { + claimed[c.Name] = strings.Join(c.Serves, ",") + } + for _, seat := range []string{packageseats.NpmSeat, packageseats.GoSeat} { + if claimed[seat] != strings.Join(packageseats.Verbs, ",") { + t.Errorf("%s is claimed serving %q, want %v", seat, claimed[seat], packageseats.Verbs) + } + } + names := map[string]bool{} + for _, tool := range SeatTools(nil) { + names[tool.Name] = true + } + if len(names) != 8 || !names[packageseats.GoSeat+".publish-part"] { + t.Errorf("the seat tools are %v", names) + } + req, _ := json.Marshal(m.Requires) + for _, want := range []string{`"name":"incoming"`, `"name":"transfers"`, `"name":"findings"`} { + if !strings.Contains(string(req), want) { + t.Errorf("the manifest does not require %s", want) + } + } + if strings.Join(m.Invokes, ",") != "mesh-delivery-builder.builder_versions" { + t.Errorf("the manifest invokes %v", m.Invokes) + } if len(m.Tools) != 0 { t.Error("the manifest lists tools: its claims would offer them as both seats' verbs") } found := false for _, a := range m.Build.Artifacts { - if a.From == "cmd/npm-registry" { + if a.From == "cmd/package-registry" { found = true for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE", "MESH_NPM_OWNER"} { if a.Env[k] == "" { @@ -400,7 +434,7 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { } } if !found { - t.Error("the manifest builds no cmd/npm-registry bundle") + t.Error("the manifest builds no cmd/package-registry bundle") } // Refused cleanly when the environment is missing, not a crash. if _, err := tools[0].Run(map[string]any{}); err == nil { diff --git a/modules/gitea/cmd/package-registry/seats.go b/modules/gitea/cmd/package-registry/seats.go new file mode 100644 index 00000000..6de3b443 --- /dev/null +++ b/modules/gitea/cmd/package-registry/seats.go @@ -0,0 +1,366 @@ +package main + +// The two package seats' verbs (novox/hq design 51 *The four registry seats*, *One contract*): publish, publish-part, +// versions and remove on `npm-package-registry` and `go-package-registry`, served as tools named . and +// answered in each seat's own contract types (mesh-contracts npmpackageregistry, gopackageregistry; ADR 0620). The +// holding itself is internal/packageseats; this file binds it to the forge's package registry, the runtime's bus state +// and the seats' events. + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "crypto/sha1" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "strings" + "time" + + goreg "git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry" + npmreg "git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/parts" + + "gitea/internal/packageseats" +) + +// seatStore is the forge's package registry as both seats store into it. +type seatStore struct{ forge func() (*Forge, error) } + +func forgeType(seat string) string { return packageseats.Seats[seat] } + +// forgeVersion is the version as the forge keeps it: a Go module's carries Go's leading v. +func forgeVersion(seat, version string) string { + if seat == packageseats.GoSeat && !strings.HasPrefix(version, "v") { + return "v" + version + } + return version +} + +func (f *Forge) packagePath(typ, name, version string) string { + return fmt.Sprintf("/api/v1/packages/%s/%s/%s/%s", url.PathEscape(f.Owner), typ, url.PathEscape(name), url.PathEscape(version)) +} + +func (s seatStore) Held(seat, name, version string) (string, bool, error) { + f, err := s.forge() + if err != nil { + return "", false, err + } + var files []struct { + SHA256 string `json:"sha256"` + } + err = f.GetJSON(context.Background(), f.packagePath(forgeType(seat), name, forgeVersion(seat, version))+"/files", &files) + switch { + case errors.Is(err, errNotFound): + return "", false, nil + case err != nil: + return "", false, err + case len(files) == 0: + return "", false, fmt.Errorf("%s@%s is listed with no file", name, version) + } + return "sha256:" + files[0].SHA256, true, nil +} + +func (s seatStore) Put(seat string, p packageseats.Publish, file []byte) (string, error) { + f, err := s.forge() + if err != nil { + return "", err + } + ctx := context.Background() + if seat == packageseats.GoSeat { + module, err := goModuleOf(file, forgeVersion(seat, p.Version)) + if err != nil { + return "", err + } + _, err = f.Send(ctx, http.MethodPut, fmt.Sprintf("/api/packages/%s/go/upload", url.PathEscape(f.Owner)), file, "application/zip", maxBody) + return module, err + } + doc, err := npmDocument(f.URL, f.Owner, p, file) + if err != nil { + return "", err + } + _, err = f.Do(ctx, http.MethodPut, fmt.Sprintf("/api/packages/%s/npm/%s", url.PathEscape(f.Owner), url.PathEscape(p.Name)), doc, maxBody) + return "", err +} + +func (s seatStore) Versions(seat, name string) ([]packageseats.Version, error) { + f, err := s.forge() + if err != nil { + return nil, err + } + var out []packageseats.Version + for page := 1; page <= 1000; page++ { + var batch []PackageVersion + p := fmt.Sprintf("/api/v1/packages/%s?type=%s&q=%s&page=%d&limit=%d", url.PathEscape(f.Owner), forgeType(seat), url.QueryEscape(name), page, pageSize) + if err := f.GetJSON(context.Background(), p, &batch); err != nil { + return nil, err + } + for _, v := range batch { + if v.Name != name { + continue + } + d, _, err := s.Held(seat, name, v.Version) + if err != nil { + return nil, err + } + out = append(out, packageseats.Version{Version: strings.TrimPrefix(v.Version, "v"), Digest: d, Published: v.CreatedAt}) + } + if len(batch) < pageSize { + break + } + } + // Newest first, by when each was published. + for i := 1; i < len(out); i++ { + for j := i; j > 0 && out[j].Published.After(out[j-1].Published); j-- { + out[j], out[j-1] = out[j-1], out[j] + } + } + return out, nil +} + +func (s seatStore) Remove(seat, name, version string) error { + f, err := s.forge() + if err != nil { + return err + } + _, err = f.do(context.Background(), http.MethodDelete, f.packagePath(forgeType(seat), name, forgeVersion(seat, version))) + return err +} + +// goModuleOf is the Go module path of a module zip whose every file sits under @/, as Go's module +// proxy protocol has it; a zip of another version is refused before the forge sees it. +func goModuleOf(file []byte, version string) (string, error) { + z, err := zip.NewReader(bytes.NewReader(file), int64(len(file))) + if err != nil { + return "", fmt.Errorf("the Go package is not a module zip: %w", err) + } + if len(z.File) == 0 { + return "", errors.New("the Go package's zip holds no file") + } + module, rest, at := strings.Cut(z.File[0].Name, "@") + v, _, ok := strings.Cut(rest, "/") + if !ok || !at || module == "" { + return "", fmt.Errorf("the Go package's zip does not hold @/: %s", z.File[0].Name) + } + if v != version { + return "", fmt.Errorf("the Go package's zip is %s, not %s", v, version) + } + return module, nil +} + +// npmDocument is the body npm's publish sends: the package.json of the tarball's package/, its dist, the tarball as +// its one attachment, and the dist-tag: the version's tag for a tagged version, so that it never becomes latest. +func npmDocument(base, owner string, p packageseats.Publish, tgz []byte) (map[string]any, error) { + manifest, err := npmManifest(tgz) + if err != nil { + return nil, err + } + if manifest["name"] != p.Name || manifest["version"] != p.Version { + return nil, fmt.Errorf("the tarball is %v@%v, not %s@%s", manifest["name"], manifest["version"], p.Name, p.Version) + } + file := path.Base(p.Name) + "-" + p.Version + ".tgz" + s1, s512 := sha1.Sum(tgz), sha512.Sum512(tgz) + manifest["_id"] = p.Name + "@" + p.Version + manifest["dist"] = map[string]any{"shasum": hex.EncodeToString(s1[:]), "integrity": "sha512-" + base64.StdEncoding.EncodeToString(s512[:]), + "tarball": fmt.Sprintf("%s/api/packages/%s/npm/%s/-/%s/%s", base, owner, p.Name, p.Version, file)} + tag := "latest" + if p.Tag != "" { + tag = p.Tag + } + return map[string]any{"_id": p.Name, "name": p.Name, "dist-tags": map[string]string{tag: p.Version}, + "versions": map[string]any{p.Version: manifest}, + "_attachments": map[string]any{file: map[string]any{"content_type": "application/octet-stream", "data": base64.StdEncoding.EncodeToString(tgz), "length": len(tgz)}}}, nil +} + +func npmManifest(tgz []byte) (map[string]any, error) { + gz, err := gzip.NewReader(bytes.NewReader(tgz)) + if err != nil { + return nil, fmt.Errorf("the npm package is not a gzipped tarball: %w", err) + } + tr := tar.NewReader(gz) + for { + h, err := tr.Next() + if err == io.EOF { + return nil, errors.New("the npm package holds no package/package.json") + } + if err != nil { + return nil, fmt.Errorf("the npm package's tarball cannot be read: %w", err) + } + if h.Name != "package/package.json" { + continue + } + var m map[string]any + if err := json.NewDecoder(io.LimitReader(tr, 1<<20)).Decode(&m); err != nil { + return nil, fmt.Errorf("the npm package's package.json is not JSON: %w", err) + } + return m, nil + } +} + +// as is v in T's shape: what crosses the bus is the seat's contract type, read or written through its JSON. +func as[T any](v any) (T, error) { + var out T + raw, err := json.Marshal(v) + if err == nil { + err = json.Unmarshal(raw, &out) + } + return out, err +} + +// outcomeBody is an outcome as the seat's contract's Outcome names it. +func outcomeBody(o packageseats.Outcome) map[string]any { + return map[string]any{"name": o.Name, "version": o.Version, "kind": o.Kind, "tag": o.Tag, "digest": o.Digest, + "fingerprint": o.Fingerprint, "commit": o.Commit, "holder": packageseats.Self, "transfer": o.Transfer, + "reference": o.Reference, "reason": o.Reason, "module": o.Module} +} + +// sayOn says an event on the seat's family, in the seat's contract's Outcome. +func sayOn(seat, event string, o packageseats.Outcome) error { + var body any + var err error + if seat == packageseats.GoSeat { + body, err = as[goreg.Outcome](outcomeBody(o)) + } else { + body, err = as[npmreg.Outcome](outcomeBody(o)) + } + if err != nil { + return err + } + return stdio.SeatPublish("mesh.seat."+seat+".event."+event, body, "") +} + +func finding(kind, subject, words string, raised bool) { + key := health.FindingKey(health.ScopeSeat, packageseats.Self, kind+"."+strings.NewReplacer(".", "_", "@", "_", "/", "_").Replace(subject)) + if !raised { + _ = stdio.State(health.FindingsState).Delete(key) + return + } + f := health.Finding{V: health.FindingVersion, Key: key, Kind: kind, Ref: key[strings.LastIndex(key, ".")+1:], + Subject: health.Subject{Scope: health.ScopeSeat, ID: packageseats.Self}, Severity: health.Warning, + Summary: words, Headline: words, Explanation: words, Resolved: "cleared once it no longer holds", + Source: kind, Observer: packageseats.Self, At: time.Now().UTC()} + if _, err := stdio.State(health.FindingsState).Put(key, f); err != nil { + fmt.Fprintf(os.Stderr, "the finding %s could not be written: %v\n", key, err) + } +} + +// busHolder is the holder over the runtime's bus state, the builder's numbering and the forge. +func busHolder(forge func() (*Forge, error)) (*packageseats.Holder, error) { + callers, err := packageseats.ConfiguredCallers() + if err != nil { + return nil, err + } + return &packageseats.Holder{Callers: callers, Objects: parts.BusObjects(packageseats.IncomingObjects), + Transfers: parts.BusTransfers[packageseats.Opening](packageseats.TransfersState), + Builder: delivery.BuilderVersions{Ask: stdio.Ask}, Store: seatStore{forge}, Say: sayOn, Finding: finding}, nil +} + +// SeatTools are both seats' verbs as tools .. +func SeatTools(h func() (*packageseats.Holder, error)) []stdio.Tool { + return append(verbTools[npmreg.PublishInput, npmreg.PublishAnswer, npmreg.PublishPartInput, npmreg.PublishPartAnswer, + npmreg.VersionsInput, npmreg.VersionsAnswer, npmreg.RemoveInput, npmreg.RemoveAnswer](packageseats.NpmSeat, h), + verbTools[goreg.PublishInput, goreg.PublishAnswer, goreg.PublishPartInput, goreg.PublishPartAnswer, + goreg.VersionsInput, goreg.VersionsAnswer, goreg.RemoveInput, goreg.RemoveAnswer](packageseats.GoSeat, h)...) +} + +// chunk is publish-part's input as the holder reads it. +type chunk struct { + Data, Part, Transfer string + Last bool + Sequence int +} + +// verbTools are one seat's four verbs, each reading its input as the seat's contract type and answering in it. +func verbTools[PI, PA, PPI, PPA, VI, VA, RI, RA any](seat string, holder func() (*packageseats.Holder, error)) []stdio.Tool { + obj := map[string]any{"type": "object"} + run := func(do func(h *packageseats.Holder, args map[string]any) (any, error)) func(map[string]any) (any, error) { + return func(args map[string]any) (any, error) { + h, err := holder() + if err != nil { + return nil, err + } + return do(h, args) + } + } + return []stdio.Tool{ + {Name: seat + "." + packageseats.VerbPublish, Input: obj, + Description: "Open a transfer of a package's one file (ADR 0611): its name, the builder's version, kind package, its sha256 digest and the file as its one part; answers the part the forge lacks. A version held with that digest answers held and says published.", + Run: run(func(h *packageseats.Holder, args map[string]any) (any, error) { + in, err := as[PI](args) + if err != nil { + return nil, fmt.Errorf("publish: %w", err) + } + p, err := as[packageseats.Publish](in) + if err != nil { + return nil, err + } + a, err := h.Publish(seat, stdio.CallerOf(args), p) + out, _ := as[PA](map[string]any{"transfer": a.Transfer, "lacks": a.Lacks, "held": a.Ended && len(a.Lacks) == 0}) + return out, err + })}, + {Name: seat + "." + packageseats.VerbPublishPart, Input: obj, + Description: "Send one chunk (at most 760 KiB before base64) of the package's file, from the caller that opened the transfer; the last chunk ends it and the forge says published or refused.", + Run: run(func(h *packageseats.Holder, args map[string]any) (any, error) { + in, err := as[PPI](args) + if err != nil { + return nil, fmt.Errorf("publish-part: %w", err) + } + c, err := as[chunk](in) + if err != nil { + return nil, err + } + data, err := base64.StdEncoding.DecodeString(c.Data) + if err != nil { + return nil, fmt.Errorf("publish-part: data is not base64: %w", err) + } + a, err := h.PublishPart(seat, stdio.CallerOf(args), c.Transfer, c.Part, c.Sequence, c.Last, data) + out, _ := as[PPA](map[string]any{"transfer": c.Transfer, "part": c.Part, "received": c.Sequence + 1, "complete": a.Ended}) + return out, err + })}, + {Name: seat + "." + packageseats.VerbVersions, Input: obj, + Description: "The versions the seat holds of one package, newest first, each with its digest.", + Run: run(func(h *packageseats.Holder, args map[string]any) (any, error) { + in, err := as[VI](args) + if err != nil { + return nil, fmt.Errorf("versions: %w", err) + } + q, _ := as[struct{ Name string }](in) + got, err := h.Versions(seat, stdio.CallerOf(args), q.Name) + if err != nil { + return nil, err + } + list := []map[string]any{} + for _, v := range got { + list = append(list, map[string]any{"name": q.Name, "version": v.Version, "digest": v.Digest, "kind": packageseats.KindPackage}) + } + return as[VA](map[string]any{"name": q.Name, "versions": list}) + })}, + {Name: seat + "." + packageseats.VerbRemove, Input: obj, + Description: "Remove a tagged version (the builder, when its branch ended); says removed. An untagged version goes by the forge's retention alone.", + Run: run(func(h *packageseats.Holder, args map[string]any) (any, error) { + in, err := as[RI](args) + if err != nil { + return nil, fmt.Errorf("remove: %w", err) + } + r, _ := as[struct{ Name, Version, Reason string }](in) + o, err := h.Remove(seat, stdio.CallerOf(args), r.Name, r.Version, r.Reason) + if err != nil { + return nil, err + } + return as[RA](map[string]any{"removed": true, "artifact": map[string]any{"name": o.Name, "version": o.Version, "digest": o.Digest, "kind": o.Kind}}) + })}, + } +} diff --git a/modules/gitea/cmd/package-registry/seats_test.go b/modules/gitea/cmd/package-registry/seats_test.go new file mode 100644 index 00000000..0bff6366 --- /dev/null +++ b/modules/gitea/cmd/package-registry/seats_test.go @@ -0,0 +1,157 @@ +package main + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "gitea/internal/forgehttp" + "gitea/internal/packageseats" +) + +func tarball(t *testing.T, manifest string) []byte { + var b bytes.Buffer + gz := gzip.NewWriter(&b) + tw := tar.NewWriter(gz) + _ = tw.WriteHeader(&tar.Header{Name: "package/package.json", Mode: 0o644, Size: int64(len(manifest))}) + _, _ = tw.Write([]byte(manifest)) + _ = tw.Close() + _ = gz.Close() + return b.Bytes() +} + +func moduleZip(t *testing.T, prefix string) []byte { + var b bytes.Buffer + z := zip.NewWriter(&b) + w, _ := z.Create(prefix + "/go.mod") + _, _ = w.Write([]byte("module x\n")) + _ = z.Close() + return b.Bytes() +} + +func sha(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } + +// registry is the forge's package registry: what was uploaded, by path, and what the files API answers. +type registry struct { + mu sync.Mutex + uploaded map[string][]byte // "npm " or "go" + held map[string][]byte // "//" -> file + deleted []string +} + +func (g *registry) handle(w http.ResponseWriter, r *http.Request) { + g.mu.Lock() + defer g.mu.Unlock() + body, _ := io.ReadAll(r.Body) + p := r.URL.EscapedPath() + switch { + case r.Method == http.MethodPut && p == "/api/packages/acme/go/upload": + if r.Header.Get("Content-Type") != "application/zip" { + http.Error(w, "not a zip", 400) + return + } + g.uploaded["go"] = body + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodPut && strings.HasPrefix(p, "/api/packages/acme/npm/"): + g.uploaded["npm "+strings.TrimPrefix(p, "/api/packages/acme/npm/")] = body + w.WriteHeader(http.StatusCreated) + case strings.HasPrefix(p, "/api/v1/packages/acme/") && strings.HasSuffix(p, "/files"): + f, ok := g.held[strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/packages/acme/"), "/files")] + if !ok { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode([]map[string]any{{"sha256": sha(f), "size": len(f)}}) + case r.Method == http.MethodGet && p == "/api/v1/packages/acme": + var out []map[string]any + for k := range g.held { + parts := strings.Split(k, "/") + if parts[0] == r.URL.Query().Get("type") { + out = append(out, map[string]any{"name": strings.Join(parts[1:len(parts)-1], "/"), "version": parts[len(parts)-1], "created_at": time.Unix(int64(len(parts[len(parts)-1])), 0)}) + } + } + _ = json.NewEncoder(w).Encode(out) + case r.Method == http.MethodDelete: + g.deleted = append(g.deleted, strings.TrimPrefix(p, "/api/v1/packages/acme/")) + w.WriteHeader(http.StatusNoContent) + default: + http.Error(w, "unexpected "+r.Method+" "+p, 500) + } +} + +// **The package seats store into the forge's package registry** (design 51 *Publishing*): an npm package goes up as +// npm's publish sends it, its package.json from the tarball and its tag as its dist-tag; a Go module's zip goes up to +// the Go upload, its module path read from the zip; held, versions and remove read and delete through the forge's +// package API, a Go version with Go's leading v. +func TestThePackageSeatsStoreIntoTheForge(t *testing.T) { + g := ®istry{uploaded: map[string][]byte{}, held: map[string][]byte{}} + srv := httptest.NewServer(http.HandlerFunc(g.handle)) + defer srv.Close() + file := filepath.Join(t.TempDir(), "admin.secret") + _ = os.WriteFile(file, []byte("pw\n"), 0o600) + forge := &Forge{Client: forgehttp.Client{URL: srv.URL, User: "admin", PasswordFile: file}, Owner: "acme"} + s := seatStore{forge: func() (*Forge, error) { return forge, nil }} + + tgz := tarball(t, `{"name":"@acme/contracts","version":"0.1.15-feat-x.2","main":"index.js"}`) + p := packageseats.Publish{Name: "@acme/contracts", Version: "0.1.15-feat-x.2", Tag: "feat-x", Kind: packageseats.KindPackage, Digest: "sha256:" + sha(tgz)} + if module, err := s.Put(packageseats.NpmSeat, p, tgz); err != nil || module != "" { + t.Fatalf("npm put: %q, %v", module, err) + } + var doc struct { + DistTags map[string]string `json:"dist-tags"` + Versions map[string]map[string]any + Attachments map[string]struct{ Data string } `json:"_attachments"` + } + if err := json.Unmarshal(g.uploaded["npm @acme%2Fcontracts"], &doc); err != nil { + t.Fatalf("npm's publish body %v: %s", err, g.uploaded) + } + att := doc.Attachments["contracts-0.1.15-feat-x.2.tgz"].Data + if doc.DistTags["feat-x"] != "0.1.15-feat-x.2" || doc.DistTags["latest"] != "" || doc.Versions["0.1.15-feat-x.2"]["main"] != "index.js" || + att != base64.StdEncoding.EncodeToString(tgz) { + t.Errorf("npm's publish body: %+v", doc) + } + p.Name = "@acme/other" + if _, err := s.Put(packageseats.NpmSeat, p, tgz); err == nil || !strings.Contains(err.Error(), "not @acme/other") { + t.Errorf("a tarball of another package was uploaded: %v", err) + } + + zipped := moduleZip(t, "git.example/acme/sdk@v0.1.15") + gp := packageseats.Publish{Name: "git.example/acme/sdk", Version: "0.1.15", Kind: packageseats.KindPackage, Digest: "sha256:" + sha(zipped)} + if module, err := s.Put(packageseats.GoSeat, gp, zipped); err != nil || module != "git.example/acme/sdk" || !bytes.Equal(g.uploaded["go"], zipped) { + t.Fatalf("go put: %q, %v", module, err) + } + gp.Version = "0.1.16" + if _, err := s.Put(packageseats.GoSeat, gp, zipped); err == nil { + t.Error("a zip of another version was uploaded") + } + + g.held["go/git.example/acme/sdk/v0.1.15"] = zipped + if d, ok, err := s.Held(packageseats.GoSeat, "git.example/acme/sdk", "0.1.15"); err != nil || !ok || d != "sha256:"+sha(zipped) { + t.Errorf("held: %s %v %v", d, ok, err) + } + if _, ok, err := s.Held(packageseats.NpmSeat, "x", "0.1.0"); ok || err != nil { + t.Errorf("a version the forge does not hold: %v %v", ok, err) + } + g.held["go/git.example/acme/sdk/v0.1.9"] = zipped + if v, err := s.Versions(packageseats.GoSeat, "git.example/acme/sdk"); err != nil || len(v) != 2 || v[0].Version != "0.1.15" { + t.Errorf("versions: %+v %v", v, err) + } + if err := s.Remove(packageseats.GoSeat, "git.example/acme/sdk", "0.1.15-feat.1"); err != nil || len(g.deleted) != 1 || + g.deleted[0] != "go/git.example%2Facme%2Fsdk/v0.1.15-feat.1" { + t.Errorf("remove: %v %v", err, g.deleted) + } +} diff --git a/modules/gitea/cmd/npm-registry/semver.go b/modules/gitea/cmd/package-registry/semver.go similarity index 100% rename from modules/gitea/cmd/npm-registry/semver.go rename to modules/gitea/cmd/package-registry/semver.go diff --git a/modules/gitea/cmd/npm-registry/semver_test.go b/modules/gitea/cmd/package-registry/semver_test.go similarity index 100% rename from modules/gitea/cmd/npm-registry/semver_test.go rename to modules/gitea/cmd/package-registry/semver_test.go diff --git a/modules/gitea/go.mod b/modules/gitea/go.mod index 56c3ae98..782aa34c 100644 --- a/modules/gitea/go.mod +++ b/modules/gitea/go.mod @@ -3,6 +3,8 @@ module gitea go 1.26.0 require ( + git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/gitea/internal/forgehttp/forgehttp.go b/modules/gitea/internal/forgehttp/forgehttp.go index 3b3e8c89..0e770499 100644 --- a/modules/gitea/internal/forgehttp/forgehttp.go +++ b/modules/gitea/internal/forgehttp/forgehttp.go @@ -1,5 +1,5 @@ // Package forgehttp is the forge's HTTP interface as this module's Go bundles reach it (cmd/forge, -// cmd/npm-registry), on the forge's own machine. One copy for both bundles (novox/hq issue 539). +// cmd/package-registry), on the forge's own machine. One copy for both bundles (novox/hq issue 539). // // Two ways in, by what the manifest hands the bundle: // @@ -148,12 +148,21 @@ func (c *Client) authorize(req *http.Request) error { // 404 is ErrNotFound. One byte past the limit is read, so a body the limit cut is told from one that fits it // exactly: a cut body is refused, never read as if it were whole. func (c *Client) Do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) { + if payload == nil { + return c.Send(ctx, method, path, nil, "", limit) + } + raw, err := json.Marshal(payload) + if err != nil { + return nil, err + } + return c.Send(ctx, method, path, raw, "application/json", limit) +} + +// Send is Do with a body of its own content type, sent as it is (a package's file uploaded into the registry); +// a nil body sends none. +func (c *Client) Send(ctx context.Context, method, path string, raw []byte, contentType string, limit int64) ([]byte, error) { var body io.Reader - if payload != nil { - raw, err := json.Marshal(payload) - if err != nil { - return nil, err - } + if raw != nil { body = bytes.NewReader(raw) } req, err := http.NewRequestWithContext(ctx, method, c.URL+path, body) @@ -164,8 +173,8 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit return nil, err } req.Header.Set("Accept", "application/json") - if payload != nil { - req.Header.Set("Content-Type", "application/json") + if raw != nil { + req.Header.Set("Content-Type", contentType) } res, err := c.client().Do(req) if err != nil { diff --git a/modules/gitea/internal/packageseats/holder.go b/modules/gitea/internal/packageseats/holder.go index 92b5e48c..c7dfdea1 100644 --- a/modules/gitea/internal/packageseats/holder.go +++ b/modules/gitea/internal/packageseats/holder.go @@ -73,9 +73,9 @@ func DefaultCallers() Callers { VerbVersions: {stdio.AnyCaller}} } -// ParseCallers reads the config value; empty is DefaultCallers. -func ParseCallers(raw string) (Callers, error) { - return stdio.ParseVerbCallers(raw, Verbs, DefaultCallers()) +// ConfiguredCallers is the config value `callers` as mesh-config resolved it; unset is DefaultCallers. +func ConfiguredCallers() (Callers, error) { + return stdio.ConfiguredVerbCallers("callers", Verbs, DefaultCallers()) } // Publish is a publish as the seats' contract names it (mesh-contracts npmpackageregistry and gopackageregistry share diff --git a/modules/gitea/module.json b/modules/gitea/module.json index a8af60b9..cd0000e9 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -5,7 +5,28 @@ "requires": [ "postgres-database", "route", - "secret" + "secret", + { + "name": "incoming", + "storage": "object-store", + "template": "objects", + "options": { + "max-mib": 2048 + } + }, + { + "name": "transfers", + "storage": "key-value", + "template": "state", + "options": { + "ttl-seconds": 86400 + } + }, + { + "name": "findings", + "storage": "key-value", + "template": "state" + } ], "contributes": { "postgres-database": { @@ -135,7 +156,23 @@ "claims": [ { "name": "npm-package-registry", - "scope": "mesh" + "scope": "mesh", + "serves": [ + "publish", + "publish-part", + "remove", + "versions" + ] + }, + { + "name": "go-package-registry", + "scope": "mesh", + "serves": [ + "publish", + "publish-part", + "remove", + "versions" + ] }, { "name": "git", @@ -197,6 +234,12 @@ "default": [ "jschoubben" ] + }, + "callers": { + "type": "string", + "trusted": true, + "default": "", + "description": "per verb of the two package seats, the callers that may call it (JSON object of verb to callers; * every proved caller, operator the operator's session); empty is design 51's list: publish and publish-part mesh-build-go and mesh-build-node, remove mesh-delivery-builder, versions anyone" } }, "facts": { @@ -474,14 +517,14 @@ } }, { - "name": "npm-registry", + "name": "package-registry", "kind": "bundle", "language": "go", "system": "arch", - "from": "cmd/npm-registry", - "binary": "npm-registry", + "from": "cmd/package-registry", + "binary": "package-registry", "loads": [ - "npm-registry" + "package-registry" ], "env": { "MESH_GITEA_URL": "http://127.0.0.1:${port:3000}", @@ -594,5 +637,8 @@ "mesh-delivery-checker.owed", "mesh-delivery-indexer.modules", "mesh-delivery-gate.source" + ], + "invokes": [ + "mesh-delivery-builder.builder_versions" ] }