From 681de09f043936a6d560cf6636a66fa77538965b Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 00:49:42 +0100 Subject: [PATCH] RG2: the registry's list pages by module and cursor; the sweep reads it page by page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit oci-artifact-registry's list filters by name or module while it reads and answers at most 768 KB a page with capped, total and the cursor next, in order of the names; container-image-registry's answers whole as its form says. The registry takes a removal for retention whoever asks it: the refusal "not the sweep's" and its read of the sweep seat's holder go. The sweep reads the list from its cursor until no page is left. Builds: design 47 H2 §813 (7), §811, §779; H-A-350 DMC-10, H-A-399 --- .../holder_integration_test.go | 95 +++++++++++++++---- .../cmd/artifact-store-tools/listed.go | 42 ++++++-- .../cmd/artifact-store-tools/store_test.go | 15 ++- .../cmd/distribution-tools/main.go | 52 +++++++--- modules/distribution/go.mod | 2 +- .../internal/holder/holder_test.go | 22 +---- modules/distribution/internal/holder/list.go | 90 +++++++++++++++--- .../distribution/internal/holder/list_test.go | 39 +++++++- modules/distribution/internal/holder/seats.go | 3 - modules/distribution/internal/holder/verbs.go | 14 +-- 10 files changed, 282 insertions(+), 92 deletions(-) diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go index 4e81af3d0..9c2c64958 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go @@ -11,6 +11,7 @@ package main import ( "context" "encoding/json" + "fmt" "strings" "sync" "testing" @@ -244,26 +245,82 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) { } time.Sleep(20 * time.Millisecond) } - mu.Lock() - defer mu.Unlock() - if len(acts) != 3 || !strings.Contains(acts[0], archiveRef("letta", 2)) || !strings.Contains(acts[1], imageRef("letta", 2)) || - !strings.Contains(acts[2], mirror(1)) { - t.Fatalf("the hand acts recorded are %v; the store was asked %v", acts, door.asked) - } - if len(gone) != 1 || !strings.Contains(gone[0], `"fingerprint":"`+fp(2)+`"`) { - t.Fatalf("artifacts-gone said %v", gone) - } + func() { + mu.Lock() + defer mu.Unlock() + if len(acts) != 3 || !strings.Contains(acts[0], archiveRef("letta", 2)) || !strings.Contains(acts[1], imageRef("letta", 2)) || + !strings.Contains(acts[2], mirror(1)) { + t.Fatalf("the hand acts recorded are %v; the store was asked %v", acts, door.asked) + } + if len(gone) != 1 || !strings.Contains(gone[0], `"fingerprint":"`+fp(2)+`"`) { + t.Fatalf("artifacts-gone said %v", gone) + } - // S25 through the real findings bucket: letta's wanted image missing from the store is a finding there. - view, _ := mem.Listed(ctx) - repo, digest, _ := splitReference(imageRef("letta", 1)) - delete(view.L.Repos[repo].Revisions, digest) - if err := h.Observe(ctx); err != nil { - t.Fatal(err) - } - if keys, _ := bus.Keys("artifact-store-tools." + FindingsState); len(keys) != 1 || keys[0] != "module.letta.ace.image-missing" { - t.Fatalf("the findings are %v", keys) - } + // S25 through the real findings bucket: letta's wanted image missing from the store is a finding there. + view, _ := mem.Listed(ctx) + repo, digest, _ := splitReference(imageRef("letta", 1)) + delete(view.L.Repos[repo].Revisions, digest) + if err := h.Observe(ctx); err != nil { + t.Fatal(err) + } + if keys, _ := bus.Keys("artifact-store-tools." + FindingsState); len(keys) != 1 || keys[0] != "module.letta.ace.image-missing" { + t.Fatalf("the findings are %v", keys) + } + }() + + // Design 47 H2 §813 (7): a pass reads oci-artifact-registry's list page by page, each at most 768 KB, over 20,000 + // artifacts, so its view holds every one. + t.Run("a pass reads 20,000 artifacts page by page", func(t *testing.T) { + const many, pageBytes = 20000, 768 << 10 + repos := make([]map[string]any, many) + for i := range repos { + repos[i] = map[string]any{"name": fmt.Sprintf("m%05d/bundle", i), "manifests": []map[string]any{{ + "digest": fmt.Sprintf("sha256:%064x", i), "artifact": fmt.Sprintf("sha256:%064x", many+i), "size": 100}}} + } + pages := 0 + sub, err := nc.Subscribe("mesh.seat.oci-artifact-registry.tool.list.paged", func(m *nats.Msg) { + var in struct{ Next string } + _ = json.Unmarshal(m.Data, &in) + page, size, next := []map[string]any{}, 0, "" + for _, r := range repos { + if r["name"].(string) <= in.Next { + continue + } + raw, _ := json.Marshal(r) + if size+len(raw)+1 > pageBytes-1024 { // its comma, and room for the answer's other fields + next = page[len(page)-1]["name"].(string) + break + } + size += len(raw) + 1 + page = append(page, r) + } + pages++ + out, _ := json.Marshal(map[string]any{"repositories": page, "next": next, "capped": next != "", "total": many}) + _ = m.Respond(out) + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + store := &Store{Ask: func(key string, body any) (json.RawMessage, error) { + if key != "seat:oci-artifact-registry.list" { + return ask(key, body) + } + raw, _ := json.Marshal(body) + m, err := nc.Request("mesh.seat.oci-artifact-registry.tool.list.paged", raw, 5*time.Second) + if err != nil { + return nil, err + } + if len(m.Data) > pageBytes { + t.Errorf("a page of %d bytes", len(m.Data)) + } + return m.Data, nil + }} + got, err := store.ListOf("oci-artifact-registry", "") + if err != nil || len(got) != many || got[many-1].Name != "m19999/bundle" || pages < 2 { + t.Fatalf("read %d repositories in %d pages: %v", len(got), pages, err) + } + }) } // findingsAt reads and writes the holder's own findings in its bucket, artifact-store-tools_findings. diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/listed.go b/modules/artifact-store-tools/cmd/artifact-store-tools/listed.go index e83e881ea..61e7a59ac 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/listed.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/listed.go @@ -79,20 +79,46 @@ type Store struct { Ask func(key string, body any) (json.RawMessage, error) } -// ListOf asks seat's list, of one repository or, with name empty, of every one. +// ListOf asks seat's list, of one repository or, with name empty, of every one. oci-artifact-registry's list is read page by +// page from its cursor until it answers none (design 47 H2 §813 (7), §779); container-image-registry's answers whole. func (s *Store) ListOf(seat, name string) ([]reg.Repository, error) { if s == nil || s.Ask == nil { return nil, fmt.Errorf("this process reaches no registry seat") } - raw, err := s.Ask("seat:"+seat+"."+reg.VerbList, reg.ListInput{Name: name}) - if err != nil { - return nil, fmt.Errorf("%s's list: %w", seat, err) + if seat != ociseat.Seat { + raw, err := s.Ask("seat:"+seat+"."+reg.VerbList, reg.ListInput{Name: name}) + if err != nil { + return nil, fmt.Errorf("%s's list: %w", seat, err) + } + var a reg.ListAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("%s's answer to list: %w", seat, err) + } + return a.Repositories, nil } - var a reg.ListAnswer - if err := json.Unmarshal(raw, &a); err != nil { - return nil, fmt.Errorf("%s's answer to list: %w", seat, err) + var out []reg.Repository + seen := map[string]bool{} + for in := (ociseat.ListInput{Name: name}); ; { + raw, err := s.Ask("seat:"+seat+"."+reg.VerbList, in) + if err != nil { + return nil, fmt.Errorf("%s's list: %w", seat, err) + } + var a struct { + Repositories []reg.Repository `json:"repositories"` + Next string `json:"next"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("%s's answer to list: %w", seat, err) + } + out = append(out, a.Repositories...) + if a.Next == "" { + return out, nil + } + if seen[a.Next] { + return nil, fmt.Errorf("%s's list answered the cursor %q twice", seat, a.Next) + } + seen[a.Next], in.Next = true, a.Next } - return a.Repositories, nil } // View reads the store now, through both seats. diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go index 3af7f16e7..5943e7e10 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go @@ -92,7 +92,20 @@ func (f *fakeStore) ask(t *testing.T) func(string, any) (json.RawMessage, error) repos = append(repos, reg.Repository{Name: name, Manifests: ms}) } } - return json.Marshal(reg.ListAnswer{Repositories: repos}) + if seat != "oci-artifact-registry" { + return json.Marshal(reg.ListAnswer{Repositories: repos}) + } + // oci-artifact-registry pages (design 47 H2 §813 (7)): one repository a page, after the one its cursor names. + var page struct{ Next string } + _ = json.Unmarshal(raw, &page) + for len(repos) > 0 && repos[0].Name <= page.Next { + repos = repos[1:] + } + next := "" + if len(repos) > 1 { + repos, next = repos[:1], repos[0].Name + } + return json.Marshal(map[string]any{"repositories": repos, "next": next, "capped": next != ""}) } } diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index 2e5fb3b45..330c3c944 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -22,10 +22,10 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" reg "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry" + oci "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/parts" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/busstate" - "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" "distribution/internal/holder" @@ -314,15 +314,7 @@ func tools(v *holder.Verbs, windows kept[holder.Window], taken *asks) []stdio.To } return reg.RemoveAnswer{Removed: got.Removed, Kept: got.Kept, Reason: got.Reason, Tags: got.Tags, Artifact: ®.Artifact{Name: in.Name, Version: in.Version, Tag: in.Tag, Digest: in.Digest}}, nil }}, - stdio.Tool{Name: seat + "." + holder.VerbList, Description: "What the store holds of one repository, or of every repository the seat holds: each manifest's digest, whether it is an index and the manifests it names, the tags naming it, its size and whether it is held (ADR 0632 §3). Read from the store's own files; changes nothing.", - Input: obj(map[string]any{"name": s("/; every repository when not given")}), - Run: func(args map[string]any) (any, error) { - repos, err := v.List(context.Background(), seat, stdio.CallerOf(args), str(args, "name")) - if err != nil { - return nil, err - } - return reg.ListAnswer{Repositories: repos}, nil - }}, + listTool(v, seat), stdio.Tool{Name: seat + "." + holder.VerbHold, Description: "Keep one untagged bundle, archive or copied base from retention (the artifact sweep alone); says held.", Input: obj(map[string]any{"name": s("name"), "digest": s("sha256:…"), "reason": s("which of the sweep's rules keeps it")}, "name", "digest", "reason"), Run: func(args map[string]any) (any, error) { @@ -495,7 +487,7 @@ func main() { Transfers: boundedTransfers{parts.BusTransfers[holder.Opening](holder.TransfersState)}, Cap: boundedCap(parts.BusCap(holder.IncomingObjects)), Record: recordAct, Idle: boundedArm(parts.BusArm(holder.IdleState))} files := &holder.Files{Container: or(os.Getenv("MESH_REGISTRY_CONTAINER"), "mesh-registry"), Run: holder.ExecRunner, UID: os.Getuid()} v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{}, - Files: files, SweepHolder: func() (string, error) { return seatlist.HolderOf(stdio.Ask, holder.SweepSeat) }} + Files: files} go followMirrors(v) go parts.FollowIdle(holder.IdleState, parts.IDs(h.Transfers), h.Expired, func(id string, dropped bool, err error) { if err != nil { @@ -523,3 +515,41 @@ func or(v, d string) string { } return v } + +// listTool is the seat's verb list. On oci-artifact-registry it filters by name or module while it reads and pages at 768 KB +// by the cursor next (design 47 H2 §779, §811, §813 (7)); on container-image-registry, whose form takes no cursor, it answers +// whole. +func listTool(v *holder.Verbs, seat string) stdio.Tool { + s := func(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + description := "What the store holds of one repository, or of every repository the seat holds: each manifest's digest, whether it is an index and the manifests it names, the tags naming it, its size and whether it is held (ADR 0632 §3). Read from the store's own files; changes nothing." + props := map[string]any{"name": s("/; every repository when not given")} + if seat == holder.ArtifactSeat { + description += " Filtered by name or module while it reads and paged by the cursor next, at most 768 KB a page, capped saying the rest is read from next." + props["module"] = s("only the repositories of this module, /*") + props["next"] = s("the cursor the previous page answered; absent for the first page") + } + return stdio.Tool{Name: seat + "." + holder.VerbList, Description: description, Input: map[string]any{"type": "object", "properties": props}, + Run: func(args map[string]any) (any, error) { + if seat != holder.ArtifactSeat { + page, err := v.List(context.Background(), seat, stdio.CallerOf(args), holder.ListQuery{Name: str(args, "name")}) + if err != nil { + return nil, err + } + return reg.ListAnswer{Repositories: page.Repositories}, nil + } + in, err := arg[oci.ListInput](args) + if err != nil { + return nil, err + } + page, err := v.List(context.Background(), seat, stdio.CallerOf(args), holder.ListQuery{Name: in.Name, Module: in.Module, Next: in.Next, PageBytes: holder.ListPageBytes}) + if err != nil { + return nil, err + } + out := oci.ListAnswer{Form: oci.Form, Repositories: []oci.Repository{}, Capped: page.Capped, Next: page.Next, Total: int64(page.Total)} + raw, _ := json.Marshal(page.Repositories) + if err := json.Unmarshal(raw, &out.Repositories); err != nil { + return nil, err + } + return out, nil + }} +} diff --git a/modules/distribution/go.mod b/modules/distribution/go.mod index a16a3916a..4379a5784 100644 --- a/modules/distribution/go.mod +++ b/modules/distribution/go.mod @@ -10,6 +10,7 @@ require ( git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 github.com/nats-io/nats.go v1.54.0 @@ -31,7 +32,6 @@ require ( require ( git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect - git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect ) require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect diff --git a/modules/distribution/internal/holder/holder_test.go b/modules/distribution/internal/holder/holder_test.go index 127318716..069f08745 100644 --- a/modules/distribution/internal/holder/holder_test.go +++ b/modules/distribution/internal/holder/holder_test.go @@ -216,8 +216,7 @@ func newBed(builderOK bool) *bed { return nil }, Finding: func(kind, subject, _ string, raised bool) { b.findings[kind+" "+subject] = raised }} - b.v = &Verbs{Holder: b.h, Catalog: b.cat, Holds: memHolds{}, Mirrors: memMirrors{}, - SweepHolder: func() (string, error) { return "artifact-store-tools", nil }} + b.v = &Verbs{Holder: b.h, Catalog: b.cat, Holds: memHolds{}, Mirrors: memMirrors{}} return b } @@ -366,10 +365,6 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { _, err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/program", Version: "0.1.4", Reason: "branch ended"}) return err }, "has no tag"}, - {"a retention removal from another caller than the sweep", func(b *bed) error { - _, err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/program", Digest: part.Digest, Reason: ReasonRetention}) - return err - }, "artifact sweep's alone"}, {"a retention removal of a held archive", func(b *bed) error { _ = b.v.HoldOne(ArtifactSeat, "novox.artifact-store-tools", HoldRequest{Name: "m/a", Kind: KindArchive, Digest: part.Digest}) _, err := b.v.Remove(ArtifactSeat, "novox.artifact-store-tools", RemoveRequest{Name: "m/a", Digest: part.Digest, Reason: ReasonRetention}) @@ -378,16 +373,6 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { {"a hold of an image", func(b *bed) error { return b.v.HoldOne(ImageSeat, "novox.artifact-store-tools", HoldRequest{Name: "m/i", Kind: KindImage, Digest: part.Digest}) }, "never held"}, - {"a retention removal by the module that held the sweep seat before", func(b *bed) error { - b.v.SweepHolder = func() (string, error) { return "artifact-sweeper", nil } // the seat's holder as the seat says it now - _, err := b.v.Remove(ArtifactSeat, "novox.artifact-store-tools", RemoveRequest{Name: "m/program", Digest: part.Digest, Reason: ReasonRetention}) - return err - }, "artifact sweep's alone"}, - {"a retention removal while the sweep's holder cannot be read", func(b *bed) error { - b.v.SweepHolder = func() (string, error) { return "", errors.New("the seat's holders are not answered") } - _, err := b.v.Remove(ArtifactSeat, "novox.artifact-store-tools", RemoveRequest{Name: "m/program", Digest: part.Digest, Reason: ReasonRetention}) - return err - }, "cannot be read"}, {"a release in distribution's own name", func(b *bed) error { _, err := b.v.Release(ArtifactSeat, "novox.distribution", HoldRequest{Name: "m/a", Digest: part.Digest}) return err @@ -490,7 +475,8 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { t.Run("the sweep's removal, hold and release", func(t *testing.T) { b := newBed(true) sweep := "novox.artifact-store-tools" - if _, err := b.v.Remove(ArtifactSeat, sweep, RemoveRequest{Name: "m/a", Digest: part.Digest, Kind: KindArchive, Reason: ReasonRetention}); err != nil { + // A removal for retention is taken whoever asks it; the seat's holders are not read (design 47 H2 §813 (7)). + if _, err := b.v.Remove(ArtifactSeat, "novox.artifact-sweeper", RemoveRequest{Name: "m/a", Digest: part.Digest, Kind: KindArchive, Reason: ReasonRetention}); err != nil { t.Fatal(err) } if err := b.v.HoldOne(ArtifactSeat, sweep, HoldRequest{Name: "m/b", Kind: KindArchive, Digest: part.Digest}); err != nil { @@ -507,7 +493,7 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { for _, s := range b.said { events = append(events, s.event+" "+s.body.By) } - if want := []string{"removed " + sweep, "held " + sweep, "released person.jochen"}; !reflect.DeepEqual(events, want) { + if want := []string{"removed novox.artifact-sweeper", "held " + sweep, "released person.jochen"}; !reflect.DeepEqual(events, want) { t.Errorf("said %v, want %v", events, want) } b.cat.tags["m/p:0.1.5-feat-x.1"] = part.Digest diff --git a/modules/distribution/internal/holder/list.go b/modules/distribution/internal/holder/list.go index 022387c4b..4ae7e7ca7 100644 --- a/modules/distribution/internal/holder/list.go +++ b/modules/distribution/internal/holder/list.go @@ -9,9 +9,12 @@ package holder import ( "context" + "encoding/base64" + "encoding/json" "errors" "fmt" "sort" + "strings" reg "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry" ) @@ -25,27 +28,52 @@ type StoreFiles interface { Manifests(ctx context.Context, l *Listing) (map[string]*Manifest, map[string]string) } -// List is `list` on seat: the repositories it holds, or name alone; a repository the seat holds nothing of is not named, -// and name not held is an empty answer. A manifest whose content cannot be read fails the answer, by name: a listing -// missing one would read as the store not holding it. -func (v *Verbs) List(ctx context.Context, seat, caller, name string) ([]reg.Repository, error) { +// ListPageBytes is the most a page of oci-artifact-registry's list answers (design 47 H2 §811): 768 KB. +const ListPageBytes = 768 << 10 + +// ListQuery is what a list is asked: Name or Module (`/*`) filter the repositories while the files are read, Next +// is the cursor a page answered, and PageBytes the most a page answers, 0 for the whole answer at once (the seat +// container-image-registry's list, whose form takes no cursor). +type ListQuery struct { + Name, Module, Next string + PageBytes int +} + +// ListPage is one page of a list: its repositories in order of their names, Total the repositories the filters hold in +// all, and Next the cursor of the next page, empty at the end; Capped says the page stopped at PageBytes. +type ListPage struct { + Repositories []reg.Repository + Capped bool + Total int + Next string +} + +// List is `list` on seat: the repositories it holds, or those q's filters name; a repository the seat holds nothing of is +// not named, and a name not held is an empty answer. A paged list continues after the repository its cursor names, in +// order of the names (design 47 H2 §779, §811, §813 (7)). A manifest whose content cannot be read fails the answer, by +// name: a listing missing one would read as the store not holding it. +func (v *Verbs) List(ctx context.Context, seat, caller string, q ListQuery) (ListPage, error) { if seat != ImageSeat && seat != ArtifactSeat { - return nil, fmt.Errorf("list: %s is no seat of distribution's", seat) + return ListPage{}, fmt.Errorf("list: %s is no seat of distribution's", seat) } if v.Files == nil { - return nil, errors.New("list: this process reads no store files") + return ListPage{}, errors.New("list: this process reads no store files") + } + after, err := listCursor(q.Next) + if err != nil { + return ListPage{}, err } l, err := v.Files.List(ctx) if err != nil { - return nil, fmt.Errorf("list: the store's files cannot be listed: %w", err) + return ListPage{}, fmt.Errorf("list: the store's files cannot be listed: %w", err) } - if name != "" { - r, held := l.Repos[name] - l = &Listing{Blobs: l.Blobs, Repos: map[string]*Repo{}} - if held { - l.Repos[name] = r + kept := &Listing{Blobs: l.Blobs, Repos: map[string]*Repo{}} + for name, r := range l.Repos { + if (q.Name == "" || name == q.Name) && (q.Module == "" || strings.HasPrefix(name, q.Module+"/")) { + kept.Repos[name] = r } } + l = kept docs, unread := v.Files.Manifests(ctx, l) if len(unread) > 0 { keys := make([]string, 0, len(unread)) @@ -53,11 +81,11 @@ func (v *Verbs) List(ctx context.Context, seat, caller, name string) ([]reg.Repo keys = append(keys, k) } sort.Strings(keys) - return nil, fmt.Errorf("list: %d manifests cannot be read, the first %s: %s", len(keys), keys[0], unread[keys[0]]) + return ListPage{}, fmt.Errorf("list: %d manifests cannot be read, the first %s: %s", len(keys), keys[0], unread[keys[0]]) } holds, err := v.Holds.List() if err != nil { - return nil, fmt.Errorf("list: the holds cannot be read: %w", err) + return ListPage{}, fmt.Errorf("list: the holds cannot be read: %w", err) } held := map[string]bool{} for _, h := range holds { @@ -102,5 +130,37 @@ func (v *Verbs) List(ctx context.Context, seat, caller, name string) ([]reg.Repo out = append(out, reg.Repository{Name: repo, Manifests: manifests}) } } - return out, nil + page := ListPage{Repositories: []reg.Repository{}, Total: len(out)} + size := 0 + for _, r := range out { + if r.Name <= after { + continue + } + if q.PageBytes > 0 { + raw, _ := json.Marshal(r) + if size+len(raw) > q.PageBytes { + if len(page.Repositories) == 0 { + return ListPage{}, fmt.Errorf("list: the repository %s alone is over a page's %d bytes (%d)", r.Name, q.PageBytes, len(raw)) + } + page.Capped = true + page.Next = base64.RawURLEncoding.EncodeToString([]byte(page.Repositories[len(page.Repositories)-1].Name)) + break + } + size += len(raw) + } + page.Repositories = append(page.Repositories, r) + } + return page, nil +} + +// listCursor is the repository a page's cursor continues after, "" for the first page; a cursor no page answered is refused. +func listCursor(next string) (string, error) { + if next == "" { + return "", nil + } + name, err := base64.RawURLEncoding.DecodeString(next) + if err != nil || len(name) == 0 { + return "", fmt.Errorf("list: %q is no cursor a page of list answered", next) + } + return string(name), nil } diff --git a/modules/distribution/internal/holder/list_test.go b/modules/distribution/internal/holder/list_test.go index b49cd7dc7..3f1562738 100644 --- a/modules/distribution/internal/holder/list_test.go +++ b/modules/distribution/internal/holder/list_test.go @@ -2,6 +2,7 @@ package holder import ( "context" + "encoding/json" "fmt" "reflect" "strings" @@ -132,7 +133,8 @@ func TestListAnswersTheStoreThroughTheSeat(t *testing.T) { t.Fatal(err) } ctx := context.Background() - images, err := b.v.List(ctx, ImageSeat, "mesh-anyone", "") + page, err := b.v.List(ctx, ImageSeat, "mesh-anyone", ListQuery{}) + images := page.Repositories if err != nil { t.Fatal(err) } @@ -155,7 +157,8 @@ func TestListAnswersTheStoreThroughTheSeat(t *testing.T) { if !reflect.DeepEqual(images, want) { t.Errorf("the image seat lists %+v\nwant %+v", images, want) } - artifacts, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", "") + page, err = b.v.List(ctx, ArtifactSeat, "mesh-anyone", ListQuery{}) + artifacts := page.Repositories if err != nil { t.Fatal(err) } @@ -166,16 +169,42 @@ func TestListAnswersTheStoreThroughTheSeat(t *testing.T) { if !reflect.DeepEqual(artifacts, wantA) { t.Errorf("the artifact seat lists %+v\nwant %+v", artifacts, wantA) } - if one, err := b.v.List(ctx, ImageSeat, "mesh-anyone", "app/tools"); err != nil || len(one) != 0 { + if one, err := b.v.List(ctx, ImageSeat, "mesh-anyone", ListQuery{Name: "app/tools"}); err != nil || len(one.Repositories) != 0 { t.Errorf("the image seat's list of a bundle's repository: %+v, %v", one, err) } - if one, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", "mesh/facts"); err != nil || len(one) != 1 || one[0].Name != "mesh/facts" { + if one, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", ListQuery{Name: "mesh/facts"}); err != nil || len(one.Repositories) != 1 || one.Repositories[0].Name != "mesh/facts" { t.Errorf("one repository: %+v, %v", one, err) } + // Design 47 H2 §813 (7), §811, §779: the artifact seat's list filters by module while it reads, and pages by its cursor in + // order of the names, each page within its bytes, saying capped and the total the filters hold. + if mod, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", ListQuery{Module: "app", PageBytes: ListPageBytes}); err != nil || + !reflect.DeepEqual(mod, ListPage{Repositories: wantA[:1], Total: 1}) { + t.Errorf("the module app's repositories: %+v, %v", mod, err) + } + first, _ := json.Marshal(wantA[0]) + second, _ := json.Marshal(wantA[1]) + var paged []reg.Repository + q := ListQuery{PageBytes: max(len(first), len(second))} // one repository a page + for pages := 0; ; pages++ { + p, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", q) + if err != nil || p.Total != 2 || p.Capped != (p.Next != "") || pages > 2 { + t.Fatalf("page %d: %+v, %v", pages, p, err) + } + paged = append(paged, p.Repositories...) + if q.Next = p.Next; q.Next == "" { + break + } + } + if !reflect.DeepEqual(paged, wantA) { + t.Errorf("the pages hold %+v, want %+v", paged, wantA) + } + if _, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", ListQuery{Next: "!"}); err == nil { + t.Error("a cursor no page answered was taken") + } // A manifest whose content cannot be read fails the answer: a listing without it would read as the store not holding it. delete(f.manifests, dg("m-old")) b.v.Files = &Files{Container: "mesh-registry", Run: f.run(t)} // what was read is kept: a fresh reader reads again - if _, err := b.v.List(ctx, ImageSeat, "mesh-anyone", ""); err == nil || !strings.Contains(err.Error(), "cannot be read") { + if _, err := b.v.List(ctx, ImageSeat, "mesh-anyone", ListQuery{}); err == nil || !strings.Contains(err.Error(), "cannot be read") { t.Errorf("an unreadable manifest: %v", err) } // The files' listing: a repository with slashes read, a cut or stray one refused. diff --git a/modules/distribution/internal/holder/seats.go b/modules/distribution/internal/holder/seats.go index 3acec39c8..7df52723c 100644 --- a/modules/distribution/internal/holder/seats.go +++ b/modules/distribution/internal/holder/seats.go @@ -19,9 +19,6 @@ const ( OldSeat = "mesh-artifact-store" ) -// SweepSeat is the seat of the artifact sweep, whose holder alone removes an untagged version for retention. -const SweepSeat = "mesh-artifact-sweep" - // The verbs of the seats (design 26 *A seat's definition*; design 51 *One contract*). const ( VerbPublish = "publish" diff --git a/modules/distribution/internal/holder/verbs.go b/modules/distribution/internal/holder/verbs.go index 5451e1382..13493bbbb 100644 --- a/modules/distribution/internal/holder/verbs.go +++ b/modules/distribution/internal/holder/verbs.go @@ -88,9 +88,6 @@ type Verbs struct { Mirrors Mirrors // Files are the store's own files, which list reads (ADR 0632 §3). Files StoreFiles - // SweepHolder is the module holding the seat `mesh-artifact-sweep`, as the seat's holders say it now (design 47 G §7, §428); - // a removal for retention is that module's alone. - SweepHolder func() (string, error) } // VersionsAnswer is `versions`: the versions the seat holds of name. @@ -150,7 +147,7 @@ type Removal struct { const ReasonSharedDigest = reg.RemoveAnswerReasonSharedDigest // Remove is `remove`: the builder removes a tagged version; the artifact sweep's holder an untagged bundle, archive or -// copied base with the reason retention, which a hold keeps. Every removal says `removed` with its reason. +// copied base with the reason retention, which a hold keeps, its caller not judged (design 47 H2 §813 (7)). Every removal says `removed` with its reason. func (v *Verbs) Remove(seat, caller string, r RemoveRequest) (Removal, error) { if strings.TrimSpace(r.Name) == "" || strings.TrimSpace(r.Reason) == "" { return Removal{}, errors.New("remove: a name and a reason are required") @@ -158,13 +155,8 @@ func (v *Verbs) Remove(seat, caller string, r RemoveRequest) (Removal, error) { ev := Event{Name: r.Name, Version: r.Version, Kind: r.Kind, Digest: r.Digest, Holder: Self, Reason: r.Reason, By: caller} switch { case r.Reason == ReasonRetention: - sweep, err := v.SweepHolder() - if err != nil { - return Removal{}, fmt.Errorf("remove: the holder of %s cannot be read, so nothing is removed for retention: %w", SweepSeat, err) - } - if CallerModule(caller) != sweep { - return Removal{}, fmt.Errorf("remove: refused for %s: a removal for retention is the artifact sweep's alone", caller) - } + // The removal for retention the holder of mesh-artifact-sweep asks; its caller is not judged, as every account may call + // every verb (design 47 H2 §813 (7)). if r.Digest == "" { return Removal{}, errors.New("remove: a removal for retention names the digest") }