RG2: the registry's list pages by module and cursor; the sweep reads it page by page
oci-artifact-registry's list filters by name or module while it reads and answers at most 768 KB a page with capped, total and the cursor next, in order of the names; container-image-registry's answers whole as its form says. The registry takes a removal for retention whoever asks it: the refusal "not the sweep's" and its read of the sweep seat's holder go. The sweep reads the list from its cursor until no page is left. Builds: design 47 H2 §813 (7), §811, §779; H-A-350 DMC-10, H-A-399
This commit is contained in:
@@ -11,6 +11,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -244,26 +245,82 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(acts) != 3 || !strings.Contains(acts[0], archiveRef("letta", 2)) || !strings.Contains(acts[1], imageRef("letta", 2)) ||
|
||||
!strings.Contains(acts[2], mirror(1)) {
|
||||
t.Fatalf("the hand acts recorded are %v; the store was asked %v", acts, door.asked)
|
||||
}
|
||||
if len(gone) != 1 || !strings.Contains(gone[0], `"fingerprint":"`+fp(2)+`"`) {
|
||||
t.Fatalf("artifacts-gone said %v", gone)
|
||||
}
|
||||
func() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(acts) != 3 || !strings.Contains(acts[0], archiveRef("letta", 2)) || !strings.Contains(acts[1], imageRef("letta", 2)) ||
|
||||
!strings.Contains(acts[2], mirror(1)) {
|
||||
t.Fatalf("the hand acts recorded are %v; the store was asked %v", acts, door.asked)
|
||||
}
|
||||
if len(gone) != 1 || !strings.Contains(gone[0], `"fingerprint":"`+fp(2)+`"`) {
|
||||
t.Fatalf("artifacts-gone said %v", gone)
|
||||
}
|
||||
|
||||
// S25 through the real findings bucket: letta's wanted image missing from the store is a finding there.
|
||||
view, _ := mem.Listed(ctx)
|
||||
repo, digest, _ := splitReference(imageRef("letta", 1))
|
||||
delete(view.L.Repos[repo].Revisions, digest)
|
||||
if err := h.Observe(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys, _ := bus.Keys("artifact-store-tools." + FindingsState); len(keys) != 1 || keys[0] != "module.letta.ace.image-missing" {
|
||||
t.Fatalf("the findings are %v", keys)
|
||||
}
|
||||
// S25 through the real findings bucket: letta's wanted image missing from the store is a finding there.
|
||||
view, _ := mem.Listed(ctx)
|
||||
repo, digest, _ := splitReference(imageRef("letta", 1))
|
||||
delete(view.L.Repos[repo].Revisions, digest)
|
||||
if err := h.Observe(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys, _ := bus.Keys("artifact-store-tools." + FindingsState); len(keys) != 1 || keys[0] != "module.letta.ace.image-missing" {
|
||||
t.Fatalf("the findings are %v", keys)
|
||||
}
|
||||
}()
|
||||
|
||||
// Design 47 H2 §813 (7): a pass reads oci-artifact-registry's list page by page, each at most 768 KB, over 20,000
|
||||
// artifacts, so its view holds every one.
|
||||
t.Run("a pass reads 20,000 artifacts page by page", func(t *testing.T) {
|
||||
const many, pageBytes = 20000, 768 << 10
|
||||
repos := make([]map[string]any, many)
|
||||
for i := range repos {
|
||||
repos[i] = map[string]any{"name": fmt.Sprintf("m%05d/bundle", i), "manifests": []map[string]any{{
|
||||
"digest": fmt.Sprintf("sha256:%064x", i), "artifact": fmt.Sprintf("sha256:%064x", many+i), "size": 100}}}
|
||||
}
|
||||
pages := 0
|
||||
sub, err := nc.Subscribe("mesh.seat.oci-artifact-registry.tool.list.paged", func(m *nats.Msg) {
|
||||
var in struct{ Next string }
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
page, size, next := []map[string]any{}, 0, ""
|
||||
for _, r := range repos {
|
||||
if r["name"].(string) <= in.Next {
|
||||
continue
|
||||
}
|
||||
raw, _ := json.Marshal(r)
|
||||
if size+len(raw)+1 > pageBytes-1024 { // its comma, and room for the answer's other fields
|
||||
next = page[len(page)-1]["name"].(string)
|
||||
break
|
||||
}
|
||||
size += len(raw) + 1
|
||||
page = append(page, r)
|
||||
}
|
||||
pages++
|
||||
out, _ := json.Marshal(map[string]any{"repositories": page, "next": next, "capped": next != "", "total": many})
|
||||
_ = m.Respond(out)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
store := &Store{Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
if key != "seat:oci-artifact-registry.list" {
|
||||
return ask(key, body)
|
||||
}
|
||||
raw, _ := json.Marshal(body)
|
||||
m, err := nc.Request("mesh.seat.oci-artifact-registry.tool.list.paged", raw, 5*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(m.Data) > pageBytes {
|
||||
t.Errorf("a page of %d bytes", len(m.Data))
|
||||
}
|
||||
return m.Data, nil
|
||||
}}
|
||||
got, err := store.ListOf("oci-artifact-registry", "")
|
||||
if err != nil || len(got) != many || got[many-1].Name != "m19999/bundle" || pages < 2 {
|
||||
t.Fatalf("read %d repositories in %d pages: %v", len(got), pages, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// findingsAt reads and writes the holder's own findings in its bucket, artifact-store-tools_findings.
|
||||
|
||||
@@ -79,20 +79,46 @@ type Store struct {
|
||||
Ask func(key string, body any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// ListOf asks seat's list, of one repository or, with name empty, of every one.
|
||||
// ListOf asks seat's list, of one repository or, with name empty, of every one. oci-artifact-registry's list is read page by
|
||||
// page from its cursor until it answers none (design 47 H2 §813 (7), §779); container-image-registry's answers whole.
|
||||
func (s *Store) ListOf(seat, name string) ([]reg.Repository, error) {
|
||||
if s == nil || s.Ask == nil {
|
||||
return nil, fmt.Errorf("this process reaches no registry seat")
|
||||
}
|
||||
raw, err := s.Ask("seat:"+seat+"."+reg.VerbList, reg.ListInput{Name: name})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s's list: %w", seat, err)
|
||||
if seat != ociseat.Seat {
|
||||
raw, err := s.Ask("seat:"+seat+"."+reg.VerbList, reg.ListInput{Name: name})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s's list: %w", seat, err)
|
||||
}
|
||||
var a reg.ListAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s's answer to list: %w", seat, err)
|
||||
}
|
||||
return a.Repositories, nil
|
||||
}
|
||||
var a reg.ListAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s's answer to list: %w", seat, err)
|
||||
var out []reg.Repository
|
||||
seen := map[string]bool{}
|
||||
for in := (ociseat.ListInput{Name: name}); ; {
|
||||
raw, err := s.Ask("seat:"+seat+"."+reg.VerbList, in)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s's list: %w", seat, err)
|
||||
}
|
||||
var a struct {
|
||||
Repositories []reg.Repository `json:"repositories"`
|
||||
Next string `json:"next"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s's answer to list: %w", seat, err)
|
||||
}
|
||||
out = append(out, a.Repositories...)
|
||||
if a.Next == "" {
|
||||
return out, nil
|
||||
}
|
||||
if seen[a.Next] {
|
||||
return nil, fmt.Errorf("%s's list answered the cursor %q twice", seat, a.Next)
|
||||
}
|
||||
seen[a.Next], in.Next = true, a.Next
|
||||
}
|
||||
return a.Repositories, nil
|
||||
}
|
||||
|
||||
// View reads the store now, through both seats.
|
||||
|
||||
@@ -92,7 +92,20 @@ func (f *fakeStore) ask(t *testing.T) func(string, any) (json.RawMessage, error)
|
||||
repos = append(repos, reg.Repository{Name: name, Manifests: ms})
|
||||
}
|
||||
}
|
||||
return json.Marshal(reg.ListAnswer{Repositories: repos})
|
||||
if seat != "oci-artifact-registry" {
|
||||
return json.Marshal(reg.ListAnswer{Repositories: repos})
|
||||
}
|
||||
// oci-artifact-registry pages (design 47 H2 §813 (7)): one repository a page, after the one its cursor names.
|
||||
var page struct{ Next string }
|
||||
_ = json.Unmarshal(raw, &page)
|
||||
for len(repos) > 0 && repos[0].Name <= page.Next {
|
||||
repos = repos[1:]
|
||||
}
|
||||
next := ""
|
||||
if len(repos) > 1 {
|
||||
repos, next = repos[:1], repos[0].Name
|
||||
}
|
||||
return json.Marshal(map[string]any{"repositories": repos, "next": next, "capped": next != ""})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -22,10 +22,10 @@ import (
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
reg "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry"
|
||||
oci "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/parts"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/busstate"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"distribution/internal/holder"
|
||||
@@ -314,15 +314,7 @@ func tools(v *holder.Verbs, windows kept[holder.Window], taken *asks) []stdio.To
|
||||
}
|
||||
return reg.RemoveAnswer{Removed: got.Removed, Kept: got.Kept, Reason: got.Reason, Tags: got.Tags, Artifact: ®.Artifact{Name: in.Name, Version: in.Version, Tag: in.Tag, Digest: in.Digest}}, nil
|
||||
}},
|
||||
stdio.Tool{Name: seat + "." + holder.VerbList, Description: "What the store holds of one repository, or of every repository the seat holds: each manifest's digest, whether it is an index and the manifests it names, the tags naming it, its size and whether it is held (ADR 0632 §3). Read from the store's own files; changes nothing.",
|
||||
Input: obj(map[string]any{"name": s("<module>/<artifact>; every repository when not given")}),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
repos, err := v.List(context.Background(), seat, stdio.CallerOf(args), str(args, "name"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reg.ListAnswer{Repositories: repos}, nil
|
||||
}},
|
||||
listTool(v, seat),
|
||||
stdio.Tool{Name: seat + "." + holder.VerbHold, Description: "Keep one untagged bundle, archive or copied base from retention (the artifact sweep alone); says held.",
|
||||
Input: obj(map[string]any{"name": s("name"), "digest": s("sha256:…"), "reason": s("which of the sweep's rules keeps it")}, "name", "digest", "reason"),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
@@ -495,7 +487,7 @@ func main() {
|
||||
Transfers: boundedTransfers{parts.BusTransfers[holder.Opening](holder.TransfersState)}, Cap: boundedCap(parts.BusCap(holder.IncomingObjects)), Record: recordAct, Idle: boundedArm(parts.BusArm(holder.IdleState))}
|
||||
files := &holder.Files{Container: or(os.Getenv("MESH_REGISTRY_CONTAINER"), "mesh-registry"), Run: holder.ExecRunner, UID: os.Getuid()}
|
||||
v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{},
|
||||
Files: files, SweepHolder: func() (string, error) { return seatlist.HolderOf(stdio.Ask, holder.SweepSeat) }}
|
||||
Files: files}
|
||||
go followMirrors(v)
|
||||
go parts.FollowIdle(holder.IdleState, parts.IDs(h.Transfers), h.Expired, func(id string, dropped bool, err error) {
|
||||
if err != nil {
|
||||
@@ -523,3 +515,41 @@ func or(v, d string) string {
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// listTool is the seat's verb list. On oci-artifact-registry it filters by name or module while it reads and pages at 768 KB
|
||||
// by the cursor next (design 47 H2 §779, §811, §813 (7)); on container-image-registry, whose form takes no cursor, it answers
|
||||
// whole.
|
||||
func listTool(v *holder.Verbs, seat string) stdio.Tool {
|
||||
s := func(d string) map[string]any { return map[string]any{"type": "string", "description": d} }
|
||||
description := "What the store holds of one repository, or of every repository the seat holds: each manifest's digest, whether it is an index and the manifests it names, the tags naming it, its size and whether it is held (ADR 0632 §3). Read from the store's own files; changes nothing."
|
||||
props := map[string]any{"name": s("<module>/<artifact>; every repository when not given")}
|
||||
if seat == holder.ArtifactSeat {
|
||||
description += " Filtered by name or module while it reads and paged by the cursor next, at most 768 KB a page, capped saying the rest is read from next."
|
||||
props["module"] = s("only the repositories of this module, <module>/*")
|
||||
props["next"] = s("the cursor the previous page answered; absent for the first page")
|
||||
}
|
||||
return stdio.Tool{Name: seat + "." + holder.VerbList, Description: description, Input: map[string]any{"type": "object", "properties": props},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
if seat != holder.ArtifactSeat {
|
||||
page, err := v.List(context.Background(), seat, stdio.CallerOf(args), holder.ListQuery{Name: str(args, "name")})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reg.ListAnswer{Repositories: page.Repositories}, nil
|
||||
}
|
||||
in, err := arg[oci.ListInput](args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page, err := v.List(context.Background(), seat, stdio.CallerOf(args), holder.ListQuery{Name: in.Name, Module: in.Module, Next: in.Next, PageBytes: holder.ListPageBytes})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := oci.ListAnswer{Form: oci.Form, Repositories: []oci.Repository{}, Capped: page.Capped, Next: page.Next, Total: int64(page.Total)}
|
||||
raw, _ := json.Marshal(page.Repositories)
|
||||
if err := json.Unmarshal(raw, &out.Repositories); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ require (
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
@@ -31,7 +32,6 @@ require (
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 // indirect
|
||||
)
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect
|
||||
|
||||
@@ -216,8 +216,7 @@ func newBed(builderOK bool) *bed {
|
||||
return nil
|
||||
},
|
||||
Finding: func(kind, subject, _ string, raised bool) { b.findings[kind+" "+subject] = raised }}
|
||||
b.v = &Verbs{Holder: b.h, Catalog: b.cat, Holds: memHolds{}, Mirrors: memMirrors{},
|
||||
SweepHolder: func() (string, error) { return "artifact-store-tools", nil }}
|
||||
b.v = &Verbs{Holder: b.h, Catalog: b.cat, Holds: memHolds{}, Mirrors: memMirrors{}}
|
||||
return b
|
||||
}
|
||||
|
||||
@@ -366,10 +365,6 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
_, err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/program", Version: "0.1.4", Reason: "branch ended"})
|
||||
return err
|
||||
}, "has no tag"},
|
||||
{"a retention removal from another caller than the sweep", func(b *bed) error {
|
||||
_, err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/program", Digest: part.Digest, Reason: ReasonRetention})
|
||||
return err
|
||||
}, "artifact sweep's alone"},
|
||||
{"a retention removal of a held archive", func(b *bed) error {
|
||||
_ = b.v.HoldOne(ArtifactSeat, "novox.artifact-store-tools", HoldRequest{Name: "m/a", Kind: KindArchive, Digest: part.Digest})
|
||||
_, err := b.v.Remove(ArtifactSeat, "novox.artifact-store-tools", RemoveRequest{Name: "m/a", Digest: part.Digest, Reason: ReasonRetention})
|
||||
@@ -378,16 +373,6 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
{"a hold of an image", func(b *bed) error {
|
||||
return b.v.HoldOne(ImageSeat, "novox.artifact-store-tools", HoldRequest{Name: "m/i", Kind: KindImage, Digest: part.Digest})
|
||||
}, "never held"},
|
||||
{"a retention removal by the module that held the sweep seat before", func(b *bed) error {
|
||||
b.v.SweepHolder = func() (string, error) { return "artifact-sweeper", nil } // the seat's holder as the seat says it now
|
||||
_, err := b.v.Remove(ArtifactSeat, "novox.artifact-store-tools", RemoveRequest{Name: "m/program", Digest: part.Digest, Reason: ReasonRetention})
|
||||
return err
|
||||
}, "artifact sweep's alone"},
|
||||
{"a retention removal while the sweep's holder cannot be read", func(b *bed) error {
|
||||
b.v.SweepHolder = func() (string, error) { return "", errors.New("the seat's holders are not answered") }
|
||||
_, err := b.v.Remove(ArtifactSeat, "novox.artifact-store-tools", RemoveRequest{Name: "m/program", Digest: part.Digest, Reason: ReasonRetention})
|
||||
return err
|
||||
}, "cannot be read"},
|
||||
{"a release in distribution's own name", func(b *bed) error {
|
||||
_, err := b.v.Release(ArtifactSeat, "novox.distribution", HoldRequest{Name: "m/a", Digest: part.Digest})
|
||||
return err
|
||||
@@ -490,7 +475,8 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
t.Run("the sweep's removal, hold and release", func(t *testing.T) {
|
||||
b := newBed(true)
|
||||
sweep := "novox.artifact-store-tools"
|
||||
if _, err := b.v.Remove(ArtifactSeat, sweep, RemoveRequest{Name: "m/a", Digest: part.Digest, Kind: KindArchive, Reason: ReasonRetention}); err != nil {
|
||||
// A removal for retention is taken whoever asks it; the seat's holders are not read (design 47 H2 §813 (7)).
|
||||
if _, err := b.v.Remove(ArtifactSeat, "novox.artifact-sweeper", RemoveRequest{Name: "m/a", Digest: part.Digest, Kind: KindArchive, Reason: ReasonRetention}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.v.HoldOne(ArtifactSeat, sweep, HoldRequest{Name: "m/b", Kind: KindArchive, Digest: part.Digest}); err != nil {
|
||||
@@ -507,7 +493,7 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
for _, s := range b.said {
|
||||
events = append(events, s.event+" "+s.body.By)
|
||||
}
|
||||
if want := []string{"removed " + sweep, "held " + sweep, "released person.jochen"}; !reflect.DeepEqual(events, want) {
|
||||
if want := []string{"removed novox.artifact-sweeper", "held " + sweep, "released person.jochen"}; !reflect.DeepEqual(events, want) {
|
||||
t.Errorf("said %v, want %v", events, want)
|
||||
}
|
||||
b.cat.tags["m/p:0.1.5-feat-x.1"] = part.Digest
|
||||
|
||||
@@ -9,9 +9,12 @@ package holder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
reg "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry"
|
||||
)
|
||||
@@ -25,27 +28,52 @@ type StoreFiles interface {
|
||||
Manifests(ctx context.Context, l *Listing) (map[string]*Manifest, map[string]string)
|
||||
}
|
||||
|
||||
// List is `list` on seat: the repositories it holds, or name alone; a repository the seat holds nothing of is not named,
|
||||
// and name not held is an empty answer. A manifest whose content cannot be read fails the answer, by name: a listing
|
||||
// missing one would read as the store not holding it.
|
||||
func (v *Verbs) List(ctx context.Context, seat, caller, name string) ([]reg.Repository, error) {
|
||||
// ListPageBytes is the most a page of oci-artifact-registry's list answers (design 47 H2 §811): 768 KB.
|
||||
const ListPageBytes = 768 << 10
|
||||
|
||||
// ListQuery is what a list is asked: Name or Module (`<module>/*`) filter the repositories while the files are read, Next
|
||||
// is the cursor a page answered, and PageBytes the most a page answers, 0 for the whole answer at once (the seat
|
||||
// container-image-registry's list, whose form takes no cursor).
|
||||
type ListQuery struct {
|
||||
Name, Module, Next string
|
||||
PageBytes int
|
||||
}
|
||||
|
||||
// ListPage is one page of a list: its repositories in order of their names, Total the repositories the filters hold in
|
||||
// all, and Next the cursor of the next page, empty at the end; Capped says the page stopped at PageBytes.
|
||||
type ListPage struct {
|
||||
Repositories []reg.Repository
|
||||
Capped bool
|
||||
Total int
|
||||
Next string
|
||||
}
|
||||
|
||||
// List is `list` on seat: the repositories it holds, or those q's filters name; a repository the seat holds nothing of is
|
||||
// not named, and a name not held is an empty answer. A paged list continues after the repository its cursor names, in
|
||||
// order of the names (design 47 H2 §779, §811, §813 (7)). A manifest whose content cannot be read fails the answer, by
|
||||
// name: a listing missing one would read as the store not holding it.
|
||||
func (v *Verbs) List(ctx context.Context, seat, caller string, q ListQuery) (ListPage, error) {
|
||||
if seat != ImageSeat && seat != ArtifactSeat {
|
||||
return nil, fmt.Errorf("list: %s is no seat of distribution's", seat)
|
||||
return ListPage{}, fmt.Errorf("list: %s is no seat of distribution's", seat)
|
||||
}
|
||||
if v.Files == nil {
|
||||
return nil, errors.New("list: this process reads no store files")
|
||||
return ListPage{}, errors.New("list: this process reads no store files")
|
||||
}
|
||||
after, err := listCursor(q.Next)
|
||||
if err != nil {
|
||||
return ListPage{}, err
|
||||
}
|
||||
l, err := v.Files.List(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list: the store's files cannot be listed: %w", err)
|
||||
return ListPage{}, fmt.Errorf("list: the store's files cannot be listed: %w", err)
|
||||
}
|
||||
if name != "" {
|
||||
r, held := l.Repos[name]
|
||||
l = &Listing{Blobs: l.Blobs, Repos: map[string]*Repo{}}
|
||||
if held {
|
||||
l.Repos[name] = r
|
||||
kept := &Listing{Blobs: l.Blobs, Repos: map[string]*Repo{}}
|
||||
for name, r := range l.Repos {
|
||||
if (q.Name == "" || name == q.Name) && (q.Module == "" || strings.HasPrefix(name, q.Module+"/")) {
|
||||
kept.Repos[name] = r
|
||||
}
|
||||
}
|
||||
l = kept
|
||||
docs, unread := v.Files.Manifests(ctx, l)
|
||||
if len(unread) > 0 {
|
||||
keys := make([]string, 0, len(unread))
|
||||
@@ -53,11 +81,11 @@ func (v *Verbs) List(ctx context.Context, seat, caller, name string) ([]reg.Repo
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return nil, fmt.Errorf("list: %d manifests cannot be read, the first %s: %s", len(keys), keys[0], unread[keys[0]])
|
||||
return ListPage{}, fmt.Errorf("list: %d manifests cannot be read, the first %s: %s", len(keys), keys[0], unread[keys[0]])
|
||||
}
|
||||
holds, err := v.Holds.List()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list: the holds cannot be read: %w", err)
|
||||
return ListPage{}, fmt.Errorf("list: the holds cannot be read: %w", err)
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, h := range holds {
|
||||
@@ -102,5 +130,37 @@ func (v *Verbs) List(ctx context.Context, seat, caller, name string) ([]reg.Repo
|
||||
out = append(out, reg.Repository{Name: repo, Manifests: manifests})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
page := ListPage{Repositories: []reg.Repository{}, Total: len(out)}
|
||||
size := 0
|
||||
for _, r := range out {
|
||||
if r.Name <= after {
|
||||
continue
|
||||
}
|
||||
if q.PageBytes > 0 {
|
||||
raw, _ := json.Marshal(r)
|
||||
if size+len(raw) > q.PageBytes {
|
||||
if len(page.Repositories) == 0 {
|
||||
return ListPage{}, fmt.Errorf("list: the repository %s alone is over a page's %d bytes (%d)", r.Name, q.PageBytes, len(raw))
|
||||
}
|
||||
page.Capped = true
|
||||
page.Next = base64.RawURLEncoding.EncodeToString([]byte(page.Repositories[len(page.Repositories)-1].Name))
|
||||
break
|
||||
}
|
||||
size += len(raw)
|
||||
}
|
||||
page.Repositories = append(page.Repositories, r)
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// listCursor is the repository a page's cursor continues after, "" for the first page; a cursor no page answered is refused.
|
||||
func listCursor(next string) (string, error) {
|
||||
if next == "" {
|
||||
return "", nil
|
||||
}
|
||||
name, err := base64.RawURLEncoding.DecodeString(next)
|
||||
if err != nil || len(name) == 0 {
|
||||
return "", fmt.Errorf("list: %q is no cursor a page of list answered", next)
|
||||
}
|
||||
return string(name), nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package holder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
@@ -132,7 +133,8 @@ func TestListAnswersTheStoreThroughTheSeat(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
images, err := b.v.List(ctx, ImageSeat, "mesh-anyone", "")
|
||||
page, err := b.v.List(ctx, ImageSeat, "mesh-anyone", ListQuery{})
|
||||
images := page.Repositories
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -155,7 +157,8 @@ func TestListAnswersTheStoreThroughTheSeat(t *testing.T) {
|
||||
if !reflect.DeepEqual(images, want) {
|
||||
t.Errorf("the image seat lists %+v\nwant %+v", images, want)
|
||||
}
|
||||
artifacts, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", "")
|
||||
page, err = b.v.List(ctx, ArtifactSeat, "mesh-anyone", ListQuery{})
|
||||
artifacts := page.Repositories
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -166,16 +169,42 @@ func TestListAnswersTheStoreThroughTheSeat(t *testing.T) {
|
||||
if !reflect.DeepEqual(artifacts, wantA) {
|
||||
t.Errorf("the artifact seat lists %+v\nwant %+v", artifacts, wantA)
|
||||
}
|
||||
if one, err := b.v.List(ctx, ImageSeat, "mesh-anyone", "app/tools"); err != nil || len(one) != 0 {
|
||||
if one, err := b.v.List(ctx, ImageSeat, "mesh-anyone", ListQuery{Name: "app/tools"}); err != nil || len(one.Repositories) != 0 {
|
||||
t.Errorf("the image seat's list of a bundle's repository: %+v, %v", one, err)
|
||||
}
|
||||
if one, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", "mesh/facts"); err != nil || len(one) != 1 || one[0].Name != "mesh/facts" {
|
||||
if one, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", ListQuery{Name: "mesh/facts"}); err != nil || len(one.Repositories) != 1 || one.Repositories[0].Name != "mesh/facts" {
|
||||
t.Errorf("one repository: %+v, %v", one, err)
|
||||
}
|
||||
// Design 47 H2 §813 (7), §811, §779: the artifact seat's list filters by module while it reads, and pages by its cursor in
|
||||
// order of the names, each page within its bytes, saying capped and the total the filters hold.
|
||||
if mod, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", ListQuery{Module: "app", PageBytes: ListPageBytes}); err != nil ||
|
||||
!reflect.DeepEqual(mod, ListPage{Repositories: wantA[:1], Total: 1}) {
|
||||
t.Errorf("the module app's repositories: %+v, %v", mod, err)
|
||||
}
|
||||
first, _ := json.Marshal(wantA[0])
|
||||
second, _ := json.Marshal(wantA[1])
|
||||
var paged []reg.Repository
|
||||
q := ListQuery{PageBytes: max(len(first), len(second))} // one repository a page
|
||||
for pages := 0; ; pages++ {
|
||||
p, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", q)
|
||||
if err != nil || p.Total != 2 || p.Capped != (p.Next != "") || pages > 2 {
|
||||
t.Fatalf("page %d: %+v, %v", pages, p, err)
|
||||
}
|
||||
paged = append(paged, p.Repositories...)
|
||||
if q.Next = p.Next; q.Next == "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(paged, wantA) {
|
||||
t.Errorf("the pages hold %+v, want %+v", paged, wantA)
|
||||
}
|
||||
if _, err := b.v.List(ctx, ArtifactSeat, "mesh-anyone", ListQuery{Next: "!"}); err == nil {
|
||||
t.Error("a cursor no page answered was taken")
|
||||
}
|
||||
// A manifest whose content cannot be read fails the answer: a listing without it would read as the store not holding it.
|
||||
delete(f.manifests, dg("m-old"))
|
||||
b.v.Files = &Files{Container: "mesh-registry", Run: f.run(t)} // what was read is kept: a fresh reader reads again
|
||||
if _, err := b.v.List(ctx, ImageSeat, "mesh-anyone", ""); err == nil || !strings.Contains(err.Error(), "cannot be read") {
|
||||
if _, err := b.v.List(ctx, ImageSeat, "mesh-anyone", ListQuery{}); err == nil || !strings.Contains(err.Error(), "cannot be read") {
|
||||
t.Errorf("an unreadable manifest: %v", err)
|
||||
}
|
||||
// The files' listing: a repository with slashes read, a cut or stray one refused.
|
||||
|
||||
@@ -19,9 +19,6 @@ const (
|
||||
OldSeat = "mesh-artifact-store"
|
||||
)
|
||||
|
||||
// SweepSeat is the seat of the artifact sweep, whose holder alone removes an untagged version for retention.
|
||||
const SweepSeat = "mesh-artifact-sweep"
|
||||
|
||||
// The verbs of the seats (design 26 *A seat's definition*; design 51 *One contract*).
|
||||
const (
|
||||
VerbPublish = "publish"
|
||||
|
||||
@@ -88,9 +88,6 @@ type Verbs struct {
|
||||
Mirrors Mirrors
|
||||
// Files are the store's own files, which list reads (ADR 0632 §3).
|
||||
Files StoreFiles
|
||||
// SweepHolder is the module holding the seat `mesh-artifact-sweep`, as the seat's holders say it now (design 47 G §7, §428);
|
||||
// a removal for retention is that module's alone.
|
||||
SweepHolder func() (string, error)
|
||||
}
|
||||
|
||||
// VersionsAnswer is `versions`: the versions the seat holds of name.
|
||||
@@ -150,7 +147,7 @@ type Removal struct {
|
||||
const ReasonSharedDigest = reg.RemoveAnswerReasonSharedDigest
|
||||
|
||||
// Remove is `remove`: the builder removes a tagged version; the artifact sweep's holder an untagged bundle, archive or
|
||||
// copied base with the reason retention, which a hold keeps. Every removal says `removed` with its reason.
|
||||
// copied base with the reason retention, which a hold keeps, its caller not judged (design 47 H2 §813 (7)). Every removal says `removed` with its reason.
|
||||
func (v *Verbs) Remove(seat, caller string, r RemoveRequest) (Removal, error) {
|
||||
if strings.TrimSpace(r.Name) == "" || strings.TrimSpace(r.Reason) == "" {
|
||||
return Removal{}, errors.New("remove: a name and a reason are required")
|
||||
@@ -158,13 +155,8 @@ func (v *Verbs) Remove(seat, caller string, r RemoveRequest) (Removal, error) {
|
||||
ev := Event{Name: r.Name, Version: r.Version, Kind: r.Kind, Digest: r.Digest, Holder: Self, Reason: r.Reason, By: caller}
|
||||
switch {
|
||||
case r.Reason == ReasonRetention:
|
||||
sweep, err := v.SweepHolder()
|
||||
if err != nil {
|
||||
return Removal{}, fmt.Errorf("remove: the holder of %s cannot be read, so nothing is removed for retention: %w", SweepSeat, err)
|
||||
}
|
||||
if CallerModule(caller) != sweep {
|
||||
return Removal{}, fmt.Errorf("remove: refused for %s: a removal for retention is the artifact sweep's alone", caller)
|
||||
}
|
||||
// The removal for retention the holder of mesh-artifact-sweep asks; its caller is not judged, as every account may call
|
||||
// every verb (design 47 H2 §813 (7)).
|
||||
if r.Digest == "" {
|
||||
return Removal{}, errors.New("remove: a removal for retention names the digest")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user