artifact-store-tools: read the store through the registry seats' list, never its files or its read side (ADR 0632 §1, §4; issue 1966)
The docker exec listing and the HTTP door go; held and the sweep's index skip ask the seat's list too. The store's address file is the image seat's reach, used for the catalogue's definitions only. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
@@ -152,7 +152,7 @@ func TestOlderNamesNotReadAreSaidFirst(t *testing.T) {
|
||||
// references is one artifact_store_references tool, kept across calls as the runtime keeps the bundle.
|
||||
func references(t *testing.T, a *asked) func(map[string]any) (any, error) {
|
||||
f := world()
|
||||
s := &Store{Container: "mesh-registry", Run: f.run(t)}
|
||||
s := &Store{Ask: f.ask(t)}
|
||||
for _, tl := range Tools(s, recordsAsked{a}, &Catalogue{Ask: a.ask}) {
|
||||
if tl.Name == "artifact_store_references" {
|
||||
return tl.Run
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
package main
|
||||
|
||||
// The artifact store's door and the bus, as the holder reaches them through this module's runtime.
|
||||
// The artifact store, through its seats, and the bus, as the holder reaches them through this module's runtime.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -14,23 +15,43 @@ import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"artifactstoretools/internal/registry"
|
||||
)
|
||||
|
||||
// door is registry.Store at the store's address as this machine reaches it.
|
||||
type door struct{ s registry.Store }
|
||||
// Gone is the answer when the store does not hold it: the outcome wanted, already true.
|
||||
var Gone = errors.New("the store does not hold it")
|
||||
|
||||
// It is read only: what the sweep holds and removes it asks the registry seats (sweepseats.go).
|
||||
func (d *door) Address() string { return d.s.Address }
|
||||
func (d *door) Holds(ctx context.Context, r string) error { return d.s.Holds(ctx, r) }
|
||||
func (d *door) IsIndex(ctx context.Context, r string) (bool, error) {
|
||||
return d.s.IsIndex(ctx, r)
|
||||
// ErrNotOurs is a reference this sweep will not address: not the mesh's own, or naming nothing the store holds by digest.
|
||||
//
|
||||
// **A fact about the record, not about the store** (novox/hq issue 226). The two deserve opposite responses (skip one and
|
||||
// go on, abandon the sweep for the other), and collapsing them into "an error" is how a cautious loop became one that did
|
||||
// nothing while reporting the right number.
|
||||
var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
|
||||
|
||||
// door is the store as its seats' list answers it (ADR 0632 §4), at the address this machine reaches it by: it reads
|
||||
// only; what the sweep holds and removes it asks the registry seats too (sweepseats.go).
|
||||
type door struct {
|
||||
address string
|
||||
store *Store
|
||||
}
|
||||
|
||||
// DoorAt reads the store's address (host:port) from the file the manifest fills from the seat mesh-artifact-store's
|
||||
// reach, each time, so a store that moved is reached where it is now.
|
||||
func DoorAt(file string) func() (Door, error) {
|
||||
func (d *door) Address() string { return d.address }
|
||||
|
||||
// Holds is whether the store holds what a reference names: an image's manifest, a bundle's or an archive's blob. Gone when
|
||||
// its seat lists none (ADR 0526 clause 3: a registration by hand is refused at bytes that are not there).
|
||||
func (d *door) Holds(_ context.Context, r string) error {
|
||||
_, err := d.store.listed(r)
|
||||
return err
|
||||
}
|
||||
|
||||
// IsIndex is whether a reference is an index the store holds, as its seat lists it. Gone when it does not hold it.
|
||||
func (d *door) IsIndex(_ context.Context, r string) (bool, error) {
|
||||
e, err := d.store.listed(r)
|
||||
return e.Index, err
|
||||
}
|
||||
|
||||
// DoorAt reads the store's address (host:port) from the file the manifest fills from the seat container-image-registry's
|
||||
// reach, each time, so a store that moved is said where it is now; the store is read through s.
|
||||
func DoorAt(file string, s *Store) func() (Door, error) {
|
||||
return func() (Door, error) {
|
||||
if file == "" {
|
||||
return nil, fmt.Errorf("this bundle was given no file naming the artifact store's address")
|
||||
@@ -43,7 +64,7 @@ func DoorAt(file string) func() (Door, error) {
|
||||
if address == "" {
|
||||
return nil, fmt.Errorf("the artifact store's address is empty in %s", file)
|
||||
}
|
||||
return &door{s: registry.Store{Address: address}}, nil
|
||||
return &door{address: address, store: s}, nil
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -30,8 +30,6 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"artifactstoretools/internal/registry"
|
||||
)
|
||||
|
||||
// What the holder reads and keeps.
|
||||
@@ -71,7 +69,7 @@ type Bus interface {
|
||||
Publish(subject string, body any) error
|
||||
}
|
||||
|
||||
// Door is the artifact store's own door, read only: registry.Store, behind an interface so a test needs no store. What
|
||||
// Door is the artifact store as its seats' list answers it, read only, behind an interface so a test needs no store. What
|
||||
// the sweep holds and removes it asks the registry seats (Seats, sweepseats.go).
|
||||
type Door interface {
|
||||
Address() string
|
||||
@@ -83,7 +81,7 @@ type Door interface {
|
||||
type Holder struct {
|
||||
Bus Bus
|
||||
Door func() (Door, error)
|
||||
// Listed is the store's own listing now (Store.View); nil reads none, and then nothing is said gone or collected.
|
||||
// Listed is the store as its seats list it now (Store.View); nil reads none, and then nothing is said gone or collected.
|
||||
Listed func(ctx context.Context) (*View, error)
|
||||
Acts handacts.Recorder
|
||||
// Seats asks a registry seat's verb, `seat:<seat>.<verb>` (hold, remove): the runtime's stdio.Ask in main.
|
||||
@@ -576,7 +574,7 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
if !s.Confirmed {
|
||||
// An index waits for a confirmed collect (ADR 0251 §7, ADR 0257 §4).
|
||||
index, err := door.IsIndex(within, r)
|
||||
if err != nil && !errors.Is(err, registry.Gone) && !errors.Is(err, registry.ErrNotOurs) {
|
||||
if err != nil && !errors.Is(err, Gone) && !errors.Is(err, ErrNotOurs) {
|
||||
a.Stopped = fmt.Sprintf("the store could not say whether %s is an index, so nothing more was asked: %v", r, err)
|
||||
a.Left = len(eligible) - i
|
||||
break
|
||||
@@ -690,7 +688,7 @@ func (h *Holder) Held(refs []string) (delivery.HeldAnswer, error) {
|
||||
switch err := door.Holds(asking, canonical(r)); {
|
||||
case err == nil:
|
||||
a.Held = append(a.Held, r)
|
||||
case errors.Is(err, registry.Gone):
|
||||
case errors.Is(err, Gone):
|
||||
a.Gone = append(a.Gone, r)
|
||||
default:
|
||||
if a.Unknown == nil {
|
||||
|
||||
@@ -18,8 +18,6 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"artifactstoretools/internal/registry"
|
||||
)
|
||||
|
||||
// memBus is the holder's bus in memory: states by name, each key's history, and what was published.
|
||||
@@ -141,7 +139,7 @@ func (d *memDoor) seat(key string, body any) (json.RawMessage, error) {
|
||||
func (d *memDoor) Address() string { return "10.0.0.5:5000" }
|
||||
func (d *memDoor) Holds(_ context.Context, r string) error {
|
||||
if !d.holds[r] {
|
||||
return registry.Gone
|
||||
return Gone
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package main
|
||||
|
||||
// **The store, read through its seats** (novox/hq ADR 0632 §4): the read tools and the sweep read what the store holds
|
||||
// from the seats' `list`, container-image-registry's images and indexes and oci-artifact-registry's bundles and archives,
|
||||
// never from the store's files or its read side. What a manifest marks beyond itself is the seat's size of it, so a
|
||||
// blob two manifests share is counted in each, and what the store's nightly collector frees is not known before it runs.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
ociseat "git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry"
|
||||
|
||||
reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry"
|
||||
)
|
||||
|
||||
// Repo is one repository as the seats' list answers it.
|
||||
type Repo struct {
|
||||
Name string
|
||||
// Revisions are the digests of every manifest it holds, tagged or not.
|
||||
Revisions map[string]bool
|
||||
// Tags are what each tag names now.
|
||||
Tags map[string]string
|
||||
}
|
||||
|
||||
// Listing is the store as the seats' list answers it: every repository, and a size for each manifest and each
|
||||
// bundle's or archive's blob, keyed by its digest. A manifest's size is its own content and every blob it names, as the
|
||||
// seat answers it, so a layer two manifests share is counted in each.
|
||||
type Listing struct {
|
||||
Blobs map[string]int64
|
||||
Repos map[string]*Repo
|
||||
}
|
||||
|
||||
func (l *Listing) repo(name string) *Repo {
|
||||
r := l.Repos[name]
|
||||
if r == nil {
|
||||
r = &Repo{Name: name, Revisions: map[string]bool{}, Tags: map[string]string{}}
|
||||
l.Repos[name] = r
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// RepoNames are the repositories in order.
|
||||
func (l *Listing) RepoNames() []string {
|
||||
out := make([]string, 0, len(l.Repos))
|
||||
for n := range l.Repos {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Manifest is what one manifest names.
|
||||
type Manifest struct {
|
||||
MediaType string
|
||||
ConfigMedia string
|
||||
// Refs are the blobs it names: its configuration and its layers.
|
||||
Refs []string
|
||||
// Layers are its layers alone, in order.
|
||||
Layers []string
|
||||
// Children are the manifests an index names.
|
||||
Children []string
|
||||
}
|
||||
|
||||
// Holder is the shape of a manifest that keeps one blob: an empty configuration and one layer. The
|
||||
// controller holds every archive the mesh keeps this way (novox/hq issue 253), and keeps a named
|
||||
// document the same way, under a tag.
|
||||
func (m *Manifest) Holder() bool {
|
||||
return m != nil && m.ConfigMedia == "application/vnd.oci.empty.v1+json" && len(m.Layers) == 1
|
||||
}
|
||||
|
||||
// Key names one manifest in one repository.
|
||||
func Key(repo, digest string) string { return repo + "@" + digest }
|
||||
|
||||
// Store is the artifact store as its two seats' `list` answers it: Ask is the runtime's stdio.Ask in main.
|
||||
type Store struct {
|
||||
Ask func(key string, body any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// ListOf asks seat's list, of one repository or, with name empty, of every one.
|
||||
func (s *Store) ListOf(seat, name string) ([]reg.Repository, error) {
|
||||
if s == nil || s.Ask == nil {
|
||||
return nil, fmt.Errorf("this process reaches no registry seat")
|
||||
}
|
||||
raw, err := s.Ask("seat:"+seat+"."+reg.VerbList, reg.ListInput{Name: name})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s's list: %w", seat, err)
|
||||
}
|
||||
var a reg.ListAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s's answer to list: %w", seat, err)
|
||||
}
|
||||
return a.Repositories, nil
|
||||
}
|
||||
|
||||
// View reads the store now, through both seats.
|
||||
func (s *Store) View(context.Context) (*View, error) {
|
||||
images, err := s.ListOf(reg.Seat, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
artifacts, err := s.ListOf(ociseat.Seat, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ViewOf(images, artifacts), nil
|
||||
}
|
||||
|
||||
// ViewOf is the store as the two seats list it: each manifest a repository's revision, with its tags; an index naming its
|
||||
// platform manifests; a bundle's or archive's manifest keeping its one blob, so it reads as the holder of that archive.
|
||||
func ViewOf(seats ...[]reg.Repository) *View {
|
||||
l := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}}
|
||||
m := map[string]*Manifest{}
|
||||
held := map[string]bool{}
|
||||
for _, repos := range seats {
|
||||
for _, r := range repos {
|
||||
repo := l.repo(r.Name)
|
||||
for _, e := range r.Manifests {
|
||||
repo.Revisions[e.Digest] = true
|
||||
for _, t := range e.Tags {
|
||||
repo.Tags[t] = e.Digest
|
||||
}
|
||||
doc := &Manifest{Children: append([]string(nil), e.Platforms...)}
|
||||
l.Blobs[e.Digest] = e.Size
|
||||
if e.Artifact != "" {
|
||||
// The holder's size is the archive's with it: the blob itself counts nothing more.
|
||||
doc.ConfigMedia, doc.Layers, doc.Refs = emptyConfig, []string{e.Artifact}, []string{e.Artifact}
|
||||
if _, sized := l.Blobs[e.Artifact]; !sized {
|
||||
l.Blobs[e.Artifact] = 0
|
||||
}
|
||||
}
|
||||
m[Key(r.Name, e.Digest)] = doc
|
||||
if e.Held {
|
||||
held[Key(r.Name, e.Digest)] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return &View{L: l, M: m, Unread: map[string]string{}, Held: held}
|
||||
}
|
||||
|
||||
// emptyConfig is the configuration media type of a manifest keeping one blob.
|
||||
const emptyConfig = "application/vnd.oci.empty.v1+json"
|
||||
|
||||
// listed is the one entry of a reference's seat list naming it: by its manifest's digest, or a bundle's or an archive's by
|
||||
// the blob its reference names. Gone when the seat lists none.
|
||||
func (s *Store) listed(reference string) (reg.Listed, error) {
|
||||
seat, name, digest, err := seatOf(reference)
|
||||
if err != nil {
|
||||
return reg.Listed{}, fmt.Errorf("%w: %v", ErrNotOurs, err)
|
||||
}
|
||||
repos, err := s.ListOf(seat, name)
|
||||
if err != nil {
|
||||
return reg.Listed{}, err
|
||||
}
|
||||
for _, r := range repos {
|
||||
for _, e := range r.Manifests {
|
||||
if (seat == ociseat.Seat && e.Artifact == digest) || (seat != ociseat.Seat && e.Digest == digest) {
|
||||
return e, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return reg.Listed{}, Gone
|
||||
}
|
||||
@@ -1,11 +1,10 @@
|
||||
// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, ADR 0571 clause 2, to-be 51): a process the
|
||||
// node's runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its repositories, its
|
||||
// size, and what the mesh keeps of each manifest — read from the store's own files through the store's own container,
|
||||
// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end, and S25's image-missing.
|
||||
// size, and what the mesh keeps of each manifest — read through the registry seats' list (novox/hq ADR 0632 §4), and it
|
||||
// holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end, and S25's image-missing.
|
||||
//
|
||||
// **A module beside the store, not the store's own.** It requires no provision: what it holds and removes it asks the
|
||||
// registry seats (sweepseats.go), and it is assigned to the machine that holds the store, whose container its read
|
||||
// tools read through. stdout is the protocol; what this bundle says, it says on stderr.
|
||||
// **A module beside the store, not the store's own.** It requires no provision: what it reads, holds and removes it asks
|
||||
// the registry seats (listed.go, sweepseats.go), never the store's files or its read side. stdout is the protocol; what this bundle says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -24,15 +23,11 @@ import (
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
|
||||
func main() {
|
||||
s := &Store{
|
||||
Container: os.Getenv("MESH_ARTIFACT_STORE_CONTAINER"),
|
||||
Run: ExecRunner,
|
||||
UID: os.Getuid(),
|
||||
}
|
||||
s := &Store{Ask: stdio.Ask}
|
||||
v := &views.Reader{Module: "artifact-store-tools", Logf: logf, Findings: views.OwnFindings{}, Acts: handacts.Recorder{Ask: stdio.Ask}}
|
||||
v.Follow(delivery.AssignerModule, "wants", WantsTool)
|
||||
v.Follow(delivery.IndexerModule, "manifests", ManifestsTool)
|
||||
h := &Holder{Bus: runtimeBus{views: v}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View,
|
||||
h := &Holder{Bus: runtimeBus{views: v}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE"), s), Listed: s.View,
|
||||
Seats: stdio.Ask, Now: time.Now, Logf: logf}
|
||||
h.Acts = handacts.Recorder{Ask: stdio.Ask}
|
||||
v.Start(context.Background())
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
@@ -24,66 +25,72 @@ func d(name string) string {
|
||||
|
||||
func hexOf(digest string) string { return strings.TrimPrefix(digest, "sha256:") }
|
||||
|
||||
// fakeStore is a store's files and its manifests: what listScript and readScript would print.
|
||||
// fakeStore is a store's blobs and manifests, as the two registry seats' list answers them (ADR 0632 §4).
|
||||
type fakeStore struct {
|
||||
blobs map[string]int64
|
||||
revisions map[string][]string // repo -> digests
|
||||
tags map[string]map[string]string
|
||||
manifests map[string]string // digest -> content
|
||||
held map[string]bool // Key(repo, digest) -> held
|
||||
}
|
||||
|
||||
func (f *fakeStore) listing() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("#blobs\n")
|
||||
for dg, n := range f.blobs {
|
||||
fmt.Fprintf(&b, "%d blobs/sha256/%s/%s/data\n", n, hexOf(dg)[:2], hexOf(dg))
|
||||
}
|
||||
b.WriteString("#revisions\n")
|
||||
for repo, ds := range f.revisions {
|
||||
for _, dg := range ds {
|
||||
fmt.Fprintf(&b, "repositories/%s/_manifests/revisions/sha256/%s/link\n", repo, hexOf(dg))
|
||||
// ask is the seats as the runtime asks them: list on each, every repository or one; anything else fails the test, as
|
||||
// these tools only read.
|
||||
func (f *fakeStore) ask(t *testing.T) func(string, any) (json.RawMessage, error) {
|
||||
return func(key string, body any) (json.RawMessage, error) {
|
||||
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
|
||||
if verb != "list" || (seat != "container-image-registry" && seat != "oci-artifact-registry") {
|
||||
t.Fatalf("asked %s", key)
|
||||
}
|
||||
}
|
||||
b.WriteString("#tags\n")
|
||||
for repo, ts := range f.tags {
|
||||
for t, dg := range ts {
|
||||
fmt.Fprintf(&b, "repositories/%s/_manifests/tags/%s/current/link:%s\n", repo, t, dg)
|
||||
raw, _ := json.Marshal(body)
|
||||
var in reg.ListInput
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
repos := []reg.Repository{}
|
||||
names := make([]string, 0, len(f.revisions))
|
||||
for n := range f.revisions {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
b.WriteString("#end\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// run is the store's container as docker exec reaches it: the listing, or the manifests asked for, as
|
||||
// the two scripts print them. Anything else is a test failure: these tools only read.
|
||||
func (f *fakeStore) run(t *testing.T) Runner {
|
||||
return func(_ context.Context, name string, args ...string) Ran {
|
||||
if name != "docker" || len(args) < 5 || args[0] != "exec" || args[1] != "mesh-registry" || args[2] != "sh" || args[3] != "-c" {
|
||||
t.Fatalf("ran %s %v", name, args)
|
||||
}
|
||||
switch args[4] {
|
||||
case listScript:
|
||||
return Ran{Stdout: f.listing()}
|
||||
case readScript:
|
||||
var b strings.Builder
|
||||
for _, p := range args[6:] {
|
||||
fmt.Fprintf(&b, "#manifest %s\n", p)
|
||||
found := false
|
||||
for dg, body := range f.manifests {
|
||||
if blobPath(dg) == p {
|
||||
b.WriteString(body + "\n")
|
||||
found = true
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
if in.Name != "" && in.Name != name {
|
||||
continue
|
||||
}
|
||||
var ms []reg.Listed
|
||||
for _, dg := range f.revisions[name] {
|
||||
var doc struct {
|
||||
Config *struct{ MediaType, Digest string }
|
||||
Layers []struct{ Digest string }
|
||||
Manifests []struct{ Digest string }
|
||||
}
|
||||
_ = json.Unmarshal([]byte(f.manifests[dg]), &doc)
|
||||
e := reg.Listed{Digest: dg, Size: f.blobs[dg], Held: f.held[Key(name, dg)]}
|
||||
if doc.Config != nil {
|
||||
e.Size += f.blobs[doc.Config.Digest]
|
||||
}
|
||||
for _, l := range doc.Layers {
|
||||
e.Size += f.blobs[l.Digest]
|
||||
}
|
||||
for _, c := range doc.Manifests {
|
||||
e.Platforms = append(e.Platforms, c.Digest)
|
||||
}
|
||||
e.Index = len(e.Platforms) > 0
|
||||
if doc.Config != nil && doc.Config.MediaType == "application/vnd.oci.empty.v1+json" && len(doc.Layers) == 1 {
|
||||
e.Artifact = doc.Layers[0].Digest
|
||||
}
|
||||
for tag, at := range f.tags[name] {
|
||||
if at == dg {
|
||||
e.Tags = append(e.Tags, tag)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
b.WriteString("#absent\n")
|
||||
if (e.Artifact != "") == (seat == "oci-artifact-registry") {
|
||||
ms = append(ms, e)
|
||||
}
|
||||
}
|
||||
b.WriteString("#end\n")
|
||||
return Ran{Stdout: b.String()}
|
||||
if len(ms) > 0 {
|
||||
repos = append(repos, reg.Repository{Name: name, Manifests: ms})
|
||||
}
|
||||
}
|
||||
t.Fatalf("ran a script these tools do not have: %q", args[4])
|
||||
return Ran{}
|
||||
return json.Marshal(reg.ListAnswer{Repositories: repos})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,7 +162,7 @@ func records() *Records {
|
||||
|
||||
func view(t *testing.T, f *fakeStore) *View {
|
||||
t.Helper()
|
||||
s := &Store{Container: "mesh-registry", UID: 1000, Run: f.run(t)}
|
||||
s := &Store{Ask: f.ask(t)}
|
||||
v, err := s.View(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -170,50 +177,34 @@ func withFill(r *Records) *Records {
|
||||
return r
|
||||
}
|
||||
|
||||
func TestTheListingIsRead(t *testing.T) {
|
||||
l, err := ParseListing("#blobs\n12 blobs/sha256/ab/abcd/data\n#revisions\nrepositories/a/b/c/_manifests/revisions/sha256/ff/link\n#tags\nrepositories/a/b/c/_manifests/tags/v1/current/link:sha256:ff\n#end\n")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
// The store is read through its two seats' list (ADR 0632 §4): every manifest of every repository, tagged or not, an
|
||||
// index with the manifests it names, a bundle's or archive's manifest as the holder of its blob; each manifest's size is
|
||||
// the seat's, so nothing is unmarked before a manifest goes, and a manifest gone frees its size.
|
||||
func TestTheStoreIsReadThroughItsSeats(t *testing.T) {
|
||||
f := world()
|
||||
f.held = map[string]bool{Key("app/tools", d("h-kept")): true}
|
||||
v := view(t, f)
|
||||
server := v.L.Repos["app/server"]
|
||||
if server == nil || len(server.Revisions) != 4 || server.Tags["latest"] != d("m-stranger") {
|
||||
t.Fatalf("app/server reads %+v", server)
|
||||
}
|
||||
if l.Blobs["sha256:abcd"] != 12 {
|
||||
t.Errorf("blobs %v", l.Blobs)
|
||||
if m := v.M[Key("app/server", d("idx"))]; m == nil || !reflect.DeepEqual(m.Children, []string{d("m-kept")}) {
|
||||
t.Errorf("the index names %+v", m)
|
||||
}
|
||||
r := l.Repos["a/b/c"]
|
||||
if r == nil || !r.Revisions["sha256:ff"] || r.Tags["v1"] != "sha256:ff" {
|
||||
t.Errorf("repository with slashes misread: %+v", r)
|
||||
if m := v.M[Key("app/tools", d("h-kept"))]; !m.Holder() || m.Layers[0] != d("a-kept") {
|
||||
t.Errorf("the holder of a-kept reads %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACutListingIsRefused(t *testing.T) {
|
||||
for _, out := range []string{"#blobs\n12 blobs/sha256/ab/abcd/data\n", "#blobs\nnot a line\n#end\n", "stray\n#end\n"} {
|
||||
if _, err := ParseListing(out); err == nil {
|
||||
t.Errorf("accepted %q", out)
|
||||
}
|
||||
if _, held := v.L.Blobs[d("a-kept")]; !held || !v.Held[Key("app/tools", d("h-kept"))] {
|
||||
t.Errorf("a-kept is not read as held by the store, or its hold is not read: %v", v.Held)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheMarkIsTheCollectorsIncludingAnIndexAndSharedBlobs(t *testing.T) {
|
||||
v := view(t, world())
|
||||
marked := v.Marked(nil)
|
||||
for _, name := range []string{"cfg", "shared", "old-only", "stranger-layer", "a-kept", "a-old", "doc-body", "empty", "m-kept", "idx"} {
|
||||
if !marked[d(name)] {
|
||||
t.Errorf("%s not marked", name)
|
||||
}
|
||||
if want := int64(len(f.manifests[d("m-old")])) + 10 + 1000 + 500; v.L.Blobs[d("m-old")] != want {
|
||||
t.Errorf("m-old's size %d, want %d (its content and every blob it names)", v.L.Blobs[d("m-old")], want)
|
||||
}
|
||||
if marked[d("orphan")] {
|
||||
t.Error("a blob no manifest names is marked")
|
||||
if freed, n := v.Unmarked(nil); freed != 0 || n != 0 {
|
||||
t.Errorf("unmarked %d in %d before anything goes", freed, n)
|
||||
}
|
||||
freed, n := v.Unmarked(nil)
|
||||
if freed != 7000 || n != 1 {
|
||||
t.Errorf("the collector frees %d in %d, want 7000 in 1", freed, n)
|
||||
}
|
||||
// The index alone gone frees its own content and nothing m-kept still marks.
|
||||
_, n = v.Unmarked(map[string]bool{Key("app/server", d("idx")): true})
|
||||
if n != 2 {
|
||||
t.Errorf("without the index %d blobs unmarked, want 2 (orphan and the index itself)", n)
|
||||
}
|
||||
if sh := v.Shared(); !sh[d("empty")] || sh[d("cfg")] {
|
||||
t.Error("shared misjudged")
|
||||
if freed, n := v.Unmarked(map[string]bool{Key("app/server", d("idx")): true}); freed != v.L.Blobs[d("idx")] || n != 1 {
|
||||
t.Errorf("the index gone frees %d in %d, want its size", freed, n)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,16 +248,15 @@ func TestAStateThisBundleDoesNotKnowIsKept(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBytesFreedCountASharedBlobAsKept(t *testing.T) {
|
||||
func TestBytesFreedAreTheSizesOfWhatGoes(t *testing.T) {
|
||||
v := view(t, world())
|
||||
a := &CollectAnswer{DryRun: true, Eligible: 2, WouldLetGo: []string{
|
||||
"artifact-store://app/server@" + d("m-old"), "artifact-store://app/tools/blobs/" + d("a-old")}}
|
||||
out := Collected(v, a, true, nil)
|
||||
now, after := out["collector_frees_now_bytes"].(int64), out["collector_frees_after_bytes"].(int64)
|
||||
// m-old's own content, old-only and a-old and its holder go; shared and cfg stay with m-kept.
|
||||
want := int64(7000) + 500 + 4000 + v.L.Blobs[d("m-old")] + v.L.Blobs[d("h-old")]
|
||||
if now != 7000 || after != want {
|
||||
t.Errorf("now %d after %d, want 7000 and %d", now, after, want)
|
||||
// m-old and the holder of a-old go, each with the size its seat says.
|
||||
if want := v.L.Blobs[d("m-old")] + v.L.Blobs[d("h-old")]; now != 0 || after != want {
|
||||
t.Errorf("now %d after %d, want 0 and %d", now, after, want)
|
||||
}
|
||||
if out["manifests_gone"].(int) != 2 {
|
||||
t.Errorf("gone %v", out["manifests_gone"])
|
||||
@@ -277,8 +267,7 @@ func TestOnlyTheirBytesLeaveOutWhatAnotherStateMarks(t *testing.T) {
|
||||
v := view(t, world())
|
||||
entries := Classify(v, withFill(records()), nil)
|
||||
sums := Summarise(v, entries, entries)
|
||||
el := sums[StateEligible]
|
||||
if el.OnlyTheirBytes != 500+v.L.Blobs[d("m-old")] {
|
||||
if el := sums[StateEligible]; el.OnlyTheirBytes != v.L.Blobs[d("m-old")] {
|
||||
t.Errorf("eligible only-theirs %d", el.OnlyTheirBytes)
|
||||
}
|
||||
}
|
||||
@@ -395,7 +384,7 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
|
||||
func tool(t *testing.T, a *asked, name string) func(map[string]any) (any, error) {
|
||||
f := world()
|
||||
s := &Store{Container: "mesh-registry", Run: f.run(t)}
|
||||
s := &Store{Ask: f.ask(t)}
|
||||
for _, tl := range Tools(s, recordsAsked{a}, &Catalogue{Ask: a.ask}) {
|
||||
if tl.Name == name {
|
||||
return tl.Run
|
||||
@@ -476,7 +465,9 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
// CAT-AS1: what it holds and removes it asks the two registry seats the artifact store's holder serves.
|
||||
want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule, WantsTool, ManifestsTool,
|
||||
"seat:oci-artifact-registry.hold", "seat:oci-artifact-registry.release", "seat:oci-artifact-registry.remove",
|
||||
"seat:container-image-registry.hold", "seat:container-image-registry.release", "seat:container-image-registry.remove"}
|
||||
"seat:container-image-registry.hold", "seat:container-image-registry.release", "seat:container-image-registry.remove",
|
||||
// ADR 0632 §4: what the store holds it reads through the two seats' list.
|
||||
"seat:container-image-registry.list", "seat:oci-artifact-registry.list"}
|
||||
if !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes %v", m.Invokes)
|
||||
}
|
||||
@@ -490,6 +481,11 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
if len(m.Requires) != 0 {
|
||||
t.Errorf("requires %v, want nothing: the store is reached through the registry seats", m.Requires)
|
||||
}
|
||||
// Its address file is filled from the image seat holder's listen (ADR 0632 §1), never by the retired provision.
|
||||
address, err := os.ReadFile("../../files/store-address")
|
||||
if err != nil || strings.TrimSpace(string(address)) != "${seat:container-image-registry:reach}" {
|
||||
t.Errorf("the store's address file reads %q (%v), want the image seat's reach", address, err)
|
||||
}
|
||||
var verbs []string
|
||||
for _, tl := range SeatTools(&Holder{}, &Store{}) {
|
||||
verbs = append(verbs, strings.TrimPrefix(tl.Name, catalogue.ArtifactSweepSeat+"."))
|
||||
@@ -509,25 +505,3 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestASocketRefusalIsAskedAgainThroughSudo(t *testing.T) {
|
||||
var ran []string
|
||||
run := func(_ context.Context, name string, args ...string) Ran {
|
||||
ran = append(ran, name)
|
||||
if name == "docker" {
|
||||
return Ran{Status: 1, Stderr: "permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock"}
|
||||
}
|
||||
return Ran{Stdout: "#end\n"}
|
||||
}
|
||||
if _, err := docker(context.Background(), run, 1000, "exec", "x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(ran, []string{"docker", "sudo"}) {
|
||||
t.Errorf("ran %v", ran)
|
||||
}
|
||||
if _, err := docker(context.Background(), func(context.Context, string, ...string) Ran {
|
||||
return Ran{Status: 1, Stderr: "Error: No such container: mesh-registry"}
|
||||
}, 1000, "exec"); err == nil || !strings.Contains(err.Error(), "not on this machine") {
|
||||
t.Errorf("err %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,9 +27,9 @@ func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool {
|
||||
{
|
||||
Name: "artifact_store_repositories",
|
||||
Description: "Every repository the artifact store holds: its tags and what each names, how many manifests it holds " +
|
||||
"(tagged or not — the mesh pins by digest, so most are untagged) and its size, the blobs its manifests mark. " +
|
||||
"A blob two repositories share is counted in each, and shared_bytes says how much of a size that is. Read from " +
|
||||
"the store's own files; changes nothing. Replaces curl /v2/_catalog and /v2/<name>/tags/list. (r)",
|
||||
"(tagged or not — the mesh pins by digest, so most are untagged) and its size, each manifest's as its seat says it. " +
|
||||
"A layer two manifests share is counted in each. Read through the registry seats' list (ADR 0632 §4); changes " +
|
||||
"nothing. Replaces curl /v2/_catalog and /v2/<name>/tags/list. (r)",
|
||||
Input: map[string]any{"repository": repoArg},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
v, err := s.View(ctx)
|
||||
@@ -41,9 +41,9 @@ func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool {
|
||||
},
|
||||
{
|
||||
Name: "artifact_store_usage",
|
||||
Description: "How large the artifact store is: every blob's bytes together, the largest repositories, the bytes " +
|
||||
"more than one repository's manifests mark, and the bytes no manifest marks — what the store's nightly collector " +
|
||||
"frees next. Read from the store's own files; changes nothing. Replaces du on the store's directory. (r)",
|
||||
Description: "How large the artifact store is: every manifest's size together, as its seat says it (a layer two " +
|
||||
"manifests share is counted in each), and the largest repositories. Read through the registry seats' list (ADR " +
|
||||
"0632 §4); changes nothing. Replaces du on the store's directory. (r)",
|
||||
Input: map[string]any{"top": map[string]any{"type": "integer", "description": "how many of the largest repositories to name (default 10, at most 200)"}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
top, err := count(args, "top", 10, 1, 200)
|
||||
@@ -139,16 +139,6 @@ func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool {
|
||||
}
|
||||
}
|
||||
|
||||
// View reads the store now.
|
||||
func (s *Store) View(ctx context.Context) (*View, error) {
|
||||
l, err := s.List(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, unread := s.Manifests(ctx, l)
|
||||
return &View{L: l, M: m, Unread: unread}, nil
|
||||
}
|
||||
|
||||
// unreadNote says what manifests that could not be read do to the numbers, when there are any.
|
||||
func unreadNote(v *View) map[string]any {
|
||||
if len(v.Unread) == 0 {
|
||||
|
||||
@@ -4,13 +4,15 @@ import (
|
||||
"sort"
|
||||
)
|
||||
|
||||
// View is the store at one moment: its files, and every manifest's content that could be read.
|
||||
// View is the store at one moment, as its seats list it (listed.go).
|
||||
type View struct {
|
||||
L *Listing
|
||||
// M is each manifest read, by Key.
|
||||
M map[string]*Manifest
|
||||
// Unread is each manifest that could not be read, by Key, with why.
|
||||
Unread map[string]string
|
||||
// Held is each manifest a hold on its seat keeps from retention, by Key.
|
||||
Held map[string]bool
|
||||
}
|
||||
|
||||
// Marks are the blobs one manifest keeps from the store's collector: its own content, its
|
||||
|
||||
Reference in New Issue
Block a user