Hold every kept archive by a manifest so the store's collector keeps it (hq issue 253)

The store's garbage-collect marks only from manifests, and archives were
published as bare blobs, so the first real collection would delete every
archive the mesh keeps. PublishArchive now puts a deterministic OCI holder
manifest (empty config, one layer) beside each archive; the sweep holds every
kept archive before it lets anything go, which backfills existing bare blobs,
and lets go of an archive holder-first. A forgotten module no longer keeps its
five recent builds (ADR 0189). `collection [--json]` reports kept archives
held/unheld and what may be let go, so the dry run can be lifted on evidence.

internal/artifacts/hold.go

@@ -0,0 +1,345 @@
+package artifacts
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189).
+//
+// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and
+// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs
+// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what
+// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a
+// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a
+// bare blob is unreferenced, so the first real collection would have deleted every archive the
+// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The
+// collector runs `--dry-run` until this is true.
+//
+// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty
+// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The
+// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its
+// holder first, which is what lets the bytes go at the next collection.
+//
+// **Nothing a machine reads changes.** The recorded reference stays
+// `artifact-store://<module>/<artifact>/blobs/sha256:…`, and machines fetch the blob exactly as
+// before. The holder is the store's bookkeeping, not a second way to reach anything.
+//
+// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest
+// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can
+// compute which manifest holds any archive from the reference it already has plus one HEAD for the
+// size. No schema change, no second record that could disagree with the store.
+
+const (
+	// mediaManifest is the type a holder is put and asked for as.
+	mediaManifest = "application/vnd.oci.image.manifest.v1+json"
+	// mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest
+	// whose only purpose is to name its layer.
+	mediaEmpty = "application/vnd.oci.empty.v1+json"
+	// emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec.
+	emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"
+	// mediaArchive is the layer type an archive is held as. Every archive the builder publishes is
+	// `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant,
+	// not read from anywhere, because the holder must be recomputable from the reference alone.
+	mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip"
+)
+
+// emptyConfig is the content emptyDigest names.
+var emptyConfig = []byte("{}")
+
+// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a
+// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well
+// (measured 2026-09-28; internal/builder/registry.go says how that was found).
+var manifestAccept = []string{
+	mediaManifest,
+	"application/vnd.docker.distribution.manifest.v2+json",
+}
+
+type descriptor struct {
+	MediaType string `json:"mediaType"`
+	Digest    string `json:"digest"`
+	Size      int64  `json:"size"`
+}
+
+type holderManifest struct {
+	SchemaVersion int          `json:"schemaVersion"`
+	MediaType     string       `json:"mediaType"`
+	Config        descriptor   `json:"config"`
+	Layers        []descriptor `json:"layers"`
+}
+
+// Holder is the manifest that holds an archive in the store, and its digest.
+//
+// A pure function of the archive's digest and size: the same two in give the same bytes out,
+// always, because `encoding/json` writes a struct's fields in their declared order and there is
+// nothing here that varies by when or where it was composed.
+func Holder(digest string, size int64) (body []byte, holder string) {
+	body, err := json.Marshal(holderManifest{
+		SchemaVersion: 2,
+		MediaType:     mediaManifest,
+		Config:        descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))},
+		Layers:        []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}},
+	})
+	if err != nil {
+		// Marshalling a struct of strings and integers cannot fail.
+		panic(err)
+	}
+	sum := sha256.Sum256(body)
+	return body, "sha256:" + hex.EncodeToString(sum[:])
+}
+
+// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one.
+//
+// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so
+// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which
+// is what lets the sweep run it over every kept archive on every build and so backfill the bare
+// blobs published before holders existed (novox/hq issue 253).
+//
+// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a
+// fact the caller reports rather than one this invents a remedy for.
+func (s Store) Hold(ctx context.Context, reference string) (bool, error) {
+	repository, digest, archive, err := s.archive(reference)
+	if err != nil || !archive {
+		return false, err
+	}
+	size, err := s.blobSize(ctx, repository, digest)
+	if err != nil {
+		return false, err
+	}
+	return s.HoldBlob(ctx, repository, digest, size)
+}
+
+// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the
+// question an operator needs answered with "none unheld" before the collector is let loose.
+//
+// An image answers true: it is its own manifest. An archive the store does not have answers Gone.
+func (s Store) Held(ctx context.Context, reference string) (bool, error) {
+	repository, digest, archive, err := s.archive(reference)
+	if err != nil {
+		return false, err
+	}
+	if !archive {
+		return true, nil
+	}
+	size, err := s.blobSize(ctx, repository, digest)
+	if err != nil {
+		return false, err
+	}
+	_, holder := Holder(digest, size)
+	return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
+}
+
+// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is
+// there already. Answers whether it wrote one.
+//
+// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size
+// the store reports. Both arrive at the same holder, which is the point of composing it.
+func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) {
+	if s.Address == "" {
+		return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest)
+	}
+	body, holder := Holder(digest, size)
+	there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
+	if err != nil {
+		return false, err
+	}
+	if there {
+		return false, nil
+	}
+
+	// The config must be in the repository before a manifest naming it is accepted: a registry
+	// refuses a manifest whose blobs it cannot find there, which is the property that makes a
+	// holder mean something.
+	if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil {
+		return false, err
+	}
+
+	// **By digest, never by tag.** A tag would be one more name to move and one more thing the
+	// collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that
+	// it pins by digest, and an untagged manifest is kept by plain collection.
+	request, err := http.NewRequestWithContext(ctx, http.MethodPut,
+		s.url(repository, "manifests", holder), bytes.NewReader(body))
+	if err != nil {
+		return false, err
+	}
+	request.Header.Set("Content-Type", mediaManifest)
+	response, err := s.client().Do(request)
+	if err != nil {
+		return false, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusCreated {
+		said, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
+		return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s",
+			repository, digest, response.Status, strings.TrimSpace(string(said)))
+	}
+	return true, nil
+}
+
+// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes.
+//
+// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the
+// collector would keep its bytes for ever on the strength of it — the sweep would record the
+// archive collected while the disk said otherwise. Deleting the holder first and failing before
+// the link goes leaves an unheld archive that the next sweep still offers, which is safe.
+//
+// A store that no longer has the blob answers Gone: without its size the holder cannot be named,
+// and without the blob there is nothing left for a holder to keep. A store that never had a holder
+// for it — an archive published before holders, never backfilled — answers 404 to the delete, and
+// that is the outcome wanted.
+func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error {
+	size, err := s.blobSize(ctx, repository, digest)
+	if err != nil {
+		return err
+	}
+	_, holder := Holder(digest, size)
+	err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder)
+	if err == Gone {
+		return nil
+	}
+	return err
+}
+
+// archive reads a recorded reference into its repository and digest, and whether it is an archive
+// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store.
+func (s Store) archive(reference string) (repository, digest string, archive bool, err error) {
+	path, kept := catalogue.InArtifactStore(reference)
+	if !kept {
+		return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
+	}
+	if s.Address == "" {
+		return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
+	}
+	repository, kind, digest, err := split(path)
+	if err != nil {
+		return "", "", false, err
+	}
+	return repository, digest, kind == "blobs", nil
+}
+
+// blobSize is how large the store says a blob is; Gone when it does not have it.
+func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil)
+	if err != nil {
+		return 0, err
+	}
+	response, err := s.client().Do(request)
+	if err != nil {
+		return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
+	}
+	defer response.Body.Close()
+	switch response.StatusCode {
+	case http.StatusOK:
+	case http.StatusNotFound:
+		return 0, Gone
+	default:
+		return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
+	}
+	// Read from the header rather than ContentLength: a HEAD's ContentLength is what the response
+	// says it would have sent, which Go reports faithfully, but a proxy in between is free to drop
+	// it, and the header is what the registry itself wrote.
+	if length := response.Header.Get("Content-Length"); length != "" {
+		if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 {
+			return n, nil
+		}
+	}
+	if response.ContentLength >= 0 {
+		return response.ContentLength, nil
+	}
+	return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest)
+}
+
+// putBlob uploads a small blob unless the repository already has it: ask where, then put it there
+// naming the digest — the registry's own two steps, the same the builder takes for an archive.
+func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error {
+	if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil {
+		return err
+	} else if there {
+		return nil
+	}
+	start, err := http.NewRequestWithContext(ctx, http.MethodPost,
+		"http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil)
+	if err != nil {
+		return err
+	}
+	begun, err := s.client().Do(start)
+	if err != nil {
+		return fmt.Errorf("cannot start an upload to %s: %w", repository, err)
+	}
+	begun.Body.Close()
+	if begun.StatusCode != http.StatusAccepted {
+		return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository)
+	}
+	where := begun.Header.Get("Location")
+	if where == "" {
+		return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository)
+	}
+	if strings.HasPrefix(where, "/") {
+		where = "http://" + s.Address + where
+	}
+	separator := "?"
+	if strings.Contains(where, "?") {
+		separator = "&"
+	}
+	put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	put.Header.Set("Content-Type", "application/octet-stream")
+	done, err := s.client().Do(put)
+	if err != nil {
+		return err
+	}
+	defer done.Body.Close()
+	if done.StatusCode != http.StatusCreated {
+		said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
+		return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said)))
+	}
+	return nil
+}
+
+// has is whether the store answers 200 for a HEAD at that URL.
+func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
+	if err != nil {
+		return false, err
+	}
+	for _, media := range accept {
+		request.Header.Add("Accept", media)
+	}
+	response, err := s.client().Do(request)
+	if err != nil {
+		return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
+	}
+	defer response.Body.Close()
+	switch response.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusNotFound:
+		return false, nil
+	default:
+		return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url)
+	}
+}
+
+func (s Store) url(repository, kind, digest string) string {
+	return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
+}
+
+func (s Store) client() *http.Client {
+	if s.HTTP != nil {
+		return s.HTTP
+	}
+	return &http.Client{Timeout: 30 * time.Second}
+}

internal/artifacts/hold_test.go

@@ -0,0 +1,268 @@
+package artifacts
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189).
+//
+// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a
+// registry refuses — a blob whose digest does not match, a manifest whose digest does not match
+// or whose blobs the repository does not have, a manifest asked for without an Accept naming its
+// type. What is asserted is this side's decisions; the live test below asserts the registry's.
+
+type memRegistry struct {
+	mu        sync.Mutex
+	blobs     map[string][]byte // repository + "@" + digest
+	manifests map[string][]byte // repository + "@" + digest
+	writes    []string          // every PUT and DELETE, as "METHOD path"
+}
+
+func digestOf(body []byte) string {
+	sum := sha256.Sum256(body)
+	return "sha256:" + hex.EncodeToString(sum[:])
+}
+
+func (m *memRegistry) serve(t *testing.T) Store {
+	t.Helper()
+	m.blobs = map[string][]byte{}
+	m.manifests = map[string][]byte{}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		m.mu.Lock()
+		defer m.mu.Unlock()
+		path := strings.TrimPrefix(r.URL.Path, "/v2/")
+		if r.Method == http.MethodPut || r.Method == http.MethodDelete {
+			m.writes = append(m.writes, r.Method+" "+r.URL.Path)
+		}
+		switch {
+		case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"):
+			repository := strings.TrimSuffix(path, "/blobs/uploads/")
+			w.Header().Set("Location", "/upload/"+repository+"?state=x")
+			w.WriteHeader(http.StatusAccepted)
+		case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"):
+			repository := strings.TrimPrefix(r.URL.Path, "/upload/")
+			body, _ := io.ReadAll(r.Body)
+			digest := r.URL.Query().Get("digest")
+			if digest != digestOf(body) {
+				w.WriteHeader(http.StatusBadRequest)
+				return
+			}
+			m.blobs[repository+"@"+digest] = body
+			w.WriteHeader(http.StatusCreated)
+		case strings.Contains(path, "/blobs/"):
+			repository, digest, _ := strings.Cut(path, "/blobs/")
+			key := repository + "@" + digest
+			body, ok := m.blobs[key]
+			if !ok {
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+			switch r.Method {
+			case http.MethodHead:
+				w.Header().Set("Content-Length", strconv.Itoa(len(body)))
+				w.WriteHeader(http.StatusOK)
+			case http.MethodDelete:
+				delete(m.blobs, key)
+				w.WriteHeader(http.StatusAccepted)
+			default:
+				w.WriteHeader(http.StatusMethodNotAllowed)
+			}
+		case strings.Contains(path, "/manifests/"):
+			repository, digest, _ := strings.Cut(path, "/manifests/")
+			key := repository + "@" + digest
+			switch r.Method {
+			case http.MethodHead:
+				if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) {
+					w.WriteHeader(http.StatusNotFound)
+					return
+				}
+				w.WriteHeader(http.StatusOK)
+			case http.MethodPut:
+				body, _ := io.ReadAll(r.Body)
+				if digest != digestOf(body) {
+					w.WriteHeader(http.StatusBadRequest)
+					return
+				}
+				var named holderManifest
+				if err := json.Unmarshal(body, &named); err != nil {
+					w.WriteHeader(http.StatusBadRequest)
+					return
+				}
+				for _, d := range append([]descriptor{named.Config}, named.Layers...) {
+					if _, ok := m.blobs[repository+"@"+d.Digest]; !ok {
+						w.WriteHeader(http.StatusBadRequest)
+						fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest)
+						return
+					}
+				}
+				m.manifests[key] = body
+				w.WriteHeader(http.StatusCreated)
+			case http.MethodDelete:
+				if _, ok := m.manifests[key]; !ok {
+					w.WriteHeader(http.StatusNotFound)
+					return
+				}
+				delete(m.manifests, key)
+				w.WriteHeader(http.StatusAccepted)
+			}
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	t.Cleanup(server.Close)
+	return Store{Address: strings.TrimPrefix(server.URL, "http://")}
+}
+
+// bare puts an archive in the store the way the builder did before holders: a blob, nothing more.
+func (m *memRegistry) bare(repository string, body []byte) string {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	digest := digestOf(body)
+	m.blobs[repository+"@"+digest] = body
+	return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
+}
+
+func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) {
+	// The sweep must arrive at the very manifest the builder wrote, with nothing recorded between
+	// them. Same inputs, same bytes — and a different size is a different holder, so a holder can
+	// never be mistaken for one of a different blob.
+	digest := "sha256:" + strings.Repeat("a", 64)
+	one, first := Holder(digest, 42)
+	two, second := Holder(digest, 42)
+	if !bytes.Equal(one, two) || first != second {
+		t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two)
+	}
+	if _, other := Holder(digest, 43); other == first {
+		t.Fatal("a different size composed the same holder")
+	}
+	want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` +
+		`"config":{"mediaType":"application/vnd.oci.empty.v1+json",` +
+		`"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` +
+		`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}`
+	if string(one) != want {
+		t.Fatalf("the holder is\n%s\nwant\n%s", one, want)
+	}
+	if digestOf(emptyConfig) != emptyDigest {
+		t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest)
+	}
+}
+
+func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) {
+	// The archives published before this have no holder. Hold, run over every kept archive on
+	// every sweep, writes one the first time and nothing after.
+	m := &memRegistry{}
+	store := m.serve(t)
+	ctx := context.Background()
+	body := []byte("a theme")
+	reference := m.bare("shell/config", body)
+
+	if held, err := store.Held(ctx, reference); err != nil || held {
+		t.Fatalf("a bare blob reads as held=%v (%v)", held, err)
+	}
+	wrote, err := store.Hold(ctx, reference)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !wrote {
+		t.Fatal("holding a bare archive wrote nothing")
+	}
+	_, holder := Holder(digestOf(body), int64(len(body)))
+	if _, ok := m.manifests["shell/config@"+holder]; !ok {
+		t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder)
+	}
+	if held, err := store.Held(ctx, reference); err != nil || !held {
+		t.Fatalf("after holding, held=%v (%v)", held, err)
+	}
+
+	writes := len(m.writes)
+	wrote, err = store.Hold(ctx, reference)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if wrote || len(m.writes) != writes {
+		t.Fatalf("holding again wrote %v", m.writes[writes:])
+	}
+}
+
+func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) {
+	m := &memRegistry{}
+	store := m.serve(t)
+	ctx := context.Background()
+
+	// An image is its own manifest: nothing is asked.
+	wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64))
+	if err != nil || wrote || len(m.writes) != 0 {
+		t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes)
+	}
+	// An archive the store does not have is a fact to report, not something to invent a holder for.
+	_, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64))
+	if !errors.Is(err, Gone) {
+		t.Fatalf("holding a missing archive answered %v, want Gone", err)
+	}
+	// And a reference that is not the mesh's is refused as such.
+	if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) {
+		t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err)
+	}
+}
+
+func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) {
+	// A holder left behind would keep the bytes through every collection while the record said
+	// collected; the link deleted first and the holder failing after would be that exactly.
+	m := &memRegistry{}
+	store := m.serve(t)
+	ctx := context.Background()
+	body := []byte("an old theme")
+	reference := m.bare("shell/config", body)
+	if _, err := store.Hold(ctx, reference); err != nil {
+		t.Fatal(err)
+	}
+	m.writes = nil
+
+	if err := store.LetGo(ctx, reference); err != nil {
+		t.Fatal(err)
+	}
+	_, holder := Holder(digestOf(body), int64(len(body)))
+	want := []string{
+		"DELETE /v2/shell/config/manifests/" + holder,
+		"DELETE /v2/shell/config/blobs/" + digestOf(body),
+	}
+	if strings.Join(m.writes, "\n") != strings.Join(want, "\n") {
+		t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n"))
+	}
+	if len(m.manifests) != 0 {
+		t.Fatalf("a holder survived: %v", m.manifests)
+	}
+	// Asked again, the archive is already gone, which is the outcome wanted.
+	if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) {
+		t.Fatalf("letting go twice answered %v, want Gone", err)
+	}
+}
+
+func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) {
+	// An archive published before holders and let go of before any sweep held it: the holder's
+	// delete answers 404, which is the outcome wanted, and the blob still goes.
+	m := &memRegistry{}
+	store := m.serve(t)
+	reference := m.bare("shell/config", []byte("never held"))
+	if err := store.LetGo(context.Background(), reference); err != nil {
+		t.Fatal(err)
+	}
+	if len(m.blobs) != 0 {
+		t.Fatalf("the blob survived: %v", m.blobs)
+	}
+}

internal/artifacts/live_registry_test.go

@@ -0,0 +1,130 @@
+package artifacts
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// The registry's own collector keeps a held archive and takes a bare one (novox/hq issue 253,
+// ADR 0189).
+//
+// Everything else here is asserted against a fake, which can only say what this side asks. This is
+// the one question a fake cannot answer — what `registry garbage-collect` actually does with what
+// this side wrote — and it is the whole of whether the store's nightly step may stop being a dry
+// run. Against the very image the mesh's store runs:
+//
+//	docker run -d --rm --name mesh-controller-registry -p 15000:5000 \
+//	    -e REGISTRY_STORAGE_DELETE_ENABLED=true registry:2.8.3
+//	MESH_TEST_REGISTRY=127.0.0.1:15000 MESH_TEST_REGISTRY_CONTAINER=mesh-controller-registry \
+//	    go test -run Live ./internal/artifacts/
+//	docker stop mesh-controller-registry
+//
+// Skipped without both variables: it needs a registry it may write to and collect, and a container
+// to run the collector in.
+func TestLiveTheRegistrysCollectorKeepsWhatIsHeldAndTakesWhatIsNot(t *testing.T) {
+	address := os.Getenv("MESH_TEST_REGISTRY")
+	container := os.Getenv("MESH_TEST_REGISTRY_CONTAINER")
+	if address == "" || container == "" {
+		t.Skip("no MESH_TEST_REGISTRY / MESH_TEST_REGISTRY_CONTAINER; see this test's comment for the registry to raise")
+	}
+	ctx := context.Background()
+	store := Store{Address: address}
+	run := time.Now().UnixNano()
+
+	// Four archives in four repositories, each a different story. Distinct bytes per run, so a
+	// registry reused across runs cannot answer for an earlier one.
+	put := func(name string) (repository, digest string, body []byte) {
+		repository = fmt.Sprintf("live-%d/%s", run, name)
+		body = []byte(fmt.Sprintf("%s archive of run %d", name, run))
+		digest = digestOf(body)
+		if err := store.putBlob(ctx, repository, digest, body); err != nil {
+			t.Fatal(err)
+		}
+		return repository, digest, body
+	}
+	reference := func(repository, digest string) string {
+		return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
+	}
+
+	// Published held — what PublishArchive now does.
+	heldRepo, heldDigest, heldBody := put("held")
+	if _, err := store.HoldBlob(ctx, heldRepo, heldDigest, int64(len(heldBody))); err != nil {
+		t.Fatalf("the registry refused a holder: %v", err)
+	}
+	// Published bare, as before, and never held: what the collector must take.
+	_, bareDigest, _ := put("bare")
+	// Published bare and then held by the sweep: the backfill.
+	backRepo, backDigest, backBody := put("backfilled")
+	if wrote, err := store.Hold(ctx, reference(backRepo, backDigest)); err != nil || !wrote {
+		t.Fatalf("backfilling wrote=%v: %v", wrote, err)
+	}
+	if held, err := store.Held(ctx, reference(backRepo, backDigest)); err != nil || !held {
+		t.Fatalf("after backfilling, held=%v: %v", held, err)
+	}
+	// Held, and then let go of by the sweep: holder first, then the link.
+	goneRepo, goneDigest, _ := put("let-go")
+	if _, err := store.Hold(ctx, reference(goneRepo, goneDigest)); err != nil {
+		t.Fatal(err)
+	}
+	if err := store.LetGo(ctx, reference(goneRepo, goneDigest)); err != nil {
+		t.Fatalf("letting go of a held archive: %v", err)
+	}
+
+	collected, err := exec.CommandContext(ctx, "docker", "exec", container,
+		"registry", "garbage-collect", "/etc/docker/registry/config.yml").CombinedOutput()
+	if err != nil {
+		t.Fatalf("the collector failed: %v\n%s", err, collected)
+	}
+	t.Logf("the collector said:\n%s", lastLines(string(collected), 12))
+
+	// What is asserted is the bytes on the store's disk, not what the running server answers: the
+	// server caches blob descriptors in memory and can answer for a blob the collector removed.
+	onDisk := func(digest string) bool {
+		hex := strings.TrimPrefix(digest, "sha256:")
+		path := "/var/lib/registry/docker/registry/v2/blobs/sha256/" + hex[:2] + "/" + hex + "/data"
+		return exec.CommandContext(ctx, "docker", "exec", container, "test", "-f", path).Run() == nil
+	}
+	if !onDisk(heldDigest) {
+		t.Error("the collector took an archive published held")
+	}
+	if !onDisk(backDigest) {
+		t.Error("the collector took an archive the sweep backfilled a holder for")
+	}
+	if onDisk(bareDigest) {
+		t.Error("the collector kept a bare archive — then the holders prove nothing, and this test is wrong")
+	}
+	if onDisk(goneDigest) {
+		t.Error("the collector kept an archive the sweep let go of: its holder outlived its link")
+	}
+	// And what survived is still fetched exactly as machines fetch it: the blob, by digest.
+	for repository, want := range map[string][]byte{heldRepo: heldBody, backRepo: backBody} {
+		digest := digestOf(want)
+		response, err := http.Get(catalogue.Routed(reference(repository, digest), address))
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, _ := io.ReadAll(response.Body)
+		response.Body.Close()
+		if response.StatusCode != http.StatusOK || !bytes.Equal(got, want) {
+			t.Errorf("%s answered %s with %q after collection", repository, response.Status, got)
+		}
+	}
+}
+
+func lastLines(s string, n int) string {
+	lines := strings.Split(strings.TrimSpace(s), "\n")
+	if len(lines) > n {
+		lines = lines[len(lines)-n:]
+	}
+	return strings.Join(lines, "\n")
+}

internal/artifacts/store.go

@@ -1,7 +1,8 @@
 // Package artifacts speaks to the mesh's artifact store over its own door.
 //
 // Only what the mesh needs that nothing else does: letting go of something it put there
-// (novox/hq ADR 0189, issue 108). Pushing is the builder's, through the container runtime; reading
+// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's
+// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading
 // is every machine's, through its runtime. This is the one operation that belongs to the thing
 // holding the records, because it is the only one that is a decision rather than a transfer.
 package artifacts
@@ -12,7 +13,6 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
-	"time"
 
 	"github.com/novox/mesh-controller/internal/catalogue"
 )
@@ -43,6 +43,9 @@ var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
 // an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses,
 // and composes the address here at the moment of use.
 //
+// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq
+// issue 253); an image in one.
+//
 // Returns Gone when the store answers that it does not have it. That is not a failure: the sweep
 // wants the artifact absent, and it is. It is distinguished from success only so a caller can say
 // which of the two happened.
@@ -66,17 +69,24 @@ func (s Store) LetGo(ctx context.Context, reference string) error {
 	if err != nil {
 		return err
 	}
-	url := "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
+	if kind == "blobs" {
+		// **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left
+		// naming the blob would keep its bytes through every collection while the record said
+		// collected. Gone here means the store has no such blob, so there is nothing to let go.
+		if err := s.letGoOfHolder(ctx, repository, digest); err != nil {
+			return err
+		}
+	}
+	return s.remove(ctx, s.url(repository, kind, digest), reference)
+}
 
+// remove asks the store to delete what is at url. Gone when it has no such thing.
+func (s Store) remove(ctx context.Context, url, what string) error {
 	request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
 	if err != nil {
 		return err
 	}
-	client := s.HTTP
-	if client == nil {
-		client = &http.Client{Timeout: 30 * time.Second}
-	}
-	response, err := client.Do(request)
+	response, err := s.client().Do(request)
 	if err != nil {
 		return err
 	}
@@ -92,9 +102,9 @@ func (s Store) LetGo(ctx context.Context, reference string) error {
 		return fmt.Errorf(
 			"the artifact store refuses deletion: its server was started without it enabled "+
 				"(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+
-				"module is applied again (novox/hq ADR 0189). Asking about %s", reference)
+				"module is applied again (novox/hq ADR 0189). Asking about %s", what)
 	default:
-		return fmt.Errorf("the artifact store answered %s for %s", response.Status, reference)
+		return fmt.Errorf("the artifact store answered %s for %s", response.Status, what)
 	}
 }
 

internal/artifacts/store_test.go

@@ -20,6 +20,17 @@ func fakeStore(t *testing.T, answer int) (Store, *[]string) {
 	t.Helper()
 	var asked []string
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") {
+			// An archive's size, asked so its holder can be named (novox/hq issue 253). A store
+			// that does not have the thing does not have its blob either.
+			if answer == http.StatusNotFound {
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+			w.Header().Set("Content-Length", "7")
+			w.WriteHeader(http.StatusOK)
+			return
+		}
 		if r.Method != http.MethodDelete {
 			t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path)
 		}
@@ -45,8 +56,14 @@ func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) {
 	if err := store.LetGo(ctx, archive); err != nil {
 		t.Fatal(err)
 	}
-	want := []string{"/v2/web/app/manifests/sha256:abc123", "/v2/web/config/blobs/sha256:def456"}
-	if len(*asked) != 2 || (*asked)[0] != want[0] || (*asked)[1] != want[1] {
+	// The archive's holder goes first, then the archive (novox/hq issue 253).
+	_, holder := Holder("sha256:def456", 7)
+	want := []string{
+		"/v2/web/app/manifests/sha256:abc123",
+		"/v2/web/config/manifests/" + holder,
+		"/v2/web/config/blobs/sha256:def456",
+	}
+	if strings.Join(*asked, " ") != strings.Join(want, " ") {
 		t.Fatalf("the store was asked %v; want %v", *asked, want)
 	}
 }