mesh-catalog/modules/distribution/internal/holder/holder_test.go

package holder

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"reflect"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
)

type memObjects map[string][]byte

func (m memObjects) Put(n string, d []byte) error { m[n] = append([]byte(nil), d...); return nil }
func (m memObjects) Get(n string) ([]byte, error) {
	if d, ok := m[n]; ok {
		return d, nil
	}
	return nil, errors.New("no object " + n)
}
func (m memObjects) Delete(n string) error { delete(m, n); return nil }
func (m memObjects) Used() (uint64, error) {
	var u uint64
	for _, d := range m {
		u += uint64(len(d))
	}
	return u, nil
}

type memTransfers map[string]Transfer

func (m memTransfers) Get(id string) (Transfer, bool, error) { t, ok := m[id]; return t, ok, nil }
func (m memTransfers) Put(t Transfer) error                  { m[t.ID] = t; return nil }
func (m memTransfers) Delete(id string) error                { delete(m, id); return nil }
func (m memTransfers) List() ([]Transfer, error) {
	var out []Transfer
	for _, t := range m {
		out = append(out, t)
	}
	return out, nil
}

type fakeBuilder struct {
	ok  bool
	err error
}

func (b fakeBuilder) Recorded(string, string, string) (bool, string, error) {
	return b.ok, "not numbered", b.err
}

type fakeStore struct {
	has map[string]bool
	put []map[string][]byte
}

func (s *fakeStore) Has(_, d string) (bool, error) { return s.has[d], nil }
func (s *fakeStore) Put(t PublishRequest, parts map[string][]byte) (string, error) {
	s.put = append(s.put, parts)
	return t.Name + "@" + t.Digest, nil
}
func (s *fakeStore) Held(string, string, string) (string, bool, error) { return "", false, nil }

type said struct {
	seat, event string
	body        Event
}

type bed struct {
	h        *Holder
	v        *Verbs
	store    *fakeStore
	said     []said
	findings map[string]bool
	now      time.Time
	cat      *fakeCatalog
}

type fakeCatalog struct {
	tags      map[string]string // name:version -> digest
	removed   []string
	mirrorErr error
	tagErr    error
	indexes   map[string][]string // name@digest -> the platform manifests it names
}

func (c *fakeCatalog) Versions(name string) ([]string, error) {
	var out []string
	for k := range c.tags {
		if n, v, _ := strings.Cut(k, ":"); n == name {
			out = append(out, v)
		}
	}
	return out, nil
}
func (c *fakeCatalog) Digest(name, version string) (string, error) {
	return c.tags[name+":"+version], nil
}

// Remove deletes by the digest, as the store does: every tag naming the digest goes with it.
func (c *fakeCatalog) Remove(name, version string) error {
	c.removed = append(c.removed, name+":"+version)
	d := c.tags[name+":"+version]
	for k, v := range c.tags {
		if n, _, _ := strings.Cut(k, ":"); n == name && v == d {
			delete(c.tags, k)
		}
	}
	return nil
}

// RemoveTag answers as OCI.RemoveTag over registry 2.8.3 does (fakeRegistry's DELETE by tag): the store cannot
// remove a tag alone, and nothing goes. tagErr is any other failure of the store.
func (c *fakeCatalog) RemoveTag(name, version string) error {
	if c.tagErr != nil {
		return c.tagErr
	}
	return fmt.Errorf("%w: DELETE of %s:%s by its tag answered 400 DIGEST_INVALID", ErrNoTagAlone, name, version)
}
func (c *fakeCatalog) RemoveDigest(name, digest string) error {
	c.removed = append(c.removed, name+"@"+digest)
	return nil
}
func (c *fakeCatalog) Platforms(name, digest string) ([]string, error) {
	return c.indexes[name+"@"+digest], nil
}
func (c *fakeCatalog) Mirror(source, digest string) (string, error) {
	if c.mirrorErr != nil {
		return "", c.mirrorErr
	}
	src, _ := ParseSource(source)
	return c.Reference(src.Name(), digest), nil
}
func (c *fakeCatalog) Reference(name, digest string) string {
	return name + "@" + digest
}

type memMirrors map[string]MirrorRequest

func (m memMirrors) Ask(k string, r MirrorRequest) error {
	if _, ok := m[k]; !ok {
		m[k] = r
	}
	return nil
}
func (m memMirrors) Claim(k string, r MirrorRequest, rev uint64) (bool, error) {
	if cur, ok := m[k]; !ok || mirrorRev(cur) != rev {
		return false, nil
	}
	m[k] = r
	return true, nil
}
func (m memMirrors) Delete(k string) error { delete(m, k); return nil }
func (m memMirrors) List() (map[string]MirrorRequest, error) {
	out := map[string]MirrorRequest{}
	for k, r := range m {
		out[k] = r
	}
	return out, nil
}
func (m memMirrors) Put(k string, r MirrorRequest) error { m[k] = r; return nil }

// mirrorRev is a kept ask's revision in memMirrors: 1 unclaimed, 2 once claimed.
func mirrorRev(r MirrorRequest) uint64 {
	if r.Claim == "" {
		return 1
	}
	return 2
}

type memHolds map[string]Hold

func (m memHolds) Get(k string) (Hold, bool, error) { h, ok := m[k]; return h, ok, nil }
func (m memHolds) Put(h Hold) error                 { m[h.Key()] = h; return nil }
func (m memHolds) Delete(k string) error            { delete(m, k); return nil }
func (m memHolds) List() ([]Hold, error) {
	var out []Hold
	for _, h := range m {
		out = append(out, h)
	}
	return out, nil
}

func newBed(builderOK bool) *bed {
	b := &bed{store: &fakeStore{has: map[string]bool{}}, findings: map[string]bool{}, now: time.Unix(1000, 0), cat: &fakeCatalog{tags: map[string]string{}}}
	b.h = &Holder{Callers: DefaultCallers(), Objects: memObjects{}, Transfers: memTransfers{}, Builder: fakeBuilder{ok: builderOK},
		Store: b.store, Now: func() time.Time { return b.now },
		Say: func(seat, event string, body Event) error {
			b.said = append(b.said, said{seat, event, body})
			return nil
		},
		Finding: func(kind, subject, _ string, raised bool) { b.findings[kind+" "+subject] = raised }}
	b.v = &Verbs{Holder: b.h, Catalog: b.cat, Holds: memHolds{}, Mirrors: memMirrors{}}
	return b
}

// capOf is a binding's cap of n bytes.
func capOf(n uint64) func() (uint64, error) { return func() (uint64, error) { return n, nil } }

func keys[V any](m map[string]V) []string {
	var out []string
	for k := range m {
		out = append(out, k)
	}
	return out
}

func digestOf(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) }

const builder = "ace.mesh-build-go"

// TestTheHolderOfTheTwoImageSeats holds design 51's holder cases at the unit level (CAT-D1, D2, D3, D3b, D4, D4b, D5):
// one table or case per rule, the caller list and the store front's.
func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
	blob := []byte(strings.Repeat("a", 3000))
	part := Part{Digest: digestOf(blob), Size: int64(len(blob))}
	req := func(id string) PublishRequest {
		return PublishRequest{Transfer: id, Name: "m/program", Version: "0.1.4", Kind: KindBundle, Digest: part.Digest, Parts: []Part{part}}
	}

	t.Run("a transfer in parts, cut and resumed, hashed, numbered by the builder, published", func(t *testing.T) {
		b := newBed(true)
		a, err := b.h.Publish(ArtifactSeat, builder, req("t1"))
		if err != nil || !reflect.DeepEqual(a.Missing, []string{part.Digest}) {
			t.Fatalf("publish = %+v, %v", a, err)
		}
		if _, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t1", Digest: part.Digest, Sequence: 0, Data: blob[:1000]}); err != nil {
			t.Fatal(err)
		}
		// Cut: the build module opens again and is told what is still missing; its next chunk goes on from there.
		if a, _ := b.h.Publish(ArtifactSeat, builder, req("t1")); !reflect.DeepEqual(a.Missing, []string{part.Digest}) {
			t.Errorf("resumed publish = %+v", a)
		}
		if _, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t1", Digest: part.Digest, Sequence: 0, Data: blob[:1000]}); err == nil {
			t.Error("a chunk sent twice was taken: the transfer goes on from the chunks held")
		}
		a, err = b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t1", Digest: part.Digest, Sequence: 1, Last: true, Data: blob[1000:]})
		if err != nil || !a.Ended || len(b.said) != 1 || b.said[0].event != EventPublished || b.said[0].body.Reference == "" {
			t.Fatalf("last chunk = %+v, %v; said %+v", a, err, b.said)
		}
		// The reference's registry part is the seat's name, as a node-engine reads it back (ADR 0605 §9); nothing set on
		// the process names a reach.
		if seat, path, ok := catalogue.InRegistrySeat(b.said[0].body.Reference); !ok || seat != ArtifactSeat || path != "m/program@"+part.Digest {
			t.Errorf("the published reference %q is not the seat's form", b.said[0].body.Reference)
		}
		if len(b.h.Objects.(memObjects)) != 0 || len(b.h.Transfers.(memTransfers)) != 0 {
			t.Error("the transfer's chunks or key stayed after its end")
		}
	})
	t.Run("refusals", func(t *testing.T) {
		for _, c := range []struct {
			name string
			do   func(b *bed) error
			want string
		}{
			{"a caller not on the list", func(b *bed) error { _, err := b.h.Publish(ArtifactSeat, "ace.mesh-provisioner", req("t")); return err }, "not on the holder's caller list"},
			{"an unproven caller", func(b *bed) error { _, err := b.h.Publish(ArtifactSeat, Unproven, req("t")); return err }, "no caller the tool runner proved"},
			{"a part from another caller than the opener", func(b *bed) error {
				_, _ = b.h.Publish(ArtifactSeat, builder, req("t"))
				_, err := b.h.PublishPart(ArtifactSeat, "ace.mesh-build-node", PartRequest{Transfer: "t", Digest: part.Digest, Last: true, Data: blob})
				return err
			}, "opened by another caller"},
			{"a part whose bytes do not match its digest", func(b *bed) error {
				_, _ = b.h.Publish(ArtifactSeat, builder, req("t"))
				_, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t", Digest: part.Digest, Last: true, Data: []byte("other")})
				return err
			}, "hash to"},
			{"a chunk over 1 MiB", func(b *bed) error {
				_, _ = b.h.Publish(ArtifactSeat, builder, req("t"))
				_, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t", Digest: part.Digest, Data: make([]byte, MaxPart+1)})
				return err
			}, "over the"},
			{"a new transfer at the cap", func(b *bed) error {
				b.h.Cap = capOf(100)
				_, err := b.h.Publish(ArtifactSeat, builder, req("t"))
				return err
			}, "at their cap"},
			{"a version the builder did not number", func(b *bed) error {
				b.h.Builder = fakeBuilder{ok: false}
				b.store.has[part.Digest] = true // held already: the publish ends at once
				_, err := b.h.Publish(ArtifactSeat, builder, req("t"))
				return err
			}, "the builder did not number it"},
			{"an image on the artifact seat", func(b *bed) error {
				r := req("t")
				r.Kind = KindImage
				_, err := b.h.Publish(ArtifactSeat, builder, r)
				return err
			}, "holds bundles and archives"},
			{"remove of a version with no tag", func(b *bed) error {
				return b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/program", Version: "0.1.4", Reason: "branch ended"})
			}, "has no tag"},
			{"a retention removal from another caller than the sweep", func(b *bed) error {
				return b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/program", Digest: part.Digest, Reason: ReasonRetention})
			}, "artifact sweep's alone"},
			{"a retention removal of a held archive", func(b *bed) error {
				_ = b.v.HoldOne(ArtifactSeat, "novox.artifact-store-tools", HoldRequest{Name: "m/a", Kind: KindArchive, Digest: part.Digest})
				return b.v.Remove(ArtifactSeat, "novox.artifact-store-tools", RemoveRequest{Name: "m/a", Digest: part.Digest, Reason: ReasonRetention})
			}, "is held"},
			{"a hold of an image", func(b *bed) error {
				return b.v.HoldOne(ImageSeat, "novox.artifact-store-tools", HoldRequest{Name: "m/i", Kind: KindImage, Digest: part.Digest})
			}, "never held"},
			{"a hold from another caller", func(b *bed) error {
				return b.v.HoldOne(ArtifactSeat, "novox.mesh-build-go", HoldRequest{Name: "m/a", Kind: KindArchive, Digest: part.Digest})
			}, "not on the holder's caller list"},
			{"a release from a caller not on the list", func(b *bed) error {
				_, err := b.v.Release(ArtifactSeat, "novox.mesh-build-go", HoldRequest{Name: "m/a", Digest: part.Digest})
				return err
			}, "not on the holder's caller list"},
			{"a release in distribution's own name", func(b *bed) error {
				_, err := b.v.Release(ArtifactSeat, "novox.distribution", HoldRequest{Name: "m/a", Digest: part.Digest})
				return err
			}, "in its own name"},
			{"a mirror by no digest", func(b *bed) error {
				_, err := b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "docker.io/library/nats:2"})
				return err
			}, "pinned digest"},
		} {
			b := newBed(true)
			if err := c.do(b); err == nil || !strings.Contains(err.Error(), c.want) {
				t.Errorf("%s: err = %v, want one saying %q", c.name, err, c.want)
			}
		}
		if _, err := ParseCallers(`{"release":["distribution"]}`); err == nil {
			t.Error("a caller list putting distribution on release's list is taken")
		}
	})
	t.Run("only the missing layers are sent, and an idle transfer is dropped", func(t *testing.T) {
		b := newBed(true)
		other := []byte("layer two")
		r := req("t2")
		r.Kind, r.Parts = KindImage, []Part{part, {Digest: digestOf(other), Size: int64(len(other))}}
		b.store.has[part.Digest] = true
		a, err := b.h.Publish(ImageSeat, builder, r)
		if err != nil || !reflect.DeepEqual(a.Missing, []string{digestOf(other)}) {
			t.Fatalf("publish = %+v, %v; want only the layer the store lacks", a, err)
		}
		b.now = b.now.Add(TransferIdle)
		if dropped, _ := b.h.DropIdle(); !reflect.DeepEqual(dropped, []string{"t2"}) {
			t.Errorf("DropIdle = %v, want t2 after 24 hours with no part", dropped)
		}
		// Running review F2, ADR 0611 §3: the drop follows each transfer's idle key. The open arms it for 24 hours; its
		// end re-arms a transfer that took a part since for the rest, and drops one idle 24 hours, its chunks with it.
		b = newBed(true)
		armed := map[string]time.Duration{}
		b.h.Idle = func(id string, ttl time.Duration) error { armed[id] = ttl; return nil }
		if _, err := b.h.Publish(ArtifactSeat, builder, req("t4")); err != nil || armed["t4"] != TransferIdle {
			t.Fatalf("publish = %v, armed %v; want t4's idle key for %s", err, armed, TransferIdle)
		}
		b.now = b.now.Add(time.Hour)
		_, _ = b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t4", Digest: part.Digest, Data: blob[:100]})
		b.now = b.now.Add(TransferIdle - time.Hour)
		if dropped, err := b.h.Expired("t4"); err != nil || dropped || armed["t4"] != time.Hour {
			t.Errorf("the key's end an hour after a part: dropped %v, %v, armed %v; want t4 kept and armed for 1h", dropped, err, armed)
		}
		b.now = b.now.Add(time.Hour)
		if dropped, err := b.h.Expired("t4"); err != nil || !dropped {
			t.Errorf("the key's end 24 hours after the last part: dropped %v, %v; want t4 dropped", dropped, err)
		}
		if _, found, _ := b.h.Transfers.Get("t4"); found || len(b.h.Objects.(memObjects)) != 0 {
			t.Errorf("t4 after its drop: kept %v, objects %v; want both gone", found, b.h.Objects)
		}
		if dropped, err := b.h.Expired("t4"); err != nil || dropped {
			t.Errorf("the end of a key whose transfer is gone: %v, %v; want nothing done", dropped, err)
		}
		b.h.Idle = func(string, time.Duration) error { return errors.New("bus down") }
		if _, err := b.h.Publish(ArtifactSeat, builder, req("t5")); err == nil || !strings.Contains(err.Error(), "idle bound") {
			t.Errorf("a publish whose idle key is not written = %v; want it failed loudly", err)
		}
	})
	t.Run("incoming's finding at 80 %, cleared below", func(t *testing.T) {
		b := newBed(true)
		b.h.Cap = capOf(3500)
		_, _ = b.h.Publish(ArtifactSeat, builder, req("t3"))
		_, _ = b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t3", Digest: part.Digest, Data: blob[:2800]})
		if !b.findings["incoming-full incoming"] {
			t.Error("2800 of 3500 bytes wrote no finding")
		}
		_, _ = b.h.Drop("t3")
		if b.findings["incoming-full incoming"] {
			t.Error("the finding stayed after the transfer was dropped")
		}
		// A refusal for want of room stays loud on incoming until the holder takes a publish again.
		b.h.Cap = capOf(100)
		_, _ = b.h.Publish(ArtifactSeat, builder, req("t4"))
		if !b.findings["publish-refused incoming"] {
			t.Errorf("a refusal at the cap raised %v, want publish-refused on incoming", b.findings)
		}
		b.h.Cap = capOf(1 << 20)
		b.store.has[part.Digest] = true
		if _, err := b.h.Publish(ArtifactSeat, builder, req("t5")); err != nil || b.findings["publish-refused incoming"] {
			t.Errorf("the next publish taken = %v, findings %v: want the refusal at the cap cleared", err, b.findings)
		}
	})
	t.Run("the sweep's removal, hold and release", func(t *testing.T) {
		b := newBed(true)
		sweep := "novox.artifact-store-tools"
		if err := b.v.Remove(ArtifactSeat, sweep, RemoveRequest{Name: "m/a", Digest: part.Digest, Kind: KindArchive, Reason: ReasonRetention}); err != nil {
			t.Fatal(err)
		}
		if err := b.v.HoldOne(ArtifactSeat, sweep, HoldRequest{Name: "m/b", Kind: KindArchive, Digest: part.Digest}); err != nil {
			t.Fatal(err)
		}
		if released, err := b.v.Release(ArtifactSeat, "person.jochen", HoldRequest{Name: "m/b", Digest: part.Digest, Why: "by hand"}); err != nil || !released {
			t.Errorf("the operator's session releasing through registry_holds: %v, %v", released, err)
		}
		// A release of what is not held answers released false, never an error, and says nothing (the seat's definition).
		if released, err := b.v.Release(ArtifactSeat, sweep, HoldRequest{Name: "m/b", Digest: part.Digest, Why: "eligible"}); err != nil || released {
			t.Errorf("a release of what is not held: %v, %v; want false and no error", released, err)
		}
		var events []string
		for _, s := range b.said {
			events = append(events, s.event+" "+s.body.By)
		}
		if want := []string{"removed " + sweep, "held " + sweep, "released person.jochen"}; !reflect.DeepEqual(events, want) {
			t.Errorf("said %v, want %v", events, want)
		}
		b.cat.tags["m/p:0.1.5-feat-x.1"] = part.Digest
		if err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/p", Version: "0.1.5-feat-x.1", Reason: "branch merged"}); err != nil || b.cat.tags["m/p:0.1.5-feat-x.1"] != "" {
			t.Errorf("the builder's removal of a tagged version: %v", err)
		}
		// Security review F1 on registry 2.8.3: a reproducible build gives a branch's tagged version and the trunk's
		// numbered one one digest; the store cannot remove the tag alone, so nothing goes, the numbered one stays
		// pullable, and the removal is said removed with that reason: no refusal, so the builder raises no retry.
		b.cat.tags["m/p:0.1.6-feat-y.1"], b.cat.tags["m/p:0.1.6"] = part.Digest, part.Digest
		if err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/p", Version: "0.1.6-feat-y.1", Reason: "branch merged"}); err != nil {
			t.Fatalf("the removal of a tagged version sharing its digest: %v; want no error", err)
		}
		if b.cat.tags["m/p:0.1.6-feat-y.1"] != part.Digest || b.cat.tags["m/p:0.1.6"] != part.Digest {
			t.Errorf("tags after the tagged version's removal: %v; want both kept at %s", b.cat.tags, part.Digest)
		}
		if last := b.said[len(b.said)-1]; last.event != EventRemoved || last.body.Reason != "branch merged; the store cannot remove a tag alone, so the tag 0.1.6-feat-y.1 is kept and its bytes stay under 0.1.6" {
			t.Errorf("said %s %q; want removed, saying the tag is kept under 0.1.6", last.event, last.body.Reason)
		}
		// Any other failure of the store removes nothing and is refused naming the shared digest.
		b.cat.tags["m/p:0.1.7-feat-z.1"], b.cat.tags["m/p:0.1.7"] = part.Digest, part.Digest
		b.cat.tagErr = errors.New("the store answered 500")
		said := len(b.said)
		if err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/p", Version: "0.1.7-feat-z.1", Reason: "branch merged"}); err == nil || !strings.Contains(err.Error(), "shares its digest") || b.cat.tags["m/p:0.1.7"] != part.Digest || len(b.said) != said {
			t.Errorf("a store failing otherwise: %v, tags %v; want refused, 0.1.7 kept, nothing said", err, b.cat.tags)
		}
		b.cat.tagErr = nil
		// The store's own answers, as registry 2.8.3 gives them: DELETE by tag removes nothing (ErrNoTagAlone); by
		// digest the manifest goes with every tag naming it.
		r283 := newFakeRegistry()
		sts := httptest.NewServer(r283)
		defer sts.Close()
		u, _ := url.Parse(sts.URL)
		o := &OCI{Base: u}
		manifest := []byte(`{"schemaVersion":2}`)
		r283.manifests["m/p:0.1.8-feat-w.1"], r283.manifests["m/p:0.1.8"] = manifest, manifest
		if err := o.RemoveTag("m/p", "0.1.8-feat-w.1"); !errors.Is(err, ErrNoTagAlone) || r283.manifests["m/p:0.1.8-feat-w.1"] == nil {
			t.Errorf("RemoveTag on 2.8.3 = %v; want ErrNoTagAlone and the tag kept", err)
		}
		if err := o.Remove("m/p", "0.1.8"); err != nil || len(r283.manifests) != 0 {
			t.Errorf("Remove by digest on 2.8.3 = %v, %d manifests left; want both tags gone", err, len(r283.manifests))
		}
	})
	// CAT-AS1: what the sweep did over the store's door is the holder's: a removal for retention of an index removes
	// its platform manifests first, but those a tagged or held index of the repository names; a manifest such an index
	// names is refused; OCI.Platforms reads an index's from the store.
	t.Run("an index's platforms go with it, but those a kept index names", func(t *testing.T) {
		d := func(s string) string { return "sha256:" + strings.Repeat(s, 64) }
		sweep := "novox.artifact-store-tools"
		name := "upstream/docker.io/library/alpine"
		for _, c := range []struct {
			case_, remove string
			tagged, held  string // the kept index, by a tag or a hold; none when empty
			removed       []string
			refused       string
		}{
			{case_: "no kept index", remove: d("1"), removed: []string{d("a"), d("b"), d("1")}},
			{case_: "a tagged index shares a platform", remove: d("1"), tagged: d("2"), removed: []string{d("a"), d("1")}},
			{case_: "a held index shares a platform", remove: d("1"), held: d("2"), removed: []string{d("a"), d("1")}},
			{case_: "a platform of a held index", remove: d("b"), held: d("2"), refused: "is kept by the index " + d("2")},
			{case_: "an image that is no index", remove: d("c"), removed: []string{d("c")}},
		} {
			b := newBed(true)
			b.cat.indexes = map[string][]string{name + "@" + d("1"): {d("a"), d("b")}, name + "@" + d("2"): {d("b")}}
			if c.tagged != "" {
				b.cat.tags[name+":3.20"] = c.tagged
			}
			if c.held != "" {
				if err := b.v.HoldOne(ImageSeat, sweep, HoldRequest{Name: name, Digest: c.held}); err != nil {
					t.Fatal(err)
				}
			}
			err := b.v.Remove(ImageSeat, sweep, RemoveRequest{Name: name, Digest: c.remove, Reason: ReasonRetention})
			var removed []string
			for _, r := range b.cat.removed {
				removed = append(removed, strings.TrimPrefix(r, name+"@"))
			}
			if c.refused != "" {
				if err == nil || !strings.Contains(err.Error(), c.refused) || len(removed) != 0 {
					t.Errorf("%s: %v, removed %v; want refused saying %q and nothing removed", c.case_, err, removed, c.refused)
				}
				continue
			}
			if err != nil || !reflect.DeepEqual(removed, c.removed) {
				t.Errorf("%s: %v, removed %v, want %v", c.case_, err, removed, c.removed)
			}
		}
		store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			switch r.URL.Path {
			case "/v2/" + name + "/manifests/" + d("1"):
				_, _ = w.Write([]byte(`{"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"digest":"` + d("a") + `"},{"digest":"` + d("b") + `"}]}`))
			case "/v2/" + name + "/manifests/" + d("c"):
				_, _ = w.Write([]byte(`{"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`))
			default:
				w.WriteHeader(http.StatusNotFound)
			}
		}))
		defer store.Close()
		u, _ := url.Parse(store.URL)
		o := &OCI{Base: u}
		for digest, want := range map[string][]string{d("1"): {d("a"), d("b")}, d("c"): nil, d("9"): nil} {
			if got, err := o.Platforms(name, digest); err != nil || !reflect.DeepEqual(got, want) {
				t.Errorf("Platforms(%s) = %v, %v, want %v", digest[:10], got, err, want)
			}
		}
	})
	t.Run("the store's port: reads open, writes from distribution alone, and the direct-write window", func(t *testing.T) {
		store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) }))
		defer store.Close()
		u, _ := url.Parse(store.URL)
		var open atomic.Bool
		f := &Front{Store: u, Window: &open, Published: func(name, tag string) (bool, error) { return name == "m/i" && tag == "0.1.3", nil }}
		auth := ""
		do := func(method, path string) int {
			rec := httptest.NewRecorder()
			req := httptest.NewRequest(method, path, nil)
			if auth != "" {
				req.Header.Set("Authorization", "Bearer "+auth)
			}
			f.ServeHTTP(rec, req)
			return rec.Code
		}
		for _, c := range []struct {
			open         bool
			method, path string
			want         int
		}{
			{false, http.MethodGet, "/v2/m/i/manifests/0.1.3", http.StatusCreated},
			{false, http.MethodHead, "/v2/m/i/blobs/sha256:aa", http.StatusCreated},
			{false, http.MethodPut, "/v2/m/i/manifests/0.1.9", http.StatusForbidden},
			{false, http.MethodPost, "/v2/m/i/blobs/uploads/", http.StatusForbidden},
			{true, http.MethodPost, "/v2/m/i/blobs/uploads/", http.StatusCreated},
			{true, http.MethodPut, "/v2/m/i/manifests/0.1.9", http.StatusCreated},
			{true, http.MethodPut, "/v2/m/i/manifests/0.1.3", http.StatusForbidden}, // published through the seat
		} {
			open.Store(c.open)
			if got := do(c.method, c.path); got != c.want {
				t.Errorf("window open %v: %s %s = %d, want %d", c.open, c.method, c.path, got, c.want)
			}
		}
		// No credential opens the front (the lead's ENG-5 ruling): a write carrying a token, with the window closed, is
		// refused as any other.
		open.Store(false)
		auth = "s3cret"
		if got := do(http.MethodPut, "/v2/gitea/program/manifests/0.1.0"); got != http.StatusForbidden {
			t.Errorf("a write with a bearer token and the window closed = %d, want %d", got, http.StatusForbidden)
		}
	})
	t.Run("mirror copies an outside image by its pinned digest, every platform, only the parts the store lacks", func(t *testing.T) {
		up, store := newFakeRegistry(), newFakeRegistry()
		ups, sts := httptest.NewServer(up), httptest.NewServer(store)
		defer ups.Close()
		defer sts.Close()
		up.token, up.realm = "anon", ups.URL+"/token"
		host := strings.TrimPrefix(ups.URL, "http://")
		repo := "library/nats"
		layer, config := []byte(strings.Repeat("l", 5000)), []byte(`{"os":"linux"}`)
		up.blobs[repo+"@"+digestOf(layer)], up.blobs[repo+"@"+digestOf(config)] = layer, config
		img, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": mediaManifest,
			"config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": digestOf(config), "size": len(config)},
			"layers": []map[string]any{{"mediaType": mediaLayer, "digest": digestOf(layer), "size": len(layer)}}})
		up.manifests[repo+":"+digestOf(img)], up.types[repo+":"+digestOf(img)] = img, mediaManifest
		idx, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json",
			"manifests": []map[string]any{{"mediaType": mediaManifest, "digest": digestOf(img), "size": len(img)}}})
		up.manifests[repo+":"+digestOf(idx)] = idx
		base, _ := url.Parse(sts.URL)
		o := &OCI{Base: base, Upstream: &Upstream{Scheme: "http"}}
		name := "upstream/" + host + "/" + repo
		store.blobs[name+"@"+digestOf(config)] = config // held already: not fetched again
		ref, err := o.Mirror(host+"/"+repo+":2", digestOf(idx))
		if err != nil {
			t.Fatal(err)
		}
		if want := name + "@" + digestOf(idx); ref != want {
			t.Errorf("reference %s, want %s", ref, want)
		}
		if store.manifests[name+":"+digestOf(idx)] == nil || store.manifests[name+":"+digestOf(img)] == nil || store.blobs[name+"@"+digestOf(layer)] == nil {
			t.Errorf("the store holds %v and %v, want the index, its image and the layer", keys(store.manifests), keys(store.blobs))
		}
		if !reflect.DeepEqual(up.gets, []string{digestOf(layer)}) {
			t.Errorf("blobs fetched %v, want only the layer the store lacked", up.gets)
		}
		up.manifests[repo+":"+digestOf(idx)] = []byte(`{"tampered":true}`)
		if _, err := o.Mirror(host+"/"+repo, digestOf(idx)); err != nil {
			t.Errorf("a copy held whole is answered at once, without asking the upstream: %v", err)
		}
		delete(store.manifests, name+":"+digestOf(idx))
		if _, err := o.Mirror(host+"/"+repo, digestOf(idx)); err == nil || !strings.Contains(err.Error(), "does not hash") {
			t.Errorf("a manifest whose bytes are not its digest: %v", err)
		}
		for in, want := range map[string]string{"nats:2": "upstream/docker.io/library/nats", "ghcr.io/o/p@sha256:ab": "upstream/ghcr.io/o/p", "localhost:5000/x": "upstream/localhost:5000/x"} {
			if s, _ := ParseSource(in); s.Name() != want {
				t.Errorf("ParseSource(%q) = %s, want %s", in, s.Name(), want)
			}
		}
	})
	t.Run("a late publish names a held bundle with no size: tagged at the store's size, nothing sent", func(t *testing.T) {
		store := newFakeRegistry()
		sts := httptest.NewServer(store)
		defer sts.Close()
		base, _ := url.Parse(sts.URL)
		o := &OCI{Base: base}
		blob := []byte(strings.Repeat("b", 4321))
		store.blobs["m/tools@"+digestOf(blob)] = blob
		ref, err := o.Put(PublishRequest{Name: "m/tools", Version: "0.1.4", Kind: KindBundle, Digest: digestOf(blob),
			Parts: []Part{{Digest: digestOf(blob)}}}, map[string][]byte{})
		if err != nil || ref != "m/tools@"+digestOf(blob) {
			t.Fatalf("Put = %q, %v", ref, err)
		}
		var man struct {
			Layers []struct {
				Size int64 `json:"size"`
			} `json:"layers"`
		}
		if err := json.Unmarshal(store.manifests["m/tools:0.1.4"], &man); err != nil || len(man.Layers) != 1 || man.Layers[0].Size != int64(len(blob)) {
			t.Errorf("the version's manifest %s, want one layer of %d bytes", store.manifests["m/tools:0.1.4"], len(blob))
		}
		if _, err := o.Put(PublishRequest{Name: "m/tools", Version: "0.1.5", Kind: KindBundle, Digest: "sha256:" + strings.Repeat("e", 64),
			Parts: []Part{{Digest: "sha256:" + strings.Repeat("e", 64)}}}, map[string][]byte{}); err == nil {
			t.Error("a digest the store does not hold was tagged")
		}
	})
	t.Run("mirror answers at once and says the copy's end on the seat's events", func(t *testing.T) {
		b := newBed(true)
		d := "sha256:" + strings.Repeat("c", 64)
		ref, err := b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "docker.io/library/nats:2", Digest: d})
		want := "container-image-registry/upstream/docker.io/library/nats@" + d
		key := MirrorsKey("upstream/docker.io/library/nats", d)
		asked := b.v.Mirrors.(memMirrors)
		if err != nil || ref != want || len(b.said) != 0 || asked[key].By != "novox.mesh-build-image" {
			t.Fatalf("mirror = %q, %v, said %v, asks %v: want the reference at once, the copy asked, nothing said yet", ref, err, b.said, asked)
		}
		if strings.ContainsAny(key, ":@") {
			t.Errorf("the ask's key %q holds characters a key may not", key)
		}
		if err := b.v.Mirrored(asked[key]); err != nil || len(b.said) != 1 || b.said[0].event != EventPublished ||
			b.said[0].body.Reference != want || b.said[0].body.Kind != KindUpstream || len(asked) != 0 {
			t.Fatalf("the copy's end = %v, said %+v, asks %v: want published with the reference and the ask gone", err, b.said, asked)
		}
		// One copy, said once: two followers on the same ask; the second sees it claimed and passes, and the outcome
		// carries the ask's id as its event id.
		_, _ = b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "nats", Digest: d})
		ask := asked[key]
		copies := len(b.said)
		for _, follower := range []string{"one", "two"} {
			if err := b.v.Take(key, ask, 1, false, follower); err != nil {
				t.Fatalf("Take by %s: %v", follower, err)
			}
		}
		if len(b.said) != copies+1 || ask.ID == "" || b.said[copies].body.ID != ask.ID || len(asked) != 0 {
			t.Fatalf("two followers on one ask said %+v (ask id %q), asks %v: want one copy, said once under the ask's id", b.said[copies:], ask.ID, asked)
		}
		b.said = b.said[:copies]
		// registry_mirrors (seams review note 1): a kept copy is read, asked again with its claim let go, and dropped
		// by hand, which says refused under the ask's id with who and why.
		_, _ = b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "nats", Digest: d})
		asked[key] = MirrorRequest{Source: asked[key].Source, Digest: d, ID: asked[key].ID, Claim: "cut-follower"}
		if now, err := b.v.MirrorsNow(); err != nil || now[key].Claim != "cut-follower" {
			t.Fatalf("MirrorsNow = %v, %v; want the claimed ask", now, err)
		}
		if r, err := b.v.MirrorAgain(key); err != nil || r.Claim != "" || asked[key].Claim != "" {
			t.Errorf("MirrorAgain = %+v, %v (kept %+v); want the claim let go", r, err, asked[key])
		}
		if _, err := b.v.MirrorDrop(key, "person.jochen", ""); err == nil {
			t.Error("a drop with no why is taken")
		}
		if _, err := b.v.MirrorDrop(key, "person.jochen", "upstream gone"); err != nil || len(asked) != 0 ||
			b.said[copies].event != EventRefused || b.said[copies].body.ID == "" || !strings.Contains(b.said[copies].body.Reason, "dropped by hand by person.jochen: upstream gone") {
			t.Errorf("MirrorDrop = %v, said %+v, asks %v; want refused said under the ask's id and the ask gone", err, b.said[copies:], asked)
		}
		b.said = b.said[:copies]
		b.cat.mirrorErr = errors.New("the upstream said 404")
		_, _ = b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "nats", Digest: d})
		if err := b.v.Mirrored(asked[key]); err != nil || b.said[1].event != EventRefused || !strings.Contains(b.said[1].body.Reason, "404") || len(asked) != 0 {
			t.Errorf("a failed copy = %v, said %+v: want refused with why", err, b.said)
		}
	})
	t.Run("the windows", func(t *testing.T) {
		nodes := []NodeRead{
			{Node: "ace", Serves: true, Stated: []string{NewNamesCapability}, NamesArtifact: true, BuildModules: map[string]bool{"mesh-build-go": true}},
			{Node: "g14", NamesArtifact: true, BuildModules: map[string]bool{"mesh-build-go": false}}, // away, old node-engine
			{Node: "novox", Serves: true, Stated: nil, NamesArtifact: false},
		}
		if got := HoldingOldName(nodes, nil); !reflect.DeepEqual(got, []string{"g14"}) {
			t.Errorf("old name held by %v, want g14 (no manifest-reads: it states nothing)", got)
		}
		if got := HoldingDirectWrites(nodes, nil); !reflect.DeepEqual(got, []string{"g14"}) {
			t.Errorf("direct writes held by %v, want g14", got)
		}
		t0 := time.Unix(0, 0)
		all := []string{"ace", "g14", "novox"}
		w, closed, _ := Window{Name: OldNameWindow}.Step(all, []string{"g14"}, t0)
		if closed != nil || !w.Open {
			t.Fatalf("Step = %+v, %+v", w, closed)
		}
		if _, _, overdue := w.Step(all, []string{"g14"}, t0.Add(WindowFindingAfter)); !reflect.DeepEqual(overdue, []string{"g14"}) {
			t.Errorf("seven days: overdue %v", overdue)
		}
		// Security review F3: a read naming no node (an empty node_haves) judges nothing; the window stays open.
		if same, shut, _ := w.Step(nil, nil, t0.Add(time.Minute)); shut != nil || !same.Open || !reflect.DeepEqual(same.Holding, []string{"g14"}) {
			t.Errorf("a read naming no node: %+v, closed %+v; want the window open as it was", same, shut)
		}
		w, _ = w.Drop("g14", "person.jochen", "retired", t0)
		w, closed, _ = w.Step(all, HoldingOldName(nodes, w.Dropped), t0.Add(time.Hour))
		if closed == nil || closed.Window != OldNameWindow || !reflect.DeepEqual(closed.LastHolding, []string{"g14"}) || len(closed.Dropped) != 1 {
			t.Errorf("closing = %+v", closed)
		}
		// Once closed, it stays closed: a node holding it later opens nothing.
		if again, shut, _ := w.Step(all, []string{"g14"}, t0.Add(2*time.Hour)); shut != nil || again.Open {
			t.Errorf("a closed window stepped on a holder: %+v, closed %+v; want it closed, said once", again, shut)
		}
	})
	t.Run("the windows' driver: every node read on each wake, closed once, a paused build agent's same build counted", func(t *testing.T) {
		del := &fakeDelivery{haves: []NodeHave{
			{"ace", "mesh-build-go", "fp-new"}, {"novox", "mesh-build-go", "fp-new"}, {"g14", "mesh-build-go", "fp-old"},
			{"g14", "node", ""}, {"ace", "gitea", "x"}, {"novox", "gitea", "x"},
		}, stated: map[string][]string{"ace": {NewNamesCapability}, "novox": {NewNamesCapability}}}
		st := &memWindows{w: map[string]Window{}}
		st.pub.Learn("mesh-build-go", "fp-new")
		var closed []string
		found := map[string]bool{}
		now := time.Unix(0, 0)
		d := &Driver{Delivery: del, State: st, Callers: func() (Callers, error) { return DefaultCallers(), nil }, Now: func() time.Time { return now },
			Closed:  func(c Closed) error { closed = append(closed, c.Window+" "+c.Seat); return nil },
			Finding: func(kind, subject, _ string, raised bool) { found[kind+" "+subject] = raised }}
		if _, err := d.Wake("start"); err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(st.w[OldNameWindow].Holding, []string{"g14"}) || !reflect.DeepEqual(st.w[DirectWritesWindow].Holding, []string{"g14"}) {
			t.Errorf("windows = %+v, want g14 holding both (away: states nothing; runs the old build)", st.w)
		}
		// The window's record says why g14 holds it: its manifest-reads timed out, counted as stating nothing.
		if why := st.w[OldNameWindow].Why["g14"]; !strings.Contains(why, "did not answer") || !strings.Contains(why, "timeout") {
			t.Errorf("why g14 holds the old name = %q, want the timeout said", why)
		}
		// Security review F3: a wake whose node_haves names no node closes neither window.
		haves := del.haves
		del.haves = nil
		if _, err := d.Wake("delivered"); err != nil || !st.w[OldNameWindow].Open || !st.w[DirectWritesWindow].Open || len(closed) != 0 {
			t.Fatalf("a wake on an empty node_haves: %v, windows %+v, closed %v; want both open, none said", err, st.w, closed)
		}
		del.haves = haves
		now = now.Add(WindowFindingAfter)
		_, _ = d.Wake("delivered")
		if !found[FindingWindowHeld+" "+OldNameWindow+".g14"] {
			t.Errorf("findings %v, want g14 named after seven days", found)
		}
		del.haves[2].FP, del.stated["g14"] = "fp-new", []string{NewNamesCapability}
		if _, err := d.Wake("manifest-reads-changed on g14"); err != nil {
			t.Fatal(err)
		}
		_, _ = d.Wake("delivered")
		if want := []string{OldNameWindow + " " + OldSeat, DirectWritesWindow + " "}; !reflect.DeepEqual(closed, want) {
			t.Errorf("closed %v, want each window said once, %v", closed, want)
		}
		if found[FindingWindowHeld+" "+OldNameWindow+".g14"] {
			t.Error("the finding stays after the window closed")
		}
	})
	t.Run("the health check: versions of each seat through the program's port; down names the program, then clears", func(t *testing.T) {
		front := &downCatalog{fakeCatalog: fakeCatalog{tags: map[string]string{}}, down: true}
		found := map[string]bool{}
		f := func(kind, subject, _ string, raised bool) { found[kind+" "+subject] = raised }
		if _, err := Health(front, f); err == nil || !strings.Contains(err.Error(), "program distribution") || !found[FindingReadSide+" "+ImageSeat] {
			t.Errorf("program stopped: %v, findings %v", err, found)
		}
		front.down = false
		if _, err := Health(front, f); err != nil || found[FindingReadSide+" "+ImageSeat] {
			t.Errorf("program back: %v, findings %v", err, found)
		}
	})
	t.Run("the manifest", func(t *testing.T) {
		raw, err := os.ReadFile("../../module.json")
		if err != nil {
			t.Fatal(err)
		}
		var m struct {
			Provides []struct{ Name string } `json:"provides"`
			Claims   []struct {
				Name   string   `json:"name"`
				Serves []string `json:"serves"`
			} `json:"claims"`
			Resources []map[string]any `json:"resources"`
			Consumes  []string         `json:"consumes"`
		}
		if err := json.Unmarshal(raw, &m); err != nil {
			t.Fatal(err)
		}
		// E15: artifact-store is still offered until the old name closes, so a revert needs no edge.
		if len(m.Provides) != 1 || m.Provides[0].Name != "artifact-store" {
			t.Errorf("provides = %+v, want artifact-store kept", m.Provides)
		}
		claims := map[string][]string{}
		for _, c := range m.Claims {
			claims[c.Name] = c.Serves
		}
		if !reflect.DeepEqual(claims[ImageSeat], VerbsOf(ImageSeat)) || !reflect.DeepEqual(claims[ArtifactSeat], VerbsOf(ArtifactSeat)) {
			t.Errorf("claims = %v, want each seat serving its verbs", claims)
		}
		if _, ok := claims[OldSeat]; !ok {
			t.Error("the old seat mesh-artifact-store is no longer claimed while its window is open")
		}
		if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-view.delivered", "node-engine.manifest-reads-changed"}) {
			t.Errorf("consumes = %v, want delivered and manifest-reads-changed (the windows' wakes)", m.Consumes)
		}
		// E14: the registry container, on the loopback, is listed before the program, so the port is free when it starts.
		store, program := -1, -1
		for i, r := range m.Resources {
			switch r["id"] {
			case "store":
				store = i
				if ports, _ := r["ports"].([]any); len(ports) != 1 || ports[0] != "127.0.0.1:5001:5000" {
					t.Errorf("the store's ports = %v, want the loopback only", r["ports"])
				}
			case "program":
				program = i
				if h, _ := r["health"].(map[string]any); h["kind"] != "tool" || h["tool"] != "registry_health" {
					t.Errorf("the program's health = %v, want the tool registry_health, run outside its process", r["health"])
				}
			}
		}
		if store < 0 || program < 0 || store > program {
			t.Errorf("store at %d, program at %d: want the store listed first", store, program)
		}
		// Running review F2: incoming asks the bus 8 GiB, the most the objects template allows, equal to IncomingCap
		// (said only where no binding is read); transfers keeps for good, the holder dropping an idle one by its key
		// in idle, a state of keys with their own time to live.
		var req struct {
			Requires []struct {
				Name    string         `json:"name"`
				Options map[string]any `json:"options"`
			} `json:"requires"`
		}
		_ = json.Unmarshal(raw, &req)
		opts := map[string]map[string]any{}
		for _, r := range req.Requires {
			opts[r.Name] = r.Options
		}
		if mib, _ := opts[IncomingObjects]["max-mib"].(float64); uint64(mib)<<20 != IncomingCap {
			t.Errorf("incoming asks %v MiB, IncomingCap is %d bytes: want them equal", opts[IncomingObjects]["max-mib"], uint64(IncomingCap))
		}
		if _, ok := opts[TransfersState]["ttl-seconds"]; ok {
			t.Errorf("transfers has a lifetime %v: the bus would drop a transfer's key and leave its parts", opts[TransfersState])
		}
		if opts[IdleState]["key-ttl"] != true {
			t.Errorf("idle's options %v: want key-ttl, each transfer's key ending on its own", opts[IdleState])
		}
	})
}