mesh-catalog/modules/distribution/internal/holder/holder_test.go

package holder

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"reflect"
	"strings"
	"sync/atomic"
	"testing"
	"time"
)

type memObjects map[string][]byte

func (m memObjects) Put(n string, d []byte) error { m[n] = append([]byte(nil), d...); return nil }
func (m memObjects) Get(n string) ([]byte, error) {
	if d, ok := m[n]; ok {
		return d, nil
	}
	return nil, errors.New("no object " + n)
}
func (m memObjects) Delete(n string) error { delete(m, n); return nil }
func (m memObjects) Used() (uint64, error) {
	var u uint64
	for _, d := range m {
		u += uint64(len(d))
	}
	return u, nil
}

type memTransfers map[string]Transfer

func (m memTransfers) Get(id string) (Transfer, bool, error) { t, ok := m[id]; return t, ok, nil }
func (m memTransfers) Put(t Transfer) error                  { m[t.ID] = t; return nil }
func (m memTransfers) Delete(id string) error                { delete(m, id); return nil }
func (m memTransfers) List() ([]Transfer, error) {
	var out []Transfer
	for _, t := range m {
		out = append(out, t)
	}
	return out, nil
}

type fakeBuilder struct {
	ok  bool
	err error
}

func (b fakeBuilder) Recorded(string, string, string) (bool, string, error) {
	return b.ok, "not numbered", b.err
}

type fakeStore struct {
	has map[string]bool
	put []map[string][]byte
}

func (s *fakeStore) Has(_, d string) (bool, error) { return s.has[d], nil }
func (s *fakeStore) Put(t PublishRequest, parts map[string][]byte) (string, error) {
	s.put = append(s.put, parts)
	return "registry.mesh/" + t.Name + "@" + t.Digest, nil
}
func (s *fakeStore) Held(string, string, string) (string, bool, error) { return "", false, nil }

type said struct {
	seat, event string
	body        Event
}

type bed struct {
	h        *Holder
	v        *Verbs
	store    *fakeStore
	said     []said
	findings map[string]bool
	now      time.Time
	cat      *fakeCatalog
}

type fakeCatalog struct {
	tags      map[string]string // name:version -> digest
	removed   []string
	mirrorErr error
}

func (c *fakeCatalog) Versions(name string) ([]string, error) {
	var out []string
	for k := range c.tags {
		if n, v, _ := strings.Cut(k, ":"); n == name {
			out = append(out, v)
		}
	}
	return out, nil
}
func (c *fakeCatalog) Digest(name, version string) (string, error) {
	return c.tags[name+":"+version], nil
}
func (c *fakeCatalog) Remove(name, version string) error {
	c.removed = append(c.removed, name+":"+version)
	delete(c.tags, name+":"+version)
	return nil
}
func (c *fakeCatalog) RemoveDigest(name, digest string) error {
	c.removed = append(c.removed, name+"@"+digest)
	return nil
}
func (c *fakeCatalog) Mirror(source, digest string) (string, error) {
	if c.mirrorErr != nil {
		return "", c.mirrorErr
	}
	src, _ := ParseSource(source)
	return c.Reference(src.Name(), digest), nil
}
func (c *fakeCatalog) Reference(name, digest string) string {
	return "registry.mesh/" + name + "@" + digest
}

type memMirrors map[string]MirrorRequest

func (m memMirrors) Ask(k string, r MirrorRequest) error {
	if _, ok := m[k]; !ok {
		m[k] = r
	}
	return nil
}
func (m memMirrors) Claim(k string, r MirrorRequest, rev uint64) (bool, error) {
	if cur, ok := m[k]; !ok || mirrorRev(cur) != rev {
		return false, nil
	}
	m[k] = r
	return true, nil
}
func (m memMirrors) Delete(k string) error { delete(m, k); return nil }

// mirrorRev is a kept ask's revision in memMirrors: 1 unclaimed, 2 once claimed.
func mirrorRev(r MirrorRequest) uint64 {
	if r.Claim == "" {
		return 1
	}
	return 2
}

type memHolds map[string]Hold

func (m memHolds) Get(k string) (Hold, bool, error) { h, ok := m[k]; return h, ok, nil }
func (m memHolds) Put(h Hold) error                 { m[h.Key()] = h; return nil }
func (m memHolds) Delete(k string) error            { delete(m, k); return nil }
func (m memHolds) List() ([]Hold, error) {
	var out []Hold
	for _, h := range m {
		out = append(out, h)
	}
	return out, nil
}

func newBed(builderOK bool) *bed {
	b := &bed{store: &fakeStore{has: map[string]bool{}}, findings: map[string]bool{}, now: time.Unix(1000, 0), cat: &fakeCatalog{tags: map[string]string{}}}
	b.h = &Holder{Callers: DefaultCallers(), Objects: memObjects{}, Transfers: memTransfers{}, Builder: fakeBuilder{ok: builderOK},
		Store: b.store, Now: func() time.Time { return b.now },
		Say: func(seat, event string, body Event) error {
			b.said = append(b.said, said{seat, event, body})
			return nil
		},
		Finding: func(kind, subject, _ string, raised bool) { b.findings[kind+" "+subject] = raised }}
	b.v = &Verbs{Holder: b.h, Catalog: b.cat, Holds: memHolds{}, Mirrors: memMirrors{}}
	return b
}

func keys[V any](m map[string]V) []string {
	var out []string
	for k := range m {
		out = append(out, k)
	}
	return out
}

func digestOf(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) }

const builder = "ace.mesh-build-go"

// TestTheHolderOfTheTwoImageSeats holds design 51's holder cases at the unit level (CAT-D1, D2, D3, D3b, D4, D4b, D5):
// one table or case per rule, the caller list and the store front's.
func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
	blob := []byte(strings.Repeat("a", 3000))
	part := Part{Digest: digestOf(blob), Size: int64(len(blob))}
	req := func(id string) PublishRequest {
		return PublishRequest{Transfer: id, Name: "m/program", Version: "0.1.4", Kind: KindBundle, Digest: part.Digest, Parts: []Part{part}}
	}

	t.Run("a transfer in parts, cut and resumed, hashed, numbered by the builder, published", func(t *testing.T) {
		b := newBed(true)
		a, err := b.h.Publish(ArtifactSeat, builder, req("t1"))
		if err != nil || !reflect.DeepEqual(a.Missing, []string{part.Digest}) {
			t.Fatalf("publish = %+v, %v", a, err)
		}
		if _, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t1", Digest: part.Digest, Sequence: 0, Data: blob[:1000]}); err != nil {
			t.Fatal(err)
		}
		// Cut: the build module opens again and is told what is still missing; its next chunk goes on from there.
		if a, _ := b.h.Publish(ArtifactSeat, builder, req("t1")); !reflect.DeepEqual(a.Missing, []string{part.Digest}) {
			t.Errorf("resumed publish = %+v", a)
		}
		if _, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t1", Digest: part.Digest, Sequence: 0, Data: blob[:1000]}); err == nil {
			t.Error("a chunk sent twice was taken: the transfer goes on from the chunks held")
		}
		a, err = b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t1", Digest: part.Digest, Sequence: 1, Last: true, Data: blob[1000:]})
		if err != nil || !a.Ended || len(b.said) != 1 || b.said[0].event != EventPublished || b.said[0].body.Reference == "" {
			t.Fatalf("last chunk = %+v, %v; said %+v", a, err, b.said)
		}
		if len(b.h.Objects.(memObjects)) != 0 || len(b.h.Transfers.(memTransfers)) != 0 {
			t.Error("the transfer's chunks or key stayed after its end")
		}
	})
	t.Run("refusals", func(t *testing.T) {
		for _, c := range []struct {
			name string
			do   func(b *bed) error
			want string
		}{
			{"a caller not on the list", func(b *bed) error { _, err := b.h.Publish(ArtifactSeat, "ace.mesh-provisioner", req("t")); return err }, "not on the holder's caller list"},
			{"an unproven caller", func(b *bed) error { _, err := b.h.Publish(ArtifactSeat, Unproven, req("t")); return err }, "no caller the tool runner proved"},
			{"a part from another caller than the opener", func(b *bed) error {
				_, _ = b.h.Publish(ArtifactSeat, builder, req("t"))
				_, err := b.h.PublishPart(ArtifactSeat, "ace.mesh-build-node", PartRequest{Transfer: "t", Digest: part.Digest, Last: true, Data: blob})
				return err
			}, "opened by another caller"},
			{"a part whose bytes do not match its digest", func(b *bed) error {
				_, _ = b.h.Publish(ArtifactSeat, builder, req("t"))
				_, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t", Digest: part.Digest, Last: true, Data: []byte("other")})
				return err
			}, "hash to"},
			{"a chunk over 1 MiB", func(b *bed) error {
				_, _ = b.h.Publish(ArtifactSeat, builder, req("t"))
				_, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t", Digest: part.Digest, Data: make([]byte, MaxPart+1)})
				return err
			}, "over the"},
			{"a new transfer at the cap", func(b *bed) error { b.h.Cap = 100; _, err := b.h.Publish(ArtifactSeat, builder, req("t")); return err }, "at their cap"},
			{"a version the builder did not number", func(b *bed) error {
				b.h.Builder = fakeBuilder{ok: false}
				b.store.has[part.Digest] = true // held already: the publish ends at once
				_, err := b.h.Publish(ArtifactSeat, builder, req("t"))
				return err
			}, "the builder did not number it"},
			{"an image on the artifact seat", func(b *bed) error {
				r := req("t")
				r.Kind = KindImage
				_, err := b.h.Publish(ArtifactSeat, builder, r)
				return err
			}, "holds bundles and archives"},
			{"remove of a version with no tag", func(b *bed) error {
				return b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/program", Version: "0.1.4", Reason: "branch ended"})
			}, "has no tag"},
			{"a retention removal from another caller than the sweep", func(b *bed) error {
				return b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/program", Digest: part.Digest, Reason: ReasonRetention})
			}, "artifact sweep's alone"},
			{"a retention removal of a held archive", func(b *bed) error {
				_ = b.v.HoldOne(ArtifactSeat, "novox.artifact-store-tools", HoldRequest{Name: "m/a", Kind: KindArchive, Digest: part.Digest})
				return b.v.Remove(ArtifactSeat, "novox.artifact-store-tools", RemoveRequest{Name: "m/a", Digest: part.Digest, Reason: ReasonRetention})
			}, "is held"},
			{"a hold of an image", func(b *bed) error {
				return b.v.HoldOne(ImageSeat, "novox.artifact-store-tools", HoldRequest{Name: "m/i", Kind: KindImage, Digest: part.Digest})
			}, "never held"},
			{"a hold from another caller", func(b *bed) error {
				return b.v.HoldOne(ArtifactSeat, "novox.mesh-build-go", HoldRequest{Name: "m/a", Kind: KindArchive, Digest: part.Digest})
			}, "not on the holder's caller list"},
			{"a release from a caller not on the list", func(b *bed) error {
				return b.v.Release(ArtifactSeat, "novox.mesh-build-go", HoldRequest{Name: "m/a", Digest: part.Digest})
			}, "not on the holder's caller list"},
			{"a release in distribution's own name", func(b *bed) error {
				return b.v.Release(ArtifactSeat, "novox.distribution", HoldRequest{Name: "m/a", Digest: part.Digest})
			}, "in its own name"},
			{"a mirror by no digest", func(b *bed) error {
				_, err := b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "docker.io/library/nats:2"})
				return err
			}, "pinned digest"},
		} {
			b := newBed(true)
			if err := c.do(b); err == nil || !strings.Contains(err.Error(), c.want) {
				t.Errorf("%s: err = %v, want one saying %q", c.name, err, c.want)
			}
		}
		if _, err := ParseCallers(`{"release":["distribution"]}`); err == nil {
			t.Error("a caller list putting distribution on release's list is taken")
		}
	})
	t.Run("only the missing layers are sent, and an idle transfer is dropped", func(t *testing.T) {
		b := newBed(true)
		other := []byte("layer two")
		r := req("t2")
		r.Kind, r.Parts = KindImage, []Part{part, {Digest: digestOf(other), Size: int64(len(other))}}
		b.store.has[part.Digest] = true
		a, err := b.h.Publish(ImageSeat, builder, r)
		if err != nil || !reflect.DeepEqual(a.Missing, []string{digestOf(other)}) {
			t.Fatalf("publish = %+v, %v; want only the layer the store lacks", a, err)
		}
		b.now = b.now.Add(TransferIdle)
		if dropped, _ := b.h.DropIdle(); !reflect.DeepEqual(dropped, []string{"t2"}) {
			t.Errorf("DropIdle = %v, want t2 after 24 hours with no part", dropped)
		}
	})
	t.Run("incoming's finding at 80 %, cleared below", func(t *testing.T) {
		b := newBed(true)
		b.h.Cap = 3500
		_, _ = b.h.Publish(ArtifactSeat, builder, req("t3"))
		_, _ = b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t3", Digest: part.Digest, Data: blob[:2800]})
		if !b.findings["incoming-full incoming"] {
			t.Error("2800 of 3500 bytes wrote no finding")
		}
		_, _ = b.h.Drop("t3")
		if b.findings["incoming-full incoming"] {
			t.Error("the finding stayed after the transfer was dropped")
		}
		// A refusal for want of room stays loud on incoming until the holder takes a publish again.
		b.h.Cap = 100
		_, _ = b.h.Publish(ArtifactSeat, builder, req("t4"))
		if !b.findings["publish-refused incoming"] {
			t.Errorf("a refusal at the cap raised %v, want publish-refused on incoming", b.findings)
		}
		b.h.Cap = 1 << 20
		b.store.has[part.Digest] = true
		if _, err := b.h.Publish(ArtifactSeat, builder, req("t5")); err != nil || b.findings["publish-refused incoming"] {
			t.Errorf("the next publish taken = %v, findings %v: want the refusal at the cap cleared", err, b.findings)
		}
	})
	t.Run("the sweep's removal, hold and release", func(t *testing.T) {
		b := newBed(true)
		sweep := "novox.artifact-store-tools"
		if err := b.v.Remove(ArtifactSeat, sweep, RemoveRequest{Name: "m/a", Digest: part.Digest, Kind: KindArchive, Reason: ReasonRetention}); err != nil {
			t.Fatal(err)
		}
		if err := b.v.HoldOne(ArtifactSeat, sweep, HoldRequest{Name: "m/b", Kind: KindArchive, Digest: part.Digest}); err != nil {
			t.Fatal(err)
		}
		if err := b.v.Release(ArtifactSeat, "person.jochen", HoldRequest{Name: "m/b", Digest: part.Digest, Why: "by hand"}); err != nil {
			t.Errorf("the operator's session releasing through registry_holds: %v", err)
		}
		var events []string
		for _, s := range b.said {
			events = append(events, s.event+" "+s.body.By)
		}
		if want := []string{"removed " + sweep, "held " + sweep, "released person.jochen"}; !reflect.DeepEqual(events, want) {
			t.Errorf("said %v, want %v", events, want)
		}
		b.cat.tags["m/p:0.1.5-feat-x.1"] = part.Digest
		if err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/p", Version: "0.1.5-feat-x.1", Reason: "branch merged"}); err != nil || b.cat.tags["m/p:0.1.5-feat-x.1"] != "" {
			t.Errorf("the builder's removal of a tagged version: %v", err)
		}
	})
	t.Run("the store's port: reads open, writes from distribution alone, and the direct-write window", func(t *testing.T) {
		store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) }))
		defer store.Close()
		u, _ := url.Parse(store.URL)
		var open atomic.Bool
		token, kept := "s3cret", true
		f := &Front{Store: u, Window: &open, Published: func(name, tag string) (bool, error) { return name == "m/i" && tag == "0.1.3", nil },
			Bootstrap: func() (string, bool) { return token, kept }}
		auth := ""
		do := func(method, path string) int {
			rec := httptest.NewRecorder()
			req := httptest.NewRequest(method, path, nil)
			if auth != "" {
				req.Header.Set("Authorization", "Bearer "+auth)
			}
			f.ServeHTTP(rec, req)
			return rec.Code
		}
		for _, c := range []struct {
			open         bool
			method, path string
			want         int
		}{
			{false, http.MethodGet, "/v2/m/i/manifests/0.1.3", http.StatusCreated},
			{false, http.MethodHead, "/v2/m/i/blobs/sha256:aa", http.StatusCreated},
			{false, http.MethodPut, "/v2/m/i/manifests/0.1.9", http.StatusForbidden},
			{false, http.MethodPost, "/v2/m/i/blobs/uploads/", http.StatusForbidden},
			{true, http.MethodPost, "/v2/m/i/blobs/uploads/", http.StatusCreated},
			{true, http.MethodPut, "/v2/m/i/manifests/0.1.9", http.StatusCreated},
			{true, http.MethodPut, "/v2/m/i/manifests/0.1.3", http.StatusForbidden}, // published through the seat
		} {
			open.Store(c.open)
			if got := do(c.method, c.path); got != c.want {
				t.Errorf("window open %v: %s %s = %d, want %d", c.open, c.method, c.path, got, c.want)
			}
		}
		// The bootstrap's one-off credential writes with the window closed, never over a published tag, and not once
		// its file is gone; a wrong token writes nothing.
		open.Store(false)
		for _, c := range []struct {
			auth string
			kept bool
			path string
			want int
		}{
			{"s3cret", true, "/v2/gitea/program/manifests/0.1.0", http.StatusCreated},
			{"s3cret", true, "/v2/m/i/manifests/0.1.3", http.StatusForbidden},
			{"wrong", true, "/v2/gitea/program/manifests/0.1.0", http.StatusForbidden},
			{"s3cret", false, "/v2/gitea/program/manifests/0.1.0", http.StatusForbidden},
		} {
			auth, kept = c.auth, c.kept
			if got := do(http.MethodPut, c.path); got != c.want {
				t.Errorf("bootstrap %q kept %v: PUT %s = %d, want %d", c.auth, c.kept, c.path, got, c.want)
			}
		}
	})
	t.Run("mirror copies an outside image by its pinned digest, every platform, only the parts the store lacks", func(t *testing.T) {
		up, store := newFakeRegistry(), newFakeRegistry()
		ups, sts := httptest.NewServer(up), httptest.NewServer(store)
		defer ups.Close()
		defer sts.Close()
		up.token, up.realm = "anon", ups.URL+"/token"
		host := strings.TrimPrefix(ups.URL, "http://")
		repo := "library/nats"
		layer, config := []byte(strings.Repeat("l", 5000)), []byte(`{"os":"linux"}`)
		up.blobs[repo+"@"+digestOf(layer)], up.blobs[repo+"@"+digestOf(config)] = layer, config
		img, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": mediaManifest,
			"config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": digestOf(config), "size": len(config)},
			"layers": []map[string]any{{"mediaType": mediaLayer, "digest": digestOf(layer), "size": len(layer)}}})
		up.manifests[repo+":"+digestOf(img)], up.types[repo+":"+digestOf(img)] = img, mediaManifest
		idx, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json",
			"manifests": []map[string]any{{"mediaType": mediaManifest, "digest": digestOf(img), "size": len(img)}}})
		up.manifests[repo+":"+digestOf(idx)] = idx
		base, _ := url.Parse(sts.URL)
		o := &OCI{Base: base, Registry: "registry.mesh:5000", Upstream: &Upstream{Scheme: "http"}}
		name := "upstream/" + host + "/" + repo
		store.blobs[name+"@"+digestOf(config)] = config // held already: not fetched again
		ref, err := o.Mirror(host+"/"+repo+":2", digestOf(idx))
		if err != nil {
			t.Fatal(err)
		}
		if want := "registry.mesh:5000/" + name + "@" + digestOf(idx); ref != want {
			t.Errorf("reference %s, want %s", ref, want)
		}
		if store.manifests[name+":"+digestOf(idx)] == nil || store.manifests[name+":"+digestOf(img)] == nil || store.blobs[name+"@"+digestOf(layer)] == nil {
			t.Errorf("the store holds %v and %v, want the index, its image and the layer", keys(store.manifests), keys(store.blobs))
		}
		if !reflect.DeepEqual(up.gets, []string{digestOf(layer)}) {
			t.Errorf("blobs fetched %v, want only the layer the store lacked", up.gets)
		}
		up.manifests[repo+":"+digestOf(idx)] = []byte(`{"tampered":true}`)
		if _, err := o.Mirror(host+"/"+repo, digestOf(idx)); err != nil {
			t.Errorf("a copy held whole is answered at once, without asking the upstream: %v", err)
		}
		delete(store.manifests, name+":"+digestOf(idx))
		if _, err := o.Mirror(host+"/"+repo, digestOf(idx)); err == nil || !strings.Contains(err.Error(), "does not hash") {
			t.Errorf("a manifest whose bytes are not its digest: %v", err)
		}
		for in, want := range map[string]string{"nats:2": "upstream/docker.io/library/nats", "ghcr.io/o/p@sha256:ab": "upstream/ghcr.io/o/p", "localhost:5000/x": "upstream/localhost:5000/x"} {
			if s, _ := ParseSource(in); s.Name() != want {
				t.Errorf("ParseSource(%q) = %s, want %s", in, s.Name(), want)
			}
		}
	})
	t.Run("mirror answers at once and says the copy's end on the seat's events", func(t *testing.T) {
		b := newBed(true)
		d := "sha256:" + strings.Repeat("c", 64)
		ref, err := b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "docker.io/library/nats:2", Digest: d})
		want := "registry.mesh/upstream/docker.io/library/nats@" + d
		key := MirrorsKey("upstream/docker.io/library/nats", d)
		asked := b.v.Mirrors.(memMirrors)
		if err != nil || ref != want || len(b.said) != 0 || asked[key].By != "novox.mesh-build-image" {
			t.Fatalf("mirror = %q, %v, said %v, asks %v: want the reference at once, the copy asked, nothing said yet", ref, err, b.said, asked)
		}
		if strings.ContainsAny(key, ":@") {
			t.Errorf("the ask's key %q holds characters a key may not", key)
		}
		if err := b.v.Mirrored(asked[key]); err != nil || len(b.said) != 1 || b.said[0].event != EventPublished ||
			b.said[0].body.Reference != want || b.said[0].body.Kind != KindUpstream || len(asked) != 0 {
			t.Fatalf("the copy's end = %v, said %+v, asks %v: want published with the reference and the ask gone", err, b.said, asked)
		}
		// One copy, said once: two followers on the same ask; the second sees it claimed and passes, and the outcome
		// carries the ask's id as its event id.
		_, _ = b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "nats", Digest: d})
		ask := asked[key]
		copies := len(b.said)
		for _, follower := range []string{"one", "two"} {
			if err := b.v.Take(key, ask, 1, false, follower); err != nil {
				t.Fatalf("Take by %s: %v", follower, err)
			}
		}
		if len(b.said) != copies+1 || ask.ID == "" || b.said[copies].body.ID != ask.ID || len(asked) != 0 {
			t.Fatalf("two followers on one ask said %+v (ask id %q), asks %v: want one copy, said once under the ask's id", b.said[copies:], ask.ID, asked)
		}
		b.said = b.said[:copies]
		b.cat.mirrorErr = errors.New("the upstream said 404")
		_, _ = b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "nats", Digest: d})
		if err := b.v.Mirrored(asked[key]); err != nil || b.said[1].event != EventRefused || !strings.Contains(b.said[1].body.Reason, "404") || len(asked) != 0 {
			t.Errorf("a failed copy = %v, said %+v: want refused with why", err, b.said)
		}
	})
	t.Run("the windows", func(t *testing.T) {
		nodes := []NodeRead{
			{Node: "ace", Serves: true, Stated: []string{NewNamesCapability}, NamesArtifact: true, BuildModules: map[string]bool{"mesh-build-go": true}},
			{Node: "g14", NamesArtifact: true, BuildModules: map[string]bool{"mesh-build-go": false}}, // away, old node-engine
			{Node: "novox", Serves: true, Stated: nil, NamesArtifact: false},
		}
		if got := HoldingOldName(nodes, nil); !reflect.DeepEqual(got, []string{"g14"}) {
			t.Errorf("old name held by %v, want g14 (no manifest-reads: it states nothing)", got)
		}
		if got := HoldingDirectWrites(nodes, nil); !reflect.DeepEqual(got, []string{"g14"}) {
			t.Errorf("direct writes held by %v, want g14", got)
		}
		t0 := time.Unix(0, 0)
		w, closed, _ := Window{Name: OldNameWindow}.Step([]string{"g14"}, t0)
		if closed != nil || !w.Open {
			t.Fatalf("Step = %+v, %+v", w, closed)
		}
		if _, _, overdue := w.Step([]string{"g14"}, t0.Add(WindowFindingAfter)); !reflect.DeepEqual(overdue, []string{"g14"}) {
			t.Errorf("seven days: overdue %v", overdue)
		}
		w, _ = w.Drop("g14", "person.jochen", "retired", t0)
		w, closed, _ = w.Step(HoldingOldName(nodes, w.Dropped), t0.Add(time.Hour))
		if closed == nil || closed.Window != OldNameWindow || !reflect.DeepEqual(closed.LastHolding, []string{"g14"}) || len(closed.Dropped) != 1 {
			t.Errorf("closing = %+v", closed)
		}
		if _, again, _ := w.Step([]string{"g14"}, t0.Add(2*time.Hour)); again != nil {
			t.Error("a closed window closed again")
		}
	})
	t.Run("the windows' driver: every node read on each wake, closed once, a paused build agent's same build counted", func(t *testing.T) {
		del := &fakeDelivery{haves: []NodeHave{
			{"ace", "mesh-build-go", "fp-new"}, {"novox", "mesh-build-go", "fp-new"}, {"g14", "mesh-build-go", "fp-old"},
			{"g14", "node", ""}, {"ace", "gitea", "x"}, {"novox", "gitea", "x"},
		}, stated: map[string][]string{"ace": {NewNamesCapability}, "novox": {NewNamesCapability}}}
		st := &memWindows{w: map[string]Window{}}
		st.pub.Learn("mesh-build-go", "fp-new")
		var closed []string
		found := map[string]bool{}
		now := time.Unix(0, 0)
		d := &Driver{Delivery: del, State: st, Callers: func() (Callers, error) { return DefaultCallers(), nil }, Now: func() time.Time { return now },
			Closed:  func(c Closed) error { closed = append(closed, c.Window+" "+c.Seat); return nil },
			Finding: func(kind, subject, _ string, raised bool) { found[kind+" "+subject] = raised }}
		if _, err := d.Wake("start"); err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(st.w[OldNameWindow].Holding, []string{"g14"}) || !reflect.DeepEqual(st.w[DirectWritesWindow].Holding, []string{"g14"}) {
			t.Errorf("windows = %+v, want g14 holding both (away: states nothing; runs the old build)", st.w)
		}
		now = now.Add(WindowFindingAfter)
		_, _ = d.Wake("delivered")
		if !found[FindingWindowHeld+" "+OldNameWindow+".g14"] {
			t.Errorf("findings %v, want g14 named after seven days", found)
		}
		del.haves[2].FP, del.stated["g14"] = "fp-new", []string{NewNamesCapability}
		if _, err := d.Wake("manifest-reads-changed on g14"); err != nil {
			t.Fatal(err)
		}
		_, _ = d.Wake("delivered")
		if want := []string{OldNameWindow + " " + OldSeat, DirectWritesWindow + " "}; !reflect.DeepEqual(closed, want) {
			t.Errorf("closed %v, want each window said once, %v", closed, want)
		}
		if found[FindingWindowHeld+" "+OldNameWindow+".g14"] {
			t.Error("the finding stays after the window closed")
		}
	})
	t.Run("the health check: versions of each seat through the program's port; down names the program, then clears", func(t *testing.T) {
		front := &downCatalog{fakeCatalog: fakeCatalog{tags: map[string]string{}}, down: true}
		found := map[string]bool{}
		f := func(kind, subject, _ string, raised bool) { found[kind+" "+subject] = raised }
		if _, err := Health(front, f); err == nil || !strings.Contains(err.Error(), "program distribution") || !found[FindingReadSide+" "+ImageSeat] {
			t.Errorf("program stopped: %v, findings %v", err, found)
		}
		front.down = false
		if _, err := Health(front, f); err != nil || found[FindingReadSide+" "+ImageSeat] {
			t.Errorf("program back: %v, findings %v", err, found)
		}
	})
	t.Run("the manifest", func(t *testing.T) {
		raw, err := os.ReadFile("../../module.json")
		if err != nil {
			t.Fatal(err)
		}
		var m struct {
			Provides []struct{ Name string } `json:"provides"`
			Claims   []struct {
				Name   string   `json:"name"`
				Serves []string `json:"serves"`
			} `json:"claims"`
			Resources []map[string]any `json:"resources"`
			Consumes  []string         `json:"consumes"`
		}
		if err := json.Unmarshal(raw, &m); err != nil {
			t.Fatal(err)
		}
		// E15: artifact-store is still offered until the old name closes, so a revert needs no edge.
		if len(m.Provides) != 1 || m.Provides[0].Name != "artifact-store" {
			t.Errorf("provides = %+v, want artifact-store kept", m.Provides)
		}
		claims := map[string][]string{}
		for _, c := range m.Claims {
			claims[c.Name] = c.Serves
		}
		if !reflect.DeepEqual(claims[ImageSeat], VerbsOf(ImageSeat)) || !reflect.DeepEqual(claims[ArtifactSeat], VerbsOf(ArtifactSeat)) {
			t.Errorf("claims = %v, want each seat serving its verbs", claims)
		}
		if _, ok := claims[OldSeat]; !ok {
			t.Error("the old seat mesh-artifact-store is no longer claimed while its window is open")
		}
		if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-view.delivered", "node-engine.manifest-reads-changed"}) {
			t.Errorf("consumes = %v, want delivered and manifest-reads-changed (the windows' wakes)", m.Consumes)
		}
		// E14: the registry container, on the loopback, is listed before the program, so the port is free when it starts.
		store, program := -1, -1
		for i, r := range m.Resources {
			switch r["id"] {
			case "store":
				store = i
				if ports, _ := r["ports"].([]any); len(ports) != 1 || ports[0] != "127.0.0.1:5001:5000" {
					t.Errorf("the store's ports = %v, want the loopback only", r["ports"])
				}
			case "program":
				program = i
				if h, _ := r["health"].(map[string]any); h["kind"] != "tool" || h["tool"] != "registry_health" {
					t.Errorf("the program's health = %v, want the tool registry_health, run outside its process", r["health"])
				}
			}
		}
		if store < 0 || program < 0 || store > program {
			t.Errorf("store at %d, program at %d: want the store listed first", store, program)
		}
	})
}