jochen
9907df6530
A copied base was named only in what a build stood on, and nowhere when the build failed, so the store's sweep could never let one go (hq issue 321). One repository per upstream image stops each module asking the public registry for the same image again, and letting an index go now takes its own platform manifests, which otherwise kept every byte. A person can record the copies no record names through the new mirrors verb (hq ADR 0257). The forge test fix is the same commit as on feat/plain-notifications: main fails without it.
304 lines
12 KiB
Go
304 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/artifacts"
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// What the records say the registries may keep, asked on demand (novox/hq ADR 0251).
|
|
|
|
func TestTheRegistryVerbsComposeTheirCommandsAndRefuseWhatTheyDoNotTake(t *testing.T) {
|
|
for _, c := range []struct {
|
|
verb string
|
|
args map[string]any
|
|
want string
|
|
}{
|
|
{"artifacts", map[string]any{}, "artifacts --json"},
|
|
{"artifacts", map[string]any{"repository": "web/app", "collected": "true"},
|
|
"artifacts --json --repository web/app --collected"},
|
|
{"artifacts", map[string]any{"collected": "false"}, "artifacts --json"},
|
|
{"collect", map[string]any{}, "collect --json"},
|
|
{"collect", map[string]any{"most": "50"}, "collect --json --most 50"},
|
|
{"collect", map[string]any{"confirm": "true", "why": "the disk is full"},
|
|
"collect --json --confirm --why the disk is full"},
|
|
{"collect", map[string]any{"confirm": true, "why": "x", "most": "10"}, "collect --json --most 10 --confirm --why x"},
|
|
{"images", map[string]any{"node": "anchor"}, "images anchor --json"},
|
|
} {
|
|
argv, err := argvFor(c.verb, c.args)
|
|
if err != nil || strings.Join(argv, " ") != c.want {
|
|
t.Errorf("%s %v: %q %v, want %q", c.verb, c.args, argv, err, c.want)
|
|
}
|
|
}
|
|
for _, c := range []struct {
|
|
verb string
|
|
args map[string]any
|
|
}{
|
|
{"collect", map[string]any{"confirm": "true"}}, // a deletion without a why
|
|
{"collect", map[string]any{"why": "tidy"}}, // a why for a dry run, recorded nowhere
|
|
{"collect", map[string]any{"confirm": "yes", "why": "x"}}, // a switch is true or false
|
|
{"collect", map[string]any{"force": "true"}},
|
|
{"artifacts", map[string]any{"node": "anchor"}},
|
|
{"images", map[string]any{}},
|
|
{"images", map[string]any{"node": "anchor", "all": "true"}},
|
|
} {
|
|
if argv, err := argvFor(c.verb, c.args); err == nil {
|
|
t.Errorf("%s %v was composed as %q", c.verb, c.args, argv)
|
|
}
|
|
}
|
|
if repairingCommand([]string{"collect", "--json", "--confirm", "--why", "x"}) != "collect" {
|
|
t.Error("a real collect through the generic verb would go unrecorded")
|
|
}
|
|
if repairingCommand([]string{"collect", "--json"}) != "" {
|
|
t.Error("a dry run was taken for an act by hand")
|
|
}
|
|
if !personsDecision(link.HandAct{Verb: "collect"}) {
|
|
t.Error("a collect a person asked for would count toward a healer the mesh lacks")
|
|
}
|
|
}
|
|
|
|
func ref(module, artifact string, n int) string {
|
|
return fmt.Sprintf("%s%s/%s@sha256:%064x", catalogue.ArtifactStoreScheme, module, artifact, n)
|
|
}
|
|
|
|
func archiveRef(module, artifact string, n int) string {
|
|
return fmt.Sprintf("%s%s/%s/blobs/sha256:%064x", catalogue.ArtifactStoreScheme, module, artifact, n)
|
|
}
|
|
|
|
func TestArtifactsCountsEverythingAndListsTheCollectedOnlyWhenAsked(t *testing.T) {
|
|
states := []inventory.ArtifactState{
|
|
{Reference: ref("web", "app", 1), State: inventory.ArtifactCollected},
|
|
{Reference: ref("web", "app", 2), State: inventory.ArtifactEligible},
|
|
{Reference: archiveRef("web", "tools", 3), State: inventory.ArtifactKept,
|
|
Why: []string{inventory.KeptByRecentBuild, inventory.KeptByDefinition}},
|
|
{Reference: ref("db", "server", 4), State: inventory.ArtifactKept, Why: []string{inventory.KeptByDefinition}},
|
|
// Not the mesh's own store: never listed, never counted.
|
|
{Reference: "registry.invalid/x@sha256:" + strings.Repeat("a", 64), State: inventory.ArtifactKept,
|
|
Why: []string{inventory.KeptUnaddressable}},
|
|
}
|
|
a := artifactsOf(states, "", false)
|
|
if a.Counts != (artifactCounts{Kept: 2, Eligible: 1, Collected: 1}) {
|
|
t.Errorf("counts %+v", a.Counts)
|
|
}
|
|
if len(a.References) != 3 {
|
|
t.Fatalf("listed %d, want the kept and the eligible: %+v", len(a.References), a.References)
|
|
}
|
|
archive := a.References[1]
|
|
if archive.Kind != "archive" || archive.Repository != "web/tools" || archive.Digest != fmt.Sprintf("sha256:%064x", 3) ||
|
|
!slices.Equal(archive.Why, []string{"recent-build", "definition"}) {
|
|
t.Errorf("an archive read as %+v", archive)
|
|
}
|
|
if image := a.References[0]; image.Kind != "image" || image.Repository != "web/app" || image.State != "eligible" {
|
|
t.Errorf("an image read as %+v", image)
|
|
}
|
|
narrowed := artifactsOf(states, "web/app", true)
|
|
if narrowed.Counts != a.Counts || len(narrowed.References) != 2 {
|
|
t.Errorf("narrowed to one repository: %+v", narrowed)
|
|
}
|
|
raw, _ := json.Marshal(a.References[0])
|
|
if strings.Contains(string(raw), `"why"`) {
|
|
t.Errorf("an empty why is carried: %s", raw)
|
|
}
|
|
}
|
|
|
|
// fakeStore answers as a registry does for the sweep's questions, and records every request.
|
|
type fakeStore struct {
|
|
mu sync.Mutex
|
|
requests []string
|
|
refuse string // a DELETE whose path contains this is refused
|
|
}
|
|
|
|
func (f *fakeStore) serve(t *testing.T) artifacts.Store {
|
|
t.Helper()
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
f.mu.Lock()
|
|
f.requests = append(f.requests, r.Method+" "+r.URL.Path)
|
|
refuse := f.refuse
|
|
f.mu.Unlock()
|
|
switch {
|
|
case r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/"):
|
|
w.Header().Set("Content-Length", "10")
|
|
w.WriteHeader(http.StatusOK)
|
|
case r.Method == http.MethodHead:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/manifests/"):
|
|
// An image, not an index: it names no platform manifests.
|
|
w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
|
_, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`))
|
|
case r.Method == http.MethodPost:
|
|
w.Header().Set("Location", "/upload/x?state=1")
|
|
w.WriteHeader(http.StatusAccepted)
|
|
case r.Method == http.MethodPut:
|
|
w.WriteHeader(http.StatusCreated)
|
|
case r.Method == http.MethodDelete && refuse != "" && strings.Contains(r.URL.Path, refuse):
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
case r.Method == http.MethodDelete:
|
|
w.WriteHeader(http.StatusAccepted)
|
|
default:
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
}
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
return artifacts.Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
|
}
|
|
|
|
func (f *fakeStore) seen() []string {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return slices.Clone(f.requests)
|
|
}
|
|
|
|
func TestADryRunCollectAsksOnlyAndChangesNothing(t *testing.T) {
|
|
f := &fakeStore{}
|
|
store := f.serve(t)
|
|
kept := []string{archiveRef("web", "tools", 1)}
|
|
eligible := []string{ref("web", "app", 2), archiveRef("web", "tools", 3)}
|
|
a := runCollect(context.Background(), nil, store, eligible, kept, false,
|
|
sweepBounds{most: 10, budget: 5 * time.Second})
|
|
if !a.DryRun || a.Eligible != 2 || !slices.Equal(a.WouldLetGo, eligible) || len(a.LetGo) != 0 {
|
|
t.Errorf("a dry run answered %+v", a)
|
|
}
|
|
if a.KeptArchives != 1 || a.Held != 0 || a.Unheld != 1 {
|
|
t.Errorf("the kept archive is unheld in the fake store, and was said as %+v", a)
|
|
}
|
|
for _, r := range f.seen() {
|
|
if !strings.HasPrefix(r, "HEAD ") {
|
|
t.Errorf("a dry run asked the store %s", r)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCollectSaysWhatItDoesNotListAndWithNoStoreAsksNothing(t *testing.T) {
|
|
var many []string
|
|
for i := range mostListed + 5 {
|
|
many = append(many, ref("web", "app", i))
|
|
}
|
|
a := runCollect(context.Background(), nil, artifacts.Store{}, many, nil, false, sweepBounds{most: 1, budget: time.Second})
|
|
if len(a.WouldLetGo) != mostListed || a.NotListed != 5 || a.Left != len(many) || a.Stopped == "" {
|
|
t.Errorf("with no store and %d eligible: %d listed, %d not, left %d, stopped %q",
|
|
len(many), len(a.WouldLetGo), a.NotListed, a.Left, a.Stopped)
|
|
}
|
|
}
|
|
|
|
func TestARealCollectHoldsEveryKeptArchiveBeforeItLetsAnythingGo(t *testing.T) {
|
|
inv := inventory.ForTest(t)
|
|
f := &fakeStore{}
|
|
store := f.serve(t)
|
|
kept := []string{archiveRef("web", "tools", 1)}
|
|
eligible := []string{ref("web", "app", 2), ref("web", "app", 3), ref("web", "app", 4)}
|
|
madeBy(t, inv, eligible)
|
|
a := runCollect(t.Context(), inv, store, eligible, kept, true, sweepBounds{most: 2, budget: 5 * time.Second})
|
|
if a.DryRun || a.HoldersWritten != 1 || a.Held != 1 {
|
|
t.Errorf("the kept archive was not held first: %+v", a)
|
|
}
|
|
if !slices.Equal(a.LetGo, eligible[:2]) || a.Left != 1 || a.Stopped == "" {
|
|
t.Errorf("bounded at two: let go %v, left %d, stopped %q", a.LetGo, a.Left, a.Stopped)
|
|
}
|
|
requests := f.seen()
|
|
firstPut := slices.IndexFunc(requests, func(r string) bool { return strings.HasPrefix(r, "PUT ") && strings.Contains(r, "/manifests/") })
|
|
firstDelete := slices.IndexFunc(requests, func(r string) bool { return strings.HasPrefix(r, "DELETE ") })
|
|
if firstPut < 0 || firstDelete < 0 || firstPut > firstDelete {
|
|
t.Errorf("the holder was not put before the first delete: %v", requests)
|
|
}
|
|
// What was let go is recorded collected: the sweep no longer offers it.
|
|
left, err := inv.ToCollect(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !slices.Equal(left, eligible[2:]) {
|
|
t.Errorf("after letting go of two, the records offer %v", left)
|
|
}
|
|
}
|
|
|
|
// madeBy records one successful build, of a module the mesh does not hold, that made these images:
|
|
// eligible, every one.
|
|
func madeBy(t *testing.T, inv *inventory.Inventory, references []string) {
|
|
t.Helper()
|
|
b := inventory.Build{ID: "b1", Repository: "https://forge.invalid/web.git", Module: "web", On: "a-build-machine",
|
|
Commit: "c0ffee"}
|
|
for i, r := range references {
|
|
b.Made = append(b.Made, inventory.Artifact{Name: fmt.Sprintf("app%d", i), Kind: "image", Reference: r})
|
|
}
|
|
if err := inv.RecordBuild(t.Context(), b); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestARealCollectStopsAtTheStoresFirstRefusal(t *testing.T) {
|
|
inv := inventory.ForTest(t)
|
|
f := &fakeStore{refuse: fmt.Sprintf("%064x", 3)}
|
|
store := f.serve(t)
|
|
eligible := []string{ref("web", "app", 2), ref("web", "app", 3), ref("web", "app", 4)}
|
|
a := runCollect(t.Context(), inv, store, eligible, nil, true, sweepBounds{most: 10, budget: 5 * time.Second})
|
|
if !slices.Equal(a.LetGo, eligible[:1]) || a.Left != 2 || !strings.Contains(a.Stopped, "kept") {
|
|
t.Errorf("after a refusal: let go %v, left %d, stopped %q", a.LetGo, a.Left, a.Stopped)
|
|
}
|
|
}
|
|
|
|
func TestImagesAreEveryContainerImageOfTheDeclarationAndOfWhatWasSent(t *testing.T) {
|
|
body := []byte(`{"declaration":1,"resources":[
|
|
{"id":"web.server","type":"container","image":"store.invalid/web/server@sha256:aa"},
|
|
{"id":"web.collect","type":"container","image":"store.invalid/web/server@sha256:aa","schedule":"30 3 * * *"},
|
|
{"id":"db.server","type":"container","image":"postgres@sha256:bb"},
|
|
{"id":"web.bundle-tools","type":"archive","source":"http://store.invalid/v2/web/tools/blobs/sha256:cc"}]}`)
|
|
sent := []sentResource{
|
|
{ID: "web.server", Type: "container", Image: "store.invalid/web/server@sha256:99"},
|
|
{ID: "db.server", Type: "container", Image: "postgres@sha256:bb"},
|
|
{ID: "web.state", Type: "directory"},
|
|
}
|
|
a, err := imagesOf("anchor", body, sent, true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !a.SentKnown || len(a.Images) != 3 {
|
|
t.Fatalf("answered %+v", a)
|
|
}
|
|
byImage := map[string]declaredImage{}
|
|
for _, d := range a.Images {
|
|
byImage[d.Image] = d
|
|
}
|
|
if d := byImage["store.invalid/web/server@sha256:aa"]; !slices.Equal(d.Resources, []string{"web.collect", "web.server"}) ||
|
|
!slices.Equal(d.In, []string{"declaration"}) {
|
|
t.Errorf("a scheduled step's image and its server's are one: %+v", d)
|
|
}
|
|
if d := byImage["store.invalid/web/server@sha256:99"]; !slices.Equal(d.In, []string{"sent"}) {
|
|
t.Errorf("the image last sent is kept beside the one to send: %+v", d)
|
|
}
|
|
if d := byImage["postgres@sha256:bb"]; !slices.Equal(d.In, []string{"declaration", "sent"}) {
|
|
t.Errorf("an image in both: %+v", d)
|
|
}
|
|
|
|
// A summary kept before it carried images cannot say what was sent.
|
|
old, err := imagesOf("anchor", body, []sentResource{{ID: "web.server", Type: "container"}}, true)
|
|
if err != nil || old.SentKnown {
|
|
t.Errorf("an old summary was read as knowing what was sent: %+v %v", old, err)
|
|
}
|
|
none, err := imagesOf("anchor", body, nil, false)
|
|
if err != nil || none.SentKnown || len(none.Images) != 2 {
|
|
t.Errorf("with nothing sent yet: %+v %v", none, err)
|
|
}
|
|
}
|
|
|
|
func TestTheSentSummaryKeepsAContainersImage(t *testing.T) {
|
|
summary, err := summarize([]byte(`{"resources":[{"id":"web.server","type":"container","image":"x@sha256:aa"},
|
|
{"id":"web.state","type":"directory","path":"/srv"}]}`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if summary[0].Image != "x@sha256:aa" || summary[1].Image != "" {
|
|
t.Errorf("summarized as %+v", summary)
|
|
}
|
|
}
|