artifact-store-tools: say an image a definition pins is named by a definition, under any name the store has had (issue 323)
mesh/merge-gate pass: builds artifact-store-tools → novox; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: THE CHANGE ALTERS ITS OWN CHECK (merge-check.sh): main's version judged it; the change's judges the pull requests after it merges; it…
mesh/delivery stopped: the pull request closed unmerged
mesh/merge-gate pass: builds artifact-store-tools → novox; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: THE CHANGE ALTERS ITS OWN CHECK (merge-check.sh): main's version judged it; the change's judges the pull requests after it merges; it…
mesh/delivery stopped: the pull request closed unmerged
No record names an image the mesh did not make, so one a definition pins under an older name of the store read as unrecorded, the same word as an image nothing uses. The tool now reads every definition through the catalogue and says named-by-a-definition with the module. The store's older names are the mesh's setting older-store-names, since a catalogue names no installation. The catalogue's check warns of a pin under an older name with the current one.
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/storenames
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,132 @@
|
||||
// Command storenames warns, in every module's manifest, of an image pinned by digest under a name the
|
||||
// artifact store had before (novox/hq issue 323). Such a pin still pulls, and artifact_store_references says
|
||||
// it is named by a definition; the warning says the store's current name, so a definition moves to it when
|
||||
// it next changes. It warns and never refuses: the image runs, and nothing is wrong with it but its name.
|
||||
//
|
||||
// The older names are listed in older-names beside this check, one per line: the same names the mesh's
|
||||
// setting older-store-names of artifact-store-tools gives the store's tools, which read definitions under
|
||||
// them. A manifest names no installation, so the tools have them from the mesh's setting; this check runs
|
||||
// before the mesh is asked anything, so it has them from its list.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 only when a manifest cannot be read.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// OlderNamesFile is the list of the store's older names, beside the check.
|
||||
const OlderNamesFile = "checks/storenames/older-names"
|
||||
|
||||
// Current is the store's name a definition writes now: the scheme the controller routes to wherever the
|
||||
// store is.
|
||||
const Current = "artifact-store://"
|
||||
|
||||
// Finding is one pin under an older name.
|
||||
type Finding struct {
|
||||
Module, Pinned, Instead string
|
||||
}
|
||||
|
||||
// OlderNames reads the older names from the list: one per line, a # beginning a remark.
|
||||
func OlderNames(root string) ([]string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(root, OlderNamesFile))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if at := strings.Index(line, "#"); at >= 0 {
|
||||
line = line[:at]
|
||||
}
|
||||
if n := strings.Trim(strings.TrimSpace(line), "/"); n != "" {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// pinUnder is `<name>/<repository>@sha256:<hex>`, the name standing on its own.
|
||||
func pinUnder(name string) *regexp.Regexp {
|
||||
return regexp.MustCompile(`(?:^|[^A-Za-z0-9.-])` + regexp.QuoteMeta(name) + `/([a-z0-9][a-z0-9._/-]*@sha256:[0-9a-f]{64})`)
|
||||
}
|
||||
|
||||
// Check reads every manifest under root/modules and answers each pin under an older name.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
older, err := OlderNames(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Finding
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var doc any
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
module := filepath.Base(filepath.Dir(path))
|
||||
seen := map[string]bool{}
|
||||
var walk func(any)
|
||||
walk = func(x any) {
|
||||
switch v := x.(type) {
|
||||
case map[string]any:
|
||||
keys := make([]string, 0, len(v))
|
||||
for k := range v {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
walk(v[k])
|
||||
}
|
||||
case []any:
|
||||
for _, e := range v {
|
||||
walk(e)
|
||||
}
|
||||
case string:
|
||||
for _, name := range older {
|
||||
for _, m := range pinUnder(name).FindAllStringSubmatch(v, -1) {
|
||||
pinned := name + "/" + m[1]
|
||||
if !seen[pinned] {
|
||||
seen[pinned] = true
|
||||
out = append(out, Finding{module, pinned, Current + m[1]})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(doc)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Printf("WARNING %s pins %s under an older name of the artifact store: its current name is %s "+
|
||||
"(novox/hq issue 323; a warning, not a refusal)\n", f.Module, f.Pinned, f.Instead)
|
||||
}
|
||||
fmt.Printf("store-names: %d pins under an older name of the artifact store\n", len(found))
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func manifest(t *testing.T, root, module, body string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", module)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func older(t *testing.T, root, body string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(root, OlderNamesFile)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func digest(c string) string { return "sha256:" + strings.Repeat(c, 64) }
|
||||
|
||||
func TestAPinUnderAnOlderNameOfTheStoreIsWarnedOfWithItsCurrentName(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
older(t, root, "# the store's older names\nold.example\nolder.example/ # with a slash\n")
|
||||
manifest(t, root, "website", `{"module":"website","resources":[{"type":"container","image":"old.example/novox/www@`+digest("a")+`"}]}`)
|
||||
manifest(t, root, "tool", `{"module":"tool","resources":[{"type":"file","content":"IMAGE=older.example/novox/t@`+digest("b")+`\n"}]}`)
|
||||
manifest(t, root, "fine", `{"module":"fine","resources":[`+
|
||||
`{"type":"container","image":"artifact-store://novox/www@`+digest("a")+`"},`+
|
||||
`{"type":"container","image":"notold.example/novox/www@`+digest("c")+`"},`+
|
||||
`{"type":"container","image":"old.example/novox/www:latest"},`+
|
||||
`{"type":"container","image":"docker.io/library/nginx@`+digest("d")+`"}]}`)
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 2 {
|
||||
t.Fatalf("warned %+v", found)
|
||||
}
|
||||
if f := found[1]; f.Module != "website" || f.Pinned != "old.example/novox/www@"+digest("a") || f.Instead != "artifact-store://novox/www@"+digest("a") {
|
||||
t.Errorf("website: %+v", f)
|
||||
}
|
||||
if f := found[0]; f.Module != "tool" || f.Instead != "artifact-store://novox/t@"+digest("b") {
|
||||
t.Errorf("tool: %+v", f)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithoutOlderNamesNothingIsWarned(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
older(t, root, "# none\n")
|
||||
manifest(t, root, "website", `{"module":"website","resources":[{"type":"container","image":"old.example/novox/www@`+digest("a")+`"}]}`)
|
||||
if found, err := Check(root); err != nil || len(found) != 0 {
|
||||
t.Fatalf("warned %+v, %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheListOfOlderNamesMustBeThere(t *testing.T) {
|
||||
if _, err := Check(t.TempDir()); err == nil {
|
||||
t.Fatal("passed without its list of older names")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAManifestThatCannotBeReadFails(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
older(t, root, "")
|
||||
manifest(t, root, "broken", `{`)
|
||||
if _, err := Check(root); err == nil {
|
||||
t.Fatal("a manifest that is not JSON was passed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
# The names the artifact store had before, one per line (novox/hq issue 323). A definition that pins an image
|
||||
# by digest under one of these is warned of, with the store's current name. The same names are the mesh's
|
||||
# setting older-store-names of artifact-store-tools, under which its tools read the definitions; change both.
|
||||
registry-api.novox.be
|
||||
@@ -30,11 +30,17 @@
|
||||
# 5. the agent's home (novox/hq ADR 0266): no module unpacks an archive, or writes into a file, under
|
||||
# ${machine:agent-home} — the node-engine does both after checking the path, and the agent account owns
|
||||
# that home and could swap a link in between.
|
||||
# 6. the store's names (novox/hq issue 323): an image pinned by digest under a name the artifact store had
|
||||
# before is warned of, with the store's current name — never refused, as the image runs. The older names
|
||||
# are listed in checks/storenames/older-names, as the mesh's setting older-store-names gives them to
|
||||
# artifact-store-tools.
|
||||
set -eu
|
||||
|
||||
(cd checks/words && go test -count=1 . && go run . ../..)
|
||||
# The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
|
||||
(cd checks/agenthome && go test -count=1 . && go run . ../..)
|
||||
# The store's names (novox/hq issue 323): a pin under an older name is warned of, never refused.
|
||||
(cd checks/storenames && go test -count=1 . && go run . ../..)
|
||||
# The lists that narrow the paths check may not grow unseen: the base branch's copies are what they are
|
||||
# judged against (MESH_CHECK_BASE, main when unset), read from the clone the check runs in.
|
||||
base_lists=""
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# artifact-store-tools
|
||||
|
||||
Tools that say what the mesh's artifact store holds, and what the controller's records say of it
|
||||
(novox/hq ADR 0251 §1–3, to-be 51). It declares no resources and changes nothing on its machine.
|
||||
(novox/hq ADR 0251 §1–3, to-be 51). It changes nothing on its machine; its only resources are its state
|
||||
directory and the file the mesh writes there from its setting `older-store-names`.
|
||||
|
||||
## Why a module beside the store
|
||||
|
||||
@@ -18,7 +19,7 @@ on that machine.
|
||||
|---|---|---|
|
||||
| `artifact_store_repositories` | r | every repository: tags and what each names, how many manifests (tagged or not), size |
|
||||
| `artifact_store_usage` | r | the store's size, the largest repositories, bytes shared between repositories, bytes no manifest marks (what the nightly collector frees next) |
|
||||
| `artifact_store_references` | r | what the controller's records say of each manifest, counted and sized by state and repository; each manifest listed when one repository is asked; what the records keep that the store does not hold |
|
||||
| `artifact_store_references` | r | what the controller's records and the catalogue's definitions say of each manifest, counted and sized by state and repository; each manifest listed when one repository is asked; what the records keep that the store does not hold |
|
||||
| `artifact_store_collect` | a | a dry run unless `dry_run` is false: what the controller would let go of, and the bytes the nightly collector would free then and now. A real run needs `why` and asks the controller's `collect` |
|
||||
|
||||
### How the store is read
|
||||
@@ -43,6 +44,7 @@ it lists. The store's collector removes every blob no manifest marks, so these n
|
||||
| state | what | removable |
|
||||
|---|---|---|
|
||||
| `kept` | a definition names it, or one of the five most recent builds of a module the mesh holds; `why` says which | no |
|
||||
| `named-by-a-definition` | no record names it, and a definition in the catalogue pins it by digest, under the store's current name or an older one; `why` names the module (hq issue 323) | no |
|
||||
| `holder-of-kept-archive` | the manifest that keeps a kept archive's blob (hq issue 253) | no |
|
||||
| `eligible` | the mesh made it and keeps it for no reason | through the controller's `collect` |
|
||||
| `holder-of-eligible-archive` | the manifest that keeps an eligible archive's blob | with its archive |
|
||||
@@ -56,6 +58,22 @@ its own right; the controller lets an index go with the platform manifests only
|
||||
counted as the index's. Two indexes naming one platform give it the more keeping state. An index no
|
||||
record names leaves its platform manifests unrecorded with it.
|
||||
|
||||
**What the definitions pin** (hq issue 323). A definition may pin an image of the store the mesh did not
|
||||
make: pushed before the mesh built it, and pinned under the store's current name or a name it had before.
|
||||
No record of the controller names it, yet a machine runs it, so `artifact_store_references` reads every
|
||||
definition the catalogue holds (`mesh-catalog.catalog_modules`, then `catalog_module` for each, eight at
|
||||
a time and within twenty seconds) and finds every `<name>/<repository>@sha256:…` in any of its strings,
|
||||
under each of the store's names: `artifact-store://`, the store's address as the controller answers it,
|
||||
and the names it had before. Those are the mesh's setting `older-store-names` (space-separated, `none` by
|
||||
default, set with the controller's `settings`), which the mesh writes to the module's `store-names` file: a
|
||||
catalogue names no installation. When they cannot be read, the answer's first line says so. A manifest no record
|
||||
names that a definition pins is `named-by-a-definition`; one the records name keeps the records' state. A
|
||||
definition read at its commit is kept, so the next call asks only for the list. Definitions that could
|
||||
not be read are listed under `definitions.unread`, and the answer's first line says that what they pin
|
||||
reads as unrecorded until they are. The catalogue's own check (`checks/storenames`) warns of every pin
|
||||
under an older name, with the store's current name; its list `checks/storenames/older-names` holds the same
|
||||
names as the setting.
|
||||
|
||||
**Unrecorded copies of upstream images** are listed in `unrecorded_mirrors`: each manifest at the top of
|
||||
a repository only the build's mirror writes (`upstream/<host>/<path>`, or a module's `<module>/on-<argument>`
|
||||
from before hq ADR 0257) that no record names, with what it and its platforms mark alone. They are the
|
||||
@@ -88,6 +106,8 @@ with slashes, a cut listing refused), manifests read with one absent or unreadab
|
||||
digest never becoming a path outside the blobs, the mark (an index, a shared blob, a blob nothing marks),
|
||||
every state, what the records keep that the store lacks, bytes freed counting a blob shared with a kept
|
||||
manifest as kept, a dry run never confirming, a real run without `why` refused before anything is asked,
|
||||
the fall-back when the references let go of cannot be carried, escalation through `sudo -n`, and that the
|
||||
tools served are exactly the manifest's `tools` and its `invokes` exactly the two controller verbs.
|
||||
the fall-back when the references let go of cannot be carried, a manifest a definition pins under the
|
||||
store's current or an older name, a definition not read said first, escalation through `sudo -n`, and that the
|
||||
tools served are exactly the manifest's `tools` and its `invokes` exactly the two controller verbs and the
|
||||
catalogue's two tools.
|
||||
Any command other than the two read-only scripts fails the test.
|
||||
|
||||
@@ -7,12 +7,15 @@ import (
|
||||
|
||||
// The states a manifest the store holds can be in, as the records see it (novox/hq to-be 51).
|
||||
const (
|
||||
StateKept = "kept"
|
||||
StateHolderKept = "holder-of-kept-archive"
|
||||
StateEligible = "eligible"
|
||||
StateHolderEligible = "holder-of-eligible-archive"
|
||||
StateCollectedPresent = "let-go-yet-present"
|
||||
StateNamedDocument = "named-document"
|
||||
StateKept = "kept"
|
||||
StateHolderKept = "holder-of-kept-archive"
|
||||
StateEligible = "eligible"
|
||||
StateHolderEligible = "holder-of-eligible-archive"
|
||||
StateCollectedPresent = "let-go-yet-present"
|
||||
StateNamedDocument = "named-document"
|
||||
// StateNamedByDefinition is a manifest no record names that a definition pins by digest, under the
|
||||
// store's current name or an older one (novox/hq issue 323): a machine may run it, so it is kept.
|
||||
StateNamedByDefinition = "named-by-a-definition"
|
||||
StateUnrecorded = "unrecorded"
|
||||
collectionRemovesThese = "eligible and holder-of-eligible-archive, through the controller's collect; nothing else"
|
||||
)
|
||||
@@ -45,8 +48,8 @@ var mirrorRepository = regexp.MustCompile(`^(upstream/[a-z0-9][a-z0-9._/-]*|[a-z
|
||||
// keeping orders the states a platform manifest may take from the indexes naming it: the most keeping
|
||||
// wins, so a platform two indexes name is as kept as the more kept of them.
|
||||
var keeping = map[string]int{
|
||||
StateKept: 0, StateNamedDocument: 1, StateHolderKept: 2, StateEligible: 3, StateHolderEligible: 4,
|
||||
StateCollectedPresent: 5,
|
||||
StateKept: 0, StateNamedByDefinition: 1, StateNamedDocument: 2, StateHolderKept: 3, StateEligible: 4,
|
||||
StateHolderEligible: 5, StateCollectedPresent: 6,
|
||||
}
|
||||
|
||||
// recordIndex finds a record by repository and digest.
|
||||
@@ -64,8 +67,9 @@ func indexRecords(r *Records) recordIndex {
|
||||
return idx
|
||||
}
|
||||
|
||||
// Classify gives every manifest the store holds one state.
|
||||
func Classify(v *View, recs *Records) []Entry {
|
||||
// Classify gives every manifest the store holds one state. A manifest no record names that a definition
|
||||
// pins is named by a definition; defs may be nil, when the definitions were not read.
|
||||
func Classify(v *View, recs *Records, defs *Definitions) []Entry {
|
||||
idx := indexRecords(recs)
|
||||
var out []Entry
|
||||
for _, name := range v.L.RepoNames() {
|
||||
@@ -99,6 +103,12 @@ func Classify(v *View, recs *Records) []Entry {
|
||||
e.State = StateNamedDocument
|
||||
}
|
||||
}
|
||||
if e.State == StateUnrecorded && defs != nil {
|
||||
for _, p := range defs.Pins[Key(name, d)] {
|
||||
e.State = StateNamedByDefinition
|
||||
e.Why = append(e.Why, p.why())
|
||||
}
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// What the definitions pin in the store (novox/hq issue 323). A definition may pin an image of the store by
|
||||
// digest that the mesh did not make: one pushed before the mesh built it, pinned under the store's current
|
||||
// name or under a name the store had before. No record of the controller names such a manifest, and yet a
|
||||
// machine runs it, so it is said to be named by a definition and never unrecorded.
|
||||
|
||||
// NamesFileEnv names the file the mesh writes from the setting older-store-names: a line `older=` with the
|
||||
// names the store had before, as a definition may still write them — host names, or host:port, separated by
|
||||
// spaces — or `none`. A catalogue names no installation, so the names are the mesh's setting, not the
|
||||
// manifest's.
|
||||
const NamesFileEnv = "MESH_ARTIFACT_STORE_NAMES_FILE"
|
||||
|
||||
// CatalogueModules and CatalogueModule are the catalogue's tools these definitions are read through.
|
||||
const (
|
||||
CatalogueModules = "mesh-catalog.catalog_modules"
|
||||
CatalogueModule = "mesh-catalog.catalog_module"
|
||||
)
|
||||
|
||||
// DefinitionsTimeout bounds reading every definition, below the runtime's thirty-second call limit; what
|
||||
// was read is kept, so a call that runs out reads only the rest the next time.
|
||||
const DefinitionsTimeout = 20 * time.Second
|
||||
|
||||
// definitionReaders is how many definitions are asked at once.
|
||||
const definitionReaders = 8
|
||||
|
||||
// StoreNames are the names a definition may pin an image of the store under: the scheme the mesh writes
|
||||
// (artifact-store://), the store's address now, and the names it had before.
|
||||
type StoreNames struct {
|
||||
Current []string
|
||||
Older []string
|
||||
}
|
||||
|
||||
// NamesOf is the store's names: its address as the controller answers it, and the older ones given.
|
||||
func NamesOf(address, older string) StoreNames {
|
||||
n := StoreNames{Current: []string{"artifact-store://"}}
|
||||
if a := strings.Trim(strings.TrimSpace(address), "/"); a != "" {
|
||||
n.Current = append(n.Current, a+"/")
|
||||
}
|
||||
for _, o := range strings.FieldsFunc(older, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' }) {
|
||||
if o = strings.Trim(o, "/"); o != "" && o != "none" {
|
||||
n.Older = append(n.Older, o+"/")
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// OlderNames reads the older names from the file the mesh writes, and says why when it cannot.
|
||||
func OlderNames() (string, string) {
|
||||
path := os.Getenv(NamesFileEnv)
|
||||
if path == "" {
|
||||
return "", NamesFileEnv + " is not set, so no older name of the store was read"
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", "the store's older names could not be read: " + err.Error()
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "older="); ok {
|
||||
return v, ""
|
||||
}
|
||||
}
|
||||
return "", path + " holds no line older=, so no older name of the store was read"
|
||||
}
|
||||
|
||||
// Pin is one definition pinning a manifest of the store by digest.
|
||||
type Pin struct {
|
||||
Module string `json:"module"`
|
||||
// Name is the store's name it is pinned under, as written: "artifact-store://", an address, or an older name.
|
||||
Name string `json:"name"`
|
||||
Older bool `json:"older,omitempty"`
|
||||
}
|
||||
|
||||
// pinned is one pin found in a manifest: where in the store, and under which name.
|
||||
type pinned struct {
|
||||
Repository, Digest, Name string
|
||||
Older bool
|
||||
}
|
||||
|
||||
var pinShapes sync.Map // name -> *regexp.Regexp
|
||||
|
||||
// pinShape is `<name><repository>@sha256:<hex>`, the name standing on its own: not the tail of a longer host.
|
||||
func pinShape(name string) *regexp.Regexp {
|
||||
if re, ok := pinShapes.Load(name); ok {
|
||||
return re.(*regexp.Regexp)
|
||||
}
|
||||
re := regexp.MustCompile(`(?:^|[^A-Za-z0-9.-])` + regexp.QuoteMeta(name) +
|
||||
`([a-z0-9][a-z0-9._/-]*)@(sha256:[0-9a-f]{64})`)
|
||||
pinShapes.Store(name, re)
|
||||
return re
|
||||
}
|
||||
|
||||
// PinsIn finds every image of the store a manifest pins by digest, in any string it holds: a resource's
|
||||
// image, a build's base, a file's content.
|
||||
func PinsIn(manifest json.RawMessage, names StoreNames) []pinned {
|
||||
var doc any
|
||||
if json.Unmarshal(manifest, &doc) != nil {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []pinned
|
||||
var walk func(any)
|
||||
walk = func(x any) {
|
||||
switch v := x.(type) {
|
||||
case map[string]any:
|
||||
for _, e := range v {
|
||||
walk(e)
|
||||
}
|
||||
case []any:
|
||||
for _, e := range v {
|
||||
walk(e)
|
||||
}
|
||||
case string:
|
||||
for _, group := range []struct {
|
||||
names []string
|
||||
older bool
|
||||
}{{names.Current, false}, {names.Older, true}} {
|
||||
for _, name := range group.names {
|
||||
for _, m := range pinShape(name).FindAllStringSubmatch(v, -1) {
|
||||
p := pinned{Repository: strings.Trim(m[1], "/"), Digest: m[2], Name: name, Older: group.older}
|
||||
if k := p.Repository + "@" + p.Digest + " " + p.Name; !seen[k] {
|
||||
seen[k] = true
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(doc)
|
||||
return out
|
||||
}
|
||||
|
||||
// Definitions is what the catalogue's definitions pin in the store, and what of them could not be read.
|
||||
type Definitions struct {
|
||||
// Pins are by Key(repository, digest).
|
||||
Pins map[string][]Pin
|
||||
// Read is how many definitions were read; Unread says, per module, why its definition was not.
|
||||
Read int
|
||||
Unread map[string]string
|
||||
// NotAsked says why the catalogue could not be asked at all, when it could not.
|
||||
NotAsked string
|
||||
// NamesNotRead says why the store's older names could not be read, when they could not.
|
||||
NamesNotRead string
|
||||
Names StoreNames
|
||||
}
|
||||
|
||||
// Catalogue reads the definitions through the catalogue's tools, and keeps each definition it read by its
|
||||
// module and commit, which never change together.
|
||||
type Catalogue struct {
|
||||
Ask Asking
|
||||
|
||||
mu sync.Mutex
|
||||
kept map[string][]pinned // "<module>@<commit>|<names>" -> its pins
|
||||
}
|
||||
|
||||
type catalogueEntry struct {
|
||||
Module string `json:"module"`
|
||||
Commit string `json:"commit"`
|
||||
}
|
||||
|
||||
// Read asks the catalogue for every module and reads each definition, bounded by DefinitionsTimeout.
|
||||
func (c *Catalogue) Read(ctx context.Context, names StoreNames) *Definitions {
|
||||
defs := &Definitions{Pins: map[string][]Pin{}, Unread: map[string]string{}, Names: names}
|
||||
if c == nil || c.Ask == nil {
|
||||
defs.NotAsked = "this bundle has no way to ask the catalogue"
|
||||
return defs
|
||||
}
|
||||
raw, err := c.Ask(CatalogueModules, map[string]any{})
|
||||
if err != nil {
|
||||
defs.NotAsked = "the catalogue's modules could not be asked: " + err.Error()
|
||||
return defs
|
||||
}
|
||||
answer, output, ok := answerOf(raw)
|
||||
if !ok {
|
||||
defs.NotAsked = "the catalogue refused its modules: " + lastLine(output)
|
||||
return defs
|
||||
}
|
||||
var list struct {
|
||||
Modules []catalogueEntry `json:"modules"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(answer, &list); err != nil || list.Error != "" || list.Modules == nil {
|
||||
defs.NotAsked = "the catalogue's modules answer is not readable: " + firstOf(list.Error, errText(err), lastLine(output))
|
||||
return defs
|
||||
}
|
||||
c.mu.Lock()
|
||||
if c.kept == nil {
|
||||
c.kept = map[string][]pinned{}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
nameKey := strings.Join(names.Current, ",") + ";" + strings.Join(names.Older, ",")
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, DefinitionsTimeout)
|
||||
defer cancel()
|
||||
type result struct {
|
||||
module string
|
||||
pins []pinned
|
||||
why string
|
||||
}
|
||||
results := make(chan result, len(list.Modules))
|
||||
work := make(chan catalogueEntry)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < definitionReaders; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for m := range work {
|
||||
pins, why := c.definition(m, names, nameKey)
|
||||
results <- result{m.Module, pins, why}
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
defer close(work)
|
||||
for _, m := range list.Modules {
|
||||
select {
|
||||
case work <- m:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() { wg.Wait(); close(results) }()
|
||||
|
||||
got := map[string]bool{}
|
||||
collect:
|
||||
for {
|
||||
select {
|
||||
case r, open := <-results:
|
||||
if !open {
|
||||
break collect
|
||||
}
|
||||
got[r.module] = true
|
||||
if r.why != "" {
|
||||
defs.Unread[r.module] = r.why
|
||||
continue
|
||||
}
|
||||
defs.Read++
|
||||
for _, p := range r.pins {
|
||||
k := Key(p.Repository, p.Digest)
|
||||
defs.Pins[k] = append(defs.Pins[k], Pin{Module: r.module, Name: p.Name, Older: p.Older})
|
||||
}
|
||||
case <-ctx.Done():
|
||||
break collect
|
||||
}
|
||||
}
|
||||
for _, m := range list.Modules {
|
||||
if !got[m.Module] {
|
||||
defs.Unread[m.Module] = fmt.Sprintf("not read within %d s", int(DefinitionsTimeout/time.Second))
|
||||
}
|
||||
}
|
||||
for k := range defs.Pins {
|
||||
sort.Slice(defs.Pins[k], func(i, j int) bool { return defs.Pins[k][i].Module < defs.Pins[k][j].Module })
|
||||
}
|
||||
return defs
|
||||
}
|
||||
|
||||
// definition reads one module's definition, or answers it from what was read before.
|
||||
func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey string) ([]pinned, string) {
|
||||
key := m.Module + "@" + m.Commit + "|" + nameKey
|
||||
if m.Commit != "" {
|
||||
c.mu.Lock()
|
||||
pins, had := c.kept[key]
|
||||
c.mu.Unlock()
|
||||
if had {
|
||||
return pins, ""
|
||||
}
|
||||
}
|
||||
args := map[string]any{"module": m.Module}
|
||||
if m.Commit != "" {
|
||||
args["commit"] = m.Commit
|
||||
}
|
||||
raw, err := c.Ask(CatalogueModule, args)
|
||||
if err != nil {
|
||||
return nil, "could not be asked: " + err.Error()
|
||||
}
|
||||
answer, output, ok := answerOf(raw)
|
||||
if !ok {
|
||||
return nil, "refused: " + lastLine(output)
|
||||
}
|
||||
var one struct {
|
||||
Module *struct {
|
||||
Manifest json.RawMessage `json:"manifest"`
|
||||
} `json:"module"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(answer, &one); err != nil || one.Error != "" || one.Module == nil || len(one.Module.Manifest) == 0 {
|
||||
return nil, "not readable: " + firstOf(one.Error, errText(err), "no manifest in the answer")
|
||||
}
|
||||
pins := PinsIn(one.Module.Manifest, names)
|
||||
if m.Commit != "" {
|
||||
c.mu.Lock()
|
||||
c.kept[key] = pins
|
||||
c.mu.Unlock()
|
||||
}
|
||||
return pins, ""
|
||||
}
|
||||
|
||||
// why says which definitions name a manifest, and under which of the store's names.
|
||||
func (p Pin) why() string {
|
||||
if p.Older {
|
||||
return fmt.Sprintf("a definition names it: %s, under %s, an older name of the store", p.Module, shownName(p.Name))
|
||||
}
|
||||
return "a definition names it: " + p.Module
|
||||
}
|
||||
|
||||
func firstOf(s ...string) string {
|
||||
for _, x := range s {
|
||||
if strings.TrimSpace(x) != "" {
|
||||
return x
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func errText(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
// shownName is a name of the store as a person reads it: a host without the slash a pin puts after it, the
|
||||
// scheme as written.
|
||||
func shownName(name string) string {
|
||||
if strings.HasSuffix(name, "://") {
|
||||
return name
|
||||
}
|
||||
return strings.TrimSuffix(name, "/")
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A definition may pin an image of the store the mesh did not make, under a name the store had before
|
||||
// (novox/hq issue 323): three apps the anchor runs are pinned so. No record names them; a definition does.
|
||||
|
||||
const oldName = "registry-api.example.org"
|
||||
|
||||
// container is a definition whose one container runs image.
|
||||
func container(image string) string {
|
||||
return fmt.Sprintf(`{"module":"x","resources":[{"id":"server","type":"container","image":%q}]}`, image)
|
||||
}
|
||||
|
||||
func TestAnImageADefinitionPinsUnderAnOlderNameOfTheStoreIsNamedByADefinition(t *testing.T) {
|
||||
v := view(t, world())
|
||||
names := NamesOf("store:5000", oldName)
|
||||
defs := &Definitions{Pins: map[string][]Pin{}}
|
||||
for module, manifest := range map[string]string{
|
||||
// m-stranger: no record names it; the invoicing app pins it under the store's older name.
|
||||
"invoicing": container(oldName + "/app/server@" + d("m-stranger")),
|
||||
// m-old: the records say eligible; a definition pinning it does not change what the records say.
|
||||
"website": container("store:5000/app/server@" + d("m-old")),
|
||||
} {
|
||||
for _, p := range PinsIn(json.RawMessage(manifest), names) {
|
||||
defs.Pins[Key(p.Repository, p.Digest)] = append(defs.Pins[Key(p.Repository, p.Digest)], Pin{Module: module, Name: p.Name, Older: p.Older})
|
||||
}
|
||||
}
|
||||
got := stateByName(Classify(v, withFill(records()), defs))
|
||||
stranger := got["app/server "+d("m-stranger")]
|
||||
if stranger.State != StateNamedByDefinition {
|
||||
t.Fatalf("a manifest a definition pins under an older name reads %q, want %q", stranger.State, StateNamedByDefinition)
|
||||
}
|
||||
if want := []string{"a definition names it: invoicing, under " + oldName + ", an older name of the store"}; !reflect.DeepEqual(stranger.Why, want) {
|
||||
t.Errorf("why %q, want %q", stranger.Why, want)
|
||||
}
|
||||
if s := got["app/server "+d("m-old")].State; s != StateEligible {
|
||||
t.Errorf("a recorded manifest a definition pins reads %q: the records decide what the mesh made", s)
|
||||
}
|
||||
// Without the definitions it is what it was: unrecorded.
|
||||
if s := stateByName(Classify(v, withFill(records()), nil))["app/server "+d("m-stranger")].State; s != StateUnrecorded {
|
||||
t.Errorf("without definitions %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPinsAreFoundInEveryStringAndOnlyUnderTheStoresNames(t *testing.T) {
|
||||
names := NamesOf("store:5000", oldName+", older.example/")
|
||||
manifest := `{"resources":[
|
||||
{"type":"container","image":"` + oldName + `/novox/www@` + d("www") + `"},
|
||||
{"type":"file","content":"IMAGE=older.example/novox/tool@` + d("tool") + `\n"},
|
||||
{"type":"container","image":"artifact-store://mod/art@` + d("built") + `"},
|
||||
{"type":"container","image":"store:5000/mod/art@` + d("addressed") + `"},
|
||||
{"type":"container","image":"not` + oldName + `/novox/www@` + d("lookalike") + `"},
|
||||
{"type":"container","image":"docker.io/library/nginx@` + d("upstream") + `"},
|
||||
{"type":"container","image":"` + oldName + `/novox/www:latest"}],
|
||||
"build":{"on":{"base":"` + oldName + `/novox/www@` + d("www") + `"}}}`
|
||||
var got []string
|
||||
for _, p := range PinsIn(json.RawMessage(manifest), names) {
|
||||
got = append(got, fmt.Sprintf("%s@%s %s %v", p.Repository, p.Digest, p.Name, p.Older))
|
||||
}
|
||||
want := map[string]bool{
|
||||
"novox/www@" + d("www") + " " + oldName + "/ true": true,
|
||||
"novox/tool@" + d("tool") + " older.example/ true": true,
|
||||
"mod/art@" + d("built") + " artifact-store:// false": true,
|
||||
"mod/art@" + d("addressed") + " store:5000/ false": true,
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("found %d pins, want %d (one per image and name; not a longer host's tail, not another registry, not a tag): %v", len(got), len(want), got)
|
||||
}
|
||||
for _, g := range got {
|
||||
if !want[g] {
|
||||
t.Errorf("found %s", g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPlatformTakesTheStateOfAnIndexADefinitionNames(t *testing.T) {
|
||||
v := mirrorWorld()
|
||||
defs := &Definitions{Pins: map[string][]Pin{
|
||||
Key("route-proxy/on-go_base", d("i-pre")): {{Module: "route-proxy", Name: oldName + "/", Older: true}},
|
||||
}}
|
||||
got := stateByName(Classify(v, withFill(mirrorRecords()), defs))
|
||||
for _, name := range []string{"i-pre", "p-pre"} {
|
||||
if s := got["route-proxy/on-go_base "+d(name)].State; s != StateNamedByDefinition {
|
||||
t.Errorf("%s reads %q, want %q", name, s, StateNamedByDefinition)
|
||||
}
|
||||
}
|
||||
if e := got["route-proxy/on-go_base "+d("p-pre")]; e.Index != d("i-pre") {
|
||||
t.Errorf("p-pre takes its state from %q", e.Index)
|
||||
}
|
||||
if s := got["route-proxy/on-go_base "+d("p-lost")].State; s != StateUnrecorded {
|
||||
t.Errorf("a platform nothing names reads %q", s)
|
||||
}
|
||||
for _, um := range UnrecordedMirrors(v, Classify(v, withFill(mirrorRecords()), defs)) {
|
||||
if strings.Contains(um.Reference, hexOf(d("i-pre"))) {
|
||||
t.Errorf("a copy a definition names is offered to the mirrors verb: %v", um)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// olderNames writes the file the mesh writes from the setting older-store-names.
|
||||
func olderNames(t *testing.T, names string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "store-names")
|
||||
if err := os.WriteFile(path, []byte("# written by the mesh\nolder="+names+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(NamesFileEnv, path)
|
||||
}
|
||||
|
||||
func TestTheOlderNamesAreReadFromTheFileTheMeshWrites(t *testing.T) {
|
||||
olderNames(t, oldName+" older.example")
|
||||
got, why := OlderNames()
|
||||
if why != "" || !reflect.DeepEqual(NamesOf("", got).Older, []string{oldName + "/", "older.example/"}) {
|
||||
t.Errorf("read %q (%s)", got, why)
|
||||
}
|
||||
olderNames(t, "none")
|
||||
if got, why := OlderNames(); why != "" || len(NamesOf("", got).Older) != 0 {
|
||||
t.Errorf("none read as %q (%s)", got, why)
|
||||
}
|
||||
t.Setenv(NamesFileEnv, filepath.Join(t.TempDir(), "absent"))
|
||||
if _, why := OlderNames(); why == "" {
|
||||
t.Error("an absent file was not said")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOlderNamesNotReadAreSaidFirst(t *testing.T) {
|
||||
t.Setenv(NamesFileEnv, "")
|
||||
a := &asked{definitions: map[string]string{"invoicing": container(oldName + "/app/server@" + d("m-stranger"))}}
|
||||
out, err := references(t, a)(map[string]any{"repository": "app/server"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := out.(map[string]any)
|
||||
if w := m["said"].([]string)[0]; !strings.Contains(w, NamesFileEnv) {
|
||||
t.Errorf("first said %q", w)
|
||||
}
|
||||
if s := manifestsOf(t, out)["app/server "+d("m-stranger")].State; s != StateUnrecorded {
|
||||
t.Errorf("m-stranger reads %q without the older names", s)
|
||||
}
|
||||
}
|
||||
|
||||
// references is one artifact_store_references tool, kept across calls as the runtime keeps the bundle.
|
||||
func references(t *testing.T, a *asked) func(map[string]any) (any, error) {
|
||||
f := world()
|
||||
s := &Store{Container: "mesh-registry", Run: f.run(t)}
|
||||
for _, tl := range Tools(s, Controller{Ask: a.ask}, &Catalogue{Ask: a.ask}) {
|
||||
if tl.Name == "artifact_store_references" {
|
||||
return tl.Run
|
||||
}
|
||||
}
|
||||
t.Fatal("no artifact_store_references")
|
||||
return nil
|
||||
}
|
||||
|
||||
func manifestsOf(t *testing.T, out any) map[string]Entry {
|
||||
t.Helper()
|
||||
entries, ok := out.(map[string]any)["manifests"].([]Entry)
|
||||
if !ok {
|
||||
t.Fatalf("no manifests in %v", out)
|
||||
}
|
||||
return stateByName(entries)
|
||||
}
|
||||
|
||||
func TestReferencesReadTheDefinitionsUnderTheStoresOlderNames(t *testing.T) {
|
||||
olderNames(t, oldName)
|
||||
a := &asked{definitions: map[string]string{
|
||||
"invoicing": container(oldName + "/app/server@" + d("m-stranger")),
|
||||
"grafana": container("grafana/grafana@" + d("upstream")),
|
||||
}}
|
||||
run := references(t, a)
|
||||
out, err := run(map[string]any{"repository": "app/server"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := manifestsOf(t, out)["app/server "+d("m-stranger")]
|
||||
if e.State != StateNamedByDefinition || len(e.Why) != 1 || !strings.Contains(e.Why[0], "invoicing") || !strings.Contains(e.Why[0], oldName) {
|
||||
t.Errorf("m-stranger: %+v", e)
|
||||
}
|
||||
said := out.(map[string]any)["definitions"].(map[string]any)
|
||||
if said["read"] != 2 || !reflect.DeepEqual(said["older_store_names"], []string{oldName}) ||
|
||||
!reflect.DeepEqual(said["store_names"], []string{"artifact-store://", "store:5000"}) {
|
||||
t.Errorf("definitions said %v", said)
|
||||
}
|
||||
if w := out.(map[string]any)["said"].([]string)[0]; strings.Contains(w, "could not be read") {
|
||||
t.Errorf("a warning with every definition read: %s", w)
|
||||
}
|
||||
// A definition read at its commit is kept: the next call asks only for the list.
|
||||
before := 0
|
||||
for _, c := range a.calls {
|
||||
if c == CatalogueModule {
|
||||
before++
|
||||
}
|
||||
}
|
||||
if _, err := run(map[string]any{"repository": "app/server"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := 0
|
||||
for _, c := range a.calls {
|
||||
if c == CatalogueModule {
|
||||
after++
|
||||
}
|
||||
}
|
||||
if before != 2 || after != 2 {
|
||||
t.Errorf("asked catalog_module %d times, then %d: each definition at its commit is read once", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADefinitionNotReadIsSaidFirstAndLeavesWhatItPinsUnrecorded(t *testing.T) {
|
||||
olderNames(t, oldName)
|
||||
a := &asked{
|
||||
definitions: map[string]string{"invoicing": container(oldName + "/app/server@" + d("m-stranger")), "grafana": container("x")},
|
||||
failing: map[string]bool{"invoicing": true},
|
||||
}
|
||||
out, err := references(t, a)(map[string]any{"repository": "app/server"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := out.(map[string]any)
|
||||
if s := manifestsOf(t, out)["app/server "+d("m-stranger")].State; s != StateUnrecorded {
|
||||
t.Errorf("m-stranger reads %q with its definition unread", s)
|
||||
}
|
||||
if w := m["said"].([]string)[0]; !strings.Contains(w, "1 definitions could not be read") {
|
||||
t.Errorf("first said %q", w)
|
||||
}
|
||||
if u := m["definitions"].(map[string]any)["unread"].(map[string]string); u["invoicing"] == "" || len(u) != 1 {
|
||||
t.Errorf("unread %v", u)
|
||||
}
|
||||
|
||||
// The catalogue not answering at all: the records are still answered, and the warning stands first.
|
||||
a = &asked{noCatalogue: true}
|
||||
out, err = references(t, a)(map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m = out.(map[string]any)
|
||||
if w := m["said"].([]string)[0]; !strings.Contains(w, "the definitions could not be read") || !strings.Contains(w, "no responders") {
|
||||
t.Errorf("first said %q", w)
|
||||
}
|
||||
if m["definitions"].(map[string]any)["not_asked"] == nil {
|
||||
t.Errorf("definitions %v", m["definitions"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoCatalogueIsSaidAndNothingIsAsked(t *testing.T) {
|
||||
defs := (&Catalogue{}).Read(context.Background(), NamesOf("", ""))
|
||||
if defs.NotAsked == "" || defs.Read != 0 {
|
||||
t.Errorf("%+v", defs)
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,7 @@ func main() {
|
||||
UID: os.Getuid(),
|
||||
}
|
||||
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools.
|
||||
if err := stdio.Serve("", Tools(s, Controller{Ask: stdio.Ask})); err != nil {
|
||||
if err := stdio.Serve("", Tools(s, Controller{Ask: stdio.Ask}, &Catalogue{Ask: stdio.Ask})); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ func TestAPlatformManifestTakesTheStateOfTheIndexNamingIt(t *testing.T) {
|
||||
for i := range recs.References {
|
||||
fill(&recs.References[i])
|
||||
}
|
||||
got := stateByName(Classify(v, recs))
|
||||
got := stateByName(Classify(v, recs, nil))
|
||||
up := "upstream/docker.io/library/golang "
|
||||
for name, want := range map[string]string{
|
||||
up + d("p-amd"): StateEligible, // only the eligible index names it
|
||||
@@ -92,7 +92,7 @@ func TestAPlatformManifestTakesTheStateOfTheIndexNamingIt(t *testing.T) {
|
||||
if len(mirrors) != 2 || !got["route-proxy/on-go_base "+d("i-pre")].Mirror || !got["route-proxy/on-go_base "+d("p-lost")].Mirror {
|
||||
t.Errorf("offered as unrecorded copies: %v", mirrors)
|
||||
}
|
||||
um := UnrecordedMirrors(v, Classify(v, recs))
|
||||
um := UnrecordedMirrors(v, Classify(v, recs, nil))
|
||||
if len(um) != 2 || um[0].Reference != "artifact-store://route-proxy/on-go_base@"+d("i-pre") || um[0].OnlyTheir != 1+1+1000 {
|
||||
t.Errorf("unrecorded copies: %+v", um)
|
||||
}
|
||||
@@ -104,7 +104,7 @@ func TestLettingAnIndexGoTakesItsOwnPlatformsAndSaysWhatStaysAndNamesOne(t *test
|
||||
for i := range recs.References {
|
||||
fill(&recs.References[i])
|
||||
}
|
||||
entries := Classify(v, recs)
|
||||
entries := Classify(v, recs, nil)
|
||||
old := "artifact-store://upstream/docker.io/library/golang@" + d("i-old")
|
||||
gone := LetGoKeys(v, []string{old}, Spared(v, entries))
|
||||
up := "upstream/docker.io/library/golang"
|
||||
|
||||
@@ -215,7 +215,7 @@ func TestTheMarkIsTheCollectorsIncludingAnIndexAndSharedBlobs(t *testing.T) {
|
||||
func TestEveryStateIsGiven(t *testing.T) {
|
||||
v := view(t, world())
|
||||
got := map[string]string{}
|
||||
for _, e := range Classify(v, withFill(records())) {
|
||||
for _, e := range Classify(v, withFill(records()), nil) {
|
||||
got[e.Repository+" "+e.Digest] = e.State
|
||||
}
|
||||
want := map[string]string{
|
||||
@@ -239,7 +239,7 @@ func TestALetGoReferenceStillPresentIsSaidSo(t *testing.T) {
|
||||
v := view(t, world())
|
||||
r := records()
|
||||
r.References = append(r.References, Record{Reference: "artifact-store://app/server@" + d("m-stranger"), State: "collected"})
|
||||
for _, e := range Classify(v, withFill(r)) {
|
||||
for _, e := range Classify(v, withFill(r), nil) {
|
||||
if e.Digest == d("m-stranger") && e.State != StateCollectedPresent {
|
||||
t.Errorf("state %s", e.State)
|
||||
}
|
||||
@@ -270,7 +270,7 @@ func TestBytesFreedCountASharedBlobAsKept(t *testing.T) {
|
||||
|
||||
func TestOnlyTheirBytesLeaveOutWhatAnotherStateMarks(t *testing.T) {
|
||||
v := view(t, world())
|
||||
entries := Classify(v, withFill(records()))
|
||||
entries := Classify(v, withFill(records()), nil)
|
||||
sums := Summarise(v, entries, entries)
|
||||
el := sums[StateEligible]
|
||||
if el.OnlyTheirBytes != 500+v.L.Blobs[d("m-old")] {
|
||||
@@ -284,6 +284,11 @@ type asked struct {
|
||||
calls []string
|
||||
args []map[string]any
|
||||
fail map[string]bool
|
||||
// definitions are the catalogue's modules and their manifests; failing, the modules whose
|
||||
// definition cannot be read; noCatalogue, the catalogue not answering at all.
|
||||
definitions map[string]string
|
||||
failing map[string]bool
|
||||
noCatalogue bool
|
||||
}
|
||||
|
||||
func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
@@ -299,6 +304,31 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
switch key {
|
||||
case "seat:mesh-controller.artifacts":
|
||||
answer = records()
|
||||
case CatalogueModules:
|
||||
if a.noCatalogue {
|
||||
return nil, fmt.Errorf("no responders")
|
||||
}
|
||||
ms := []map[string]any{}
|
||||
for name := range a.definitions {
|
||||
ms = append(ms, map[string]any{"module": name, "commit": "c-" + name, "path": "modules/" + name})
|
||||
}
|
||||
answer = map[string]any{"modules": ms}
|
||||
case CatalogueModule:
|
||||
name, _ := args["module"].(string)
|
||||
if a.failing[name] {
|
||||
return nil, fmt.Errorf("no answer")
|
||||
}
|
||||
if args["commit"] != "c-"+name {
|
||||
return nil, fmt.Errorf("asked %s at %v", name, args["commit"])
|
||||
}
|
||||
manifest, has := a.definitions[name]
|
||||
if !has {
|
||||
answer = map[string]any{"error": "the catalogue holds no " + name}
|
||||
break
|
||||
}
|
||||
// The catalogue's tool answers through the runtime as the protocol's content list.
|
||||
raw, _ := json.Marshal(map[string]any{"module": map[string]any{"module": name, "manifest": json.RawMessage(manifest)}})
|
||||
return json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(raw)}}})
|
||||
case "seat:mesh-controller.collect":
|
||||
if args["confirm"] == "true" {
|
||||
answer = CollectAnswer{LetGo: []string{"artifact-store://app/server@" + d("m-old")}}
|
||||
@@ -315,7 +345,7 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
func tool(t *testing.T, a *asked, name string) func(map[string]any) (any, error) {
|
||||
f := world()
|
||||
s := &Store{Container: "mesh-registry", Run: f.run(t)}
|
||||
for _, tl := range Tools(s, Controller{Ask: a.ask}) {
|
||||
for _, tl := range Tools(s, Controller{Ask: a.ask}, &Catalogue{Ask: a.ask}) {
|
||||
if tl.Name == name {
|
||||
return tl.Run
|
||||
}
|
||||
@@ -386,7 +416,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var served []string
|
||||
for _, tl := range Tools(&Store{}, Controller{}) {
|
||||
for _, tl := range Tools(&Store{}, Controller{}, &Catalogue{}) {
|
||||
if !strings.HasPrefix(tl.Name, "artifact_store_") || tl.Description == "" || tl.Run == nil {
|
||||
t.Errorf("tool %q", tl.Name)
|
||||
}
|
||||
@@ -398,7 +428,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
if !reflect.DeepEqual(served, listed) {
|
||||
t.Fatalf("served %v, manifest %v", served, listed)
|
||||
}
|
||||
want := []string{"seat:mesh-controller.artifacts", "seat:mesh-controller.collect"}
|
||||
want := []string{"seat:mesh-controller.artifacts", "seat:mesh-controller.collect", CatalogueModules, CatalogueModule}
|
||||
if !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes %v", m.Invokes)
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
// Tools are the artifact store's tools (novox/hq ADR 0251 §1). The first three only read; artifact_store_collect
|
||||
// only reads too, unless it is a real run, and then it asks the controller, which decides and deletes.
|
||||
func Tools(s *Store, c Controller) []stdio.Tool {
|
||||
func Tools(s *Store, c Controller, cat *Catalogue) []stdio.Tool {
|
||||
ctx := context.Background()
|
||||
repoArg := map[string]any{"type": "string", "description": "one repository, as the store names it (<module>/<artifact>)"}
|
||||
return []stdio.Tool{
|
||||
@@ -53,7 +53,9 @@ func Tools(s *Store, c Controller) []stdio.Tool {
|
||||
{
|
||||
Name: "artifact_store_references",
|
||||
Description: "What the controller's records say of each manifest the artifact store holds: kept (a definition " +
|
||||
"names it, or one of the five most recent builds of a module the mesh holds), the holder of a kept archive, " +
|
||||
"names it, or one of the five most recent builds of a module the mesh holds), named by a definition (no " +
|
||||
"record names it, and a definition in the catalogue pins it by digest, under the store's current name or " +
|
||||
"an older one: the module is said), the holder of a kept archive, " +
|
||||
"eligible (the mesh made it and keeps it for no reason), the holder of an eligible archive, let go yet present, " +
|
||||
"a named document the controller keeps under a tag, or unrecorded — no record names it. A platform manifest " +
|
||||
"takes the state of the recorded index naming it. Counted and sized by " +
|
||||
@@ -67,11 +69,19 @@ func Tools(s *Store, c Controller) []stdio.Tool {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The definitions are read while the store is: each bounded below the runtime's call limit.
|
||||
read := make(chan *Definitions, 1)
|
||||
go func() {
|
||||
older, why := OlderNames()
|
||||
defs := cat.Read(ctx, NamesOf(recs.Store, older))
|
||||
defs.NamesNotRead = why
|
||||
read <- defs
|
||||
}()
|
||||
v, err := s.View(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return References(v, recs, repo)
|
||||
return References(v, recs, <-read, repo)
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -116,7 +126,7 @@ func Tools(s *Store, c Controller) []stdio.Tool {
|
||||
if err != nil {
|
||||
return map[string]any{"controller": answer, "bytes_not_worked_out": "the records could not be read: " + err.Error()}, nil
|
||||
}
|
||||
return Collected(v, answer, dry, Classify(v, recs)), nil
|
||||
return Collected(v, answer, dry, Classify(v, recs, nil)), nil
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -225,11 +235,11 @@ func Usage(v *View, top int) map[string]any {
|
||||
}
|
||||
|
||||
// References answers artifact_store_references.
|
||||
func References(v *View, recs *Records, only string) (map[string]any, error) {
|
||||
func References(v *View, recs *Records, defs *Definitions, only string) (map[string]any, error) {
|
||||
if only != "" && v.L.Repos[only] == nil {
|
||||
return nil, fmt.Errorf("the store holds no repository %q", only)
|
||||
}
|
||||
entries := Classify(v, recs)
|
||||
entries := Classify(v, recs, defs)
|
||||
byRepo := map[string][]Entry{}
|
||||
for _, e := range entries {
|
||||
byRepo[e.Repository] = append(byRepo[e.Repository], e)
|
||||
@@ -243,7 +253,7 @@ func References(v *View, recs *Records, only string) (map[string]any, error) {
|
||||
}
|
||||
whole := Summarise(v, entries, entries)
|
||||
said := []string{}
|
||||
for _, state := range []string{StateKept, StateHolderKept, StateEligible, StateHolderEligible, StateCollectedPresent, StateNamedDocument, StateUnrecorded} {
|
||||
for _, state := range []string{StateKept, StateNamedByDefinition, StateHolderKept, StateEligible, StateHolderEligible, StateCollectedPresent, StateNamedDocument, StateUnrecorded} {
|
||||
if s := whole[state]; s != nil {
|
||||
said = append(said, fmt.Sprintf("%s: %d manifests marking %s, %s of it marked by nothing else", state, s.Manifests, human(s.Bytes), human(s.OnlyTheirBytes)))
|
||||
}
|
||||
@@ -276,9 +286,57 @@ func References(v *View, recs *Records, only string) (map[string]any, error) {
|
||||
if u := unreadNote(v); u != nil {
|
||||
out["unread"] = u
|
||||
}
|
||||
out["definitions"] = definitionsSaid(defs)
|
||||
if warn := definitionsWarning(defs); warn != "" {
|
||||
// Said first: what it leaves out is a manifest a machine may run, read as unrecorded.
|
||||
out["said"] = append([]string{warn}, said...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// definitionsSaid is how the definitions were read: how many, under which names, and what was not read.
|
||||
func definitionsSaid(defs *Definitions) map[string]any {
|
||||
if defs == nil {
|
||||
return map[string]any{"read": 0, "not_asked": "the definitions were not read"}
|
||||
}
|
||||
var current, older []string
|
||||
for _, n := range defs.Names.Current {
|
||||
current = append(current, shownName(n))
|
||||
}
|
||||
for _, n := range defs.Names.Older {
|
||||
older = append(older, shownName(n))
|
||||
}
|
||||
out := map[string]any{"read": defs.Read, "store_names": nonNil(current), "older_store_names": nonNil(older)}
|
||||
if defs.NotAsked != "" {
|
||||
out["not_asked"] = defs.NotAsked
|
||||
}
|
||||
if defs.NamesNotRead != "" {
|
||||
out["older_store_names_not_read"] = defs.NamesNotRead
|
||||
}
|
||||
if len(defs.Unread) > 0 {
|
||||
out["unread"] = defs.Unread
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// definitionsWarning says, when any definition was not read, that a manifest it pins reads as unrecorded.
|
||||
func definitionsWarning(defs *Definitions) string {
|
||||
switch {
|
||||
case defs == nil:
|
||||
return ""
|
||||
case defs.NotAsked != "":
|
||||
return "the definitions could not be read (" + defs.NotAsked + "), so a manifest a definition pins " +
|
||||
"and no record names reads as unrecorded here: ask again before reading unrecorded as unused"
|
||||
case defs.NamesNotRead != "":
|
||||
return defs.NamesNotRead + ", so a manifest a definition pins under an older name of the store and no " +
|
||||
"record names reads as unrecorded here"
|
||||
case len(defs.Unread) > 0:
|
||||
return fmt.Sprintf("%d definitions could not be read (listed in definitions.unread), so a manifest one "+
|
||||
"of them pins and no record names reads as unrecorded here: ask again — what was read is kept", len(defs.Unread))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Collected answers artifact_store_collect: the controller's answer, and what the store's collector frees.
|
||||
func Collected(v *View, a *CollectAnswer, dry bool, entries []Entry) map[string]any {
|
||||
refs := a.LetGo
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
"version": "1",
|
||||
"invokes": [
|
||||
"seat:mesh-controller.artifacts",
|
||||
"seat:mesh-controller.collect"
|
||||
"seat:mesh-controller.collect",
|
||||
"mesh-catalog.catalog_modules",
|
||||
"mesh-catalog.catalog_module"
|
||||
],
|
||||
"tools": [
|
||||
"artifact_store_repositories",
|
||||
@@ -11,6 +13,29 @@
|
||||
"artifact_store_references",
|
||||
"artifact_store_collect"
|
||||
],
|
||||
"settings": {
|
||||
"older-store-names": {
|
||||
"kind": "preference",
|
||||
"default": "none",
|
||||
"why": "the names the artifact store had before, as a definition may still pin an image under them: host names or host:port, separated by spaces; none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323)"
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": "."
|
||||
},
|
||||
{
|
||||
"id": "store-names",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/store-names",
|
||||
"mode": "0600",
|
||||
"trusted": false,
|
||||
"content": "# The names the artifact store had before, from the setting older-store-names, written by the mesh.\n# artifact_store_references reads every definition under these too (novox/hq issue 323).\n# Replaced on every push; change the setting, never this file.\nolder=${setting:older-store-names}\n"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
@@ -24,7 +49,8 @@
|
||||
"artifact-store-tools"
|
||||
],
|
||||
"env": {
|
||||
"MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry"
|
||||
"MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry",
|
||||
"MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user