diff --git a/.gitignore b/.gitignore index 6905fda5..ada6bda3 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image # binaries a local go build leaves in a module root /checks/paths/paths +/checks/environment/environment /modules/mesh-build-go/mesh-build-go /modules/mesh-delivery-gate/mesh-delivery-gate modules/mesh-config/cmd/mesh-config/mesh-config diff --git a/checks/environment/go.mod b/checks/environment/go.mod new file mode 100644 index 00000000..1ef96674 --- /dev/null +++ b/checks/environment/go.mod @@ -0,0 +1,3 @@ +module git.novox.be/novox/mesh-catalog/checks/environment + +go 1.22 diff --git a/checks/environment/main.go b/checks/environment/main.go new file mode 100644 index 00000000..e8cd65a6 --- /dev/null +++ b/checks/environment/main.go @@ -0,0 +1,252 @@ +// Command environment refuses a module's code reading a MESH_* word of its environment that nothing gives it: not its +// manifest, and not the node-engine or the tool runner for every process (the lead's ruling on A3n, issue 1966). +// +// A MESH_* word is the mesh's contract with a module's process: the manifest sets it (an `env` of a build artifact or a +// resource, an `environment`'s `variables`, a `NAME=` line of a file the module ships for an env-file), or the mesh gives +// it to every process (Given). A word read and set by nothing is a contract nobody keeps: the program runs on its +// default or on nothing, and says so to nobody (MESH_REGISTRY_REACH answered every published reference without its +// registry; MESH_CONFIG_CALLERS held every window shut). Found here, it is refused: the code reads what the mesh gives +// (a manifest word, a config value through its tools, a seat's name), or the manifest sets it. +// +// A read is `os.Getenv` or `os.LookupEnv` of a literal or of a constant of the same directory (Go), and `env.MESH_*` or +// `env["MESH_*"]` (TypeScript, `process.env` included); tests, test directories, node_modules and dist are not read. The +// modules still reading such a word are listed in the file `unset` beside this check, ` ` a line, until +// each is fixed: a line no longer found is refused, so the list only shrinks. +// +// go run . +// +// Exits 1 on a finding. +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// Given is every MESH_* word the mesh gives a module's processes whatever their manifest says, and who gives it. +var Given = map[string]string{ + // The tool runner, to every bundle and to a module's own runtime (mesh-tools node-tools runtime.BundleEnv). + "MESH_MODULE": "the tool runner", + "MESH_SERVED_MODULE": "the tool runner", + "MESH_NODE": "the tool runner", + // The node-engine, to the tool runner it runs as the operator's account; its bundles inherit them (mesh-host + // compose/resolution runtime.go). + "MESH_OPERATOR_ACCOUNT": "the node-engine", + "MESH_OPERATOR_HOME": "the node-engine", + // The node-engine, to every module that yields to the node's builds (ADR 0553 §3; catalogue.BuildCPUShareWord). + "MESH_BUILD_CPU_SHARE": "the node-engine", +} + +const listFile = "unset" + +var ( + goRead = regexp.MustCompile(`os\.(?:Getenv|LookupEnv)\(\s*(?:"(MESH_[A-Z0-9_]+)"|([A-Za-z_][A-Za-z0-9_]*))\s*\)`) + goConst = regexp.MustCompile(`\b([A-Za-z_][A-Za-z0-9_]*)\s*(?:string\s*)?=\s*"(MESH_[A-Z0-9_]+)"`) + tsRead = regexp.MustCompile(`\benv(?:\.|\[\s*["'])(MESH_[A-Z0-9_]+)`) + envLine = regexp.MustCompile(`(?m)^\s*(?:export\s+)?(MESH_[A-Z0-9_]+)=`) +) + +// skipped are the directories whose code no process of the module runs. +var skipped = map[string]bool{"node_modules": true, "dist": true, "test": true, "testdata": true, ".git": true} + +// Check answers, per module, each MESH_* word its code reads that its manifest does not set and the mesh does not +// give, with the files reading it. +func Check(root string) (map[string]map[string][]string, error) { + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + sort.Strings(manifests) + out := map[string]map[string][]string{} + for _, path := range manifests { + dir := filepath.Dir(path) + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var manifest any + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + set := map[string]bool{} + setBy(manifest, set) + reads := map[string]map[string]bool{} + consts := map[string]map[string]string{} // directory -> constant -> word + var goFiles []string + err = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if p != dir && skipped[d.Name()] { + return filepath.SkipDir + } + return nil + } + name := d.Name() + text, err := os.ReadFile(p) + if err != nil { + return err + } + rel, _ := filepath.Rel(dir, p) + rel = filepath.ToSlash(rel) + switch { + case strings.HasSuffix(name, "_test.go"), strings.HasSuffix(name, ".test.ts"), strings.HasSuffix(name, ".d.ts"): + case strings.HasSuffix(name, ".go"): + goFiles = append(goFiles, p) + c := consts[filepath.Dir(p)] + if c == nil { + c = map[string]string{} + consts[filepath.Dir(p)] = c + } + for _, m := range goConst.FindAllStringSubmatch(string(text), -1) { + c[m[1]] = m[2] + } + case strings.HasSuffix(name, ".ts"): + for _, m := range tsRead.FindAllStringSubmatch(string(text), -1) { + read(reads, m[1], rel) + } + case name != "module.json": + // A file the module ships: a NAME= line is a word an env-file gives. + for _, m := range envLine.FindAllStringSubmatch(string(text), -1) { + set[m[1]] = true + } + } + return nil + }) + if err != nil { + return nil, err + } + for _, p := range goFiles { + text, _ := os.ReadFile(p) + rel, _ := filepath.Rel(dir, p) + for _, m := range goRead.FindAllStringSubmatch(string(text), -1) { + word := m[1] + if word == "" { + word = consts[filepath.Dir(p)][m[2]] + } + if word != "" { + read(reads, word, filepath.ToSlash(rel)) + } + } + } + module := filepath.Base(dir) + for word, files := range reads { + if set[word] || Given[word] != "" { + continue + } + if out[module] == nil { + out[module] = map[string][]string{} + } + for f := range files { + out[module][word] = append(out[module][word], f) + } + sort.Strings(out[module][word]) + } + } + return out, nil +} + +func read(reads map[string]map[string]bool, word, file string) { + if reads[word] == nil { + reads[word] = map[string]bool{} + } + reads[word][file] = true +} + +// setBy collects the words a manifest sets: every key of an `env` object and of an `environment`'s `variables`. +func setBy(v any, set map[string]bool) { + switch v := v.(type) { + case map[string]any: + for k, e := range v { + if m, ok := e.(map[string]any); ok && (k == "env" || k == "variables") { + for word := range m { + set[word] = true + } + } + setBy(e, set) + } + case []any: + for _, e := range v { + setBy(e, set) + } + } +} + +// Listed is the list's ` ` lines; a line starting with # and a blank line are not entries. +func Listed(path string) (map[string]bool, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + out := map[string]bool{} + s := bufio.NewScanner(f) + for s.Scan() { + line := strings.Join(strings.Fields(s.Text()), " ") + if line == "" || strings.HasPrefix(line, "#") { + continue + } + out[line] = true + } + return out, s.Err() +} + +// Judge answers the reads not on the list (refused) and the list's lines no read stands for (stale). +func Judge(found map[string]map[string][]string, listed map[string]bool) (refused, stale []string) { + seen := map[string]bool{} + for module, words := range found { + for word := range words { + line := module + " " + word + seen[line] = true + if !listed[line] { + refused = append(refused, line) + } + } + } + for line := range listed { + if !seen[line] { + stale = append(stale, line) + } + } + sort.Strings(refused) + sort.Strings(stale) + return refused, stale +} + +func main() { + root := "." + if len(os.Args) > 1 { + root = os.Args[1] + } + found, err := Check(root) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + listed, err := Listed(listFile) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + refused, stale := Judge(found, listed) + for _, line := range refused { + module, word, _ := strings.Cut(line, " ") + fmt.Printf("%s reads %s (%s), which its manifest does not set and the mesh gives no process: set it in the "+ + "manifest, or read what the mesh gives\n", module, word, strings.Join(found[module][word], ", ")) + } + for _, line := range stale { + fmt.Printf("checks/environment/%s names %q, which no code reads unset any more: take it off the list\n", listFile, line) + } + if len(refused) > 0 || len(stale) > 0 { + os.Exit(1) + } + fmt.Printf("environment: every MESH_* word a module reads is set by its manifest or given by the mesh (%d still listed)\n", len(listed)) +} diff --git a/checks/environment/main_test.go b/checks/environment/main_test.go new file mode 100644 index 00000000..f79842cd --- /dev/null +++ b/checks/environment/main_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +// firstCount is how many lines the list held when it was first written: it only shrinks. +const firstCount = 63 + +func module(t *testing.T, root, name, manifest string, files map[string]string) { + t.Helper() + dir := filepath.Join(root, "modules", name) + for path, text := range map[string]string{"module.json": manifest} { + files[path] = text + } + for path, text := range files { + full := filepath.Join(dir, filepath.FromSlash(path)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } +} + +// A word read and set by nothing is found, in Go (a literal, a constant of the same directory) and in TypeScript; one +// the manifest sets (an env, an environment's variables, an env-file line it ships) or the mesh gives every process is +// not; tests, node_modules and dist are not read. +func TestAWordReadThatNothingSetsIsFound(t *testing.T) { + root := t.TempDir() + module(t, root, "reader", `{"module":"reader","build":{"artifacts":[{"name":"tools","env":{"MESH_SET":"x"}}]},`+ + `"resources":[{"id":"p","type":"process","environment":{"variables":{"MESH_VARIABLE":"y"}}}]}`, map[string]string{ + "cmd/a/main.go": `package main; const reach = "MESH_REACH"; var _ = os.Getenv(reach); var _ = os.Getenv("MESH_SET")`, + "cmd/a/other.go": `package main; var _, _ = os.LookupEnv("MESH_LITERAL"); var _ = os.Getenv("MESH_MODULE")`, + "cmd/a/write.go": `package main; var _ = []string{"MESH_WRITTEN_FOR_A_CHILD=1"}`, + "cmd/a/a_test.go": `package main; var _ = os.Getenv("MESH_IN_A_TEST")`, + "client.ts": `const a = process.env.MESH_TS ?? "d"; const b = env["MESH_BRACKET"]; const c = env.MESH_VARIABLE;`, + "files/env": "MESH_FROM_FILE=${config:x}\n", + "index.ts": `const f = env.MESH_FROM_FILE;`, + "test/x.test.ts": `env.MESH_IN_A_TEST_DIR`, + "node_modules/x.ts": `env.MESH_VENDORED`, + }) + module(t, root, "quiet", `{"module":"quiet"}`, map[string]string{"main.go": `package main; var _ = os.Getenv("MESH_NODE")`}) + got, err := Check(root) + if err != nil { + t.Fatal(err) + } + want := map[string]map[string][]string{"reader": { + "MESH_REACH": {"cmd/a/main.go"}, + "MESH_LITERAL": {"cmd/a/other.go"}, + "MESH_TS": {"client.ts"}, + "MESH_BRACKET": {"client.ts"}, + }} + if !reflect.DeepEqual(got, want) { + t.Fatalf("found %v, want %v", got, want) + } + refused, stale := Judge(got, map[string]bool{"reader MESH_REACH": true, "reader MESH_GONE": true}) + if !reflect.DeepEqual(refused, []string{"reader MESH_BRACKET", "reader MESH_LITERAL", "reader MESH_TS"}) || + !reflect.DeepEqual(stale, []string{"reader MESH_GONE"}) { + t.Errorf("refused %v, stale %v", refused, stale) + } +} + +// The catalogue itself: every word a module reads is set or given, but the list's, and the list only shrinks. +func TestTheCatalogueReadsOnlyWordsTheMeshSets(t *testing.T) { + found, err := Check(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + listed, err := Listed(listFile) + if err != nil { + t.Fatal(err) + } + refused, stale := Judge(found, listed) + for _, line := range refused { + t.Errorf("%s: read, set by nothing, and not on checks/environment/unset", line) + } + for _, line := range stale { + t.Errorf("checks/environment/unset names %q, which no code reads unset any more", line) + } + if len(listed) > firstCount { + t.Errorf("checks/environment/unset holds %d lines, more than its first %d: it only shrinks", len(listed), firstCount) + } +} diff --git a/checks/environment/reads-outside b/checks/environment/reads-outside new file mode 100644 index 00000000..e1553863 --- /dev/null +++ b/checks/environment/reads-outside @@ -0,0 +1,5 @@ +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest and the code and files of its module. +modules diff --git a/checks/environment/unset b/checks/environment/unset new file mode 100644 index 00000000..5840f12b --- /dev/null +++ b/checks/environment/unset @@ -0,0 +1,67 @@ +# The MESH_* words a module's code still reads that its manifest does not set and the mesh gives no process, +# ` ` a line, as checks/environment first found them (issue 1966). Each is a contract nobody keeps: fix +# it (the manifest sets it, or the code reads what the mesh gives) and take its line off. A line no code reads unset any +# more is refused, so this list only shrinks; a new read set by nothing is refused, never added here. +baserow MESH_BASEROW_EMAIL +baserow MESH_BASEROW_HOST +baserow MESH_BASEROW_PASSWORD +claude-code MESH_AGENT_ACCOUNT +claude-code MESH_AGENT_HOME +claude-licence-manager MESH_ANTHROPIC_TOKEN_ENDPOINT +claude-licence-manager MESH_ANTHROPIC_USAGE_ENDPOINT +confluence MESH_CONFLUENCE_EMAIL +confluence MESH_CONFLUENCE_TOKEN +confluence MESH_CONFLUENCE_URL +disk-load MESH_DISK_MEASURE_EVERY +gitlab MESH_GITLAB_TOKEN +gitlab MESH_GITLAB_URL +grafana MESH_GRAFANA_PASSWORD +grafana MESH_GRAFANA_TOKEN +grafana MESH_GRAFANA_USER +home-assistant MESH_HOMEASSISTANT_TOKEN +home-assistant MESH_HOMEASSISTANT_WAIT_SECONDS +home-assistant MESH_HOMEASSISTANT_WATCH +icecast MESH_ICECAST_ADMIN_PASSWORD +icecast MESH_ICECAST_ADMIN_USER +influxdb MESH_INFLUXDB_ORG +influxdb MESH_INFLUXDB_TOKEN +jira MESH_JIRA_EMAIL +jira MESH_JIRA_TOKEN +jira MESH_JIRA_URL +letta MESH_LETTA_PASSWORD +mailu MESH_MAILU_API_KEY +mailu MESH_MAILU_DOMAIN +mesh-cli MESH_CONSOLE +mesh-cli MESH_VERB +mesh-files MESH_OWNED_PATHS +mesh-files MESH_SECRET_PLACES +mesh-reads MESH_SECRET_PLACES +minio MESH_MINIO_ROOT_PASSWORD +mongodb MESH_MONGODB_AUTHSOURCE +mongodb MESH_MONGODB_HOST +mongodb MESH_MONGODB_PASSWORD +mongodb MESH_MONGODB_PORT +mongodb MESH_MONGODB_USER +mosquitto MESH_MQTT_ADMIN_USER +mosquitto MESH_MQTT_BROKER_GID +mosquitto MESH_MQTT_BROKER_UID +mosquitto MESH_MQTT_HOST +mosquitto MESH_MQTT_PASSWORD +mosquitto MESH_MQTT_PORT +mssql MESH_MSSQL_HOST +mssql MESH_MSSQL_PASSWORD +mssql MESH_MSSQL_PORT +mssql MESH_MSSQL_READER_PASSWORD +mssql MESH_MSSQL_USER +nextcloud MESH_NEXTCLOUD_ADMIN_PASSWORD +nextcloud MESH_NEXTCLOUD_CONTAINER +nodered MESH_NODERED_TOKEN +nodered MESH_NODERED_WAIT_SECONDS +redis MESH_REDIS_HOST +redis MESH_REDIS_PASSWORD +redis MESH_REDIS_PORT +redis MESH_REDIS_USERNAME +restic MESH_BACKUP_HOUR +unifi MESH_UNIFI_PASSWORD +unifi MESH_UNIFI_SITE +unifi MESH_UNIFI_USERNAME diff --git a/modules/artifact-store-tools/README.md b/modules/artifact-store-tools/README.md index 5a38db8f..bce43f72 100644 --- a/modules/artifact-store-tools/README.md +++ b/modules/artifact-store-tools/README.md @@ -115,3 +115,13 @@ Any command other than the two read-only scripts fails the test. Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416). - `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). + +### The sweep through the registry seats + +The sweep writes nothing to the store's door (CAT-AS1, ADR 0617 §2): it asks the registry seats the store's holder +serves. Each kept bundle or archive is held on `oci-artifact-registry` and each kept copy of an upstream base on +`container-image-registry` (`hold`, with why it is kept) before anything goes; each eligible one is released, then +removed for `retention` on its seat (`release`, `remove`). It reads a reference in both forms, `artifact-store://` +and `//@`, as one artifact, and says artifacts-gone in the form the build named it. A +call a seat refuses stops the sweep and writes the finding `sweep-refused` in the module's findings; the next sweep no +seat refuses clears it (`sweepseats.go`). diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/door.go b/modules/artifact-store-tools/cmd/artifact-store-tools/door.go index c34b3bcb..7e7177c3 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/door.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/door.go @@ -21,20 +21,12 @@ import ( // door is registry.Store at the store's address as this machine reaches it. type door struct{ s registry.Store } -func (d *door) Address() string { return d.s.Address } -func (d *door) Hold(ctx context.Context, r string) (bool, error) { return d.s.Hold(ctx, r) } -func (d *door) Holds(ctx context.Context, r string) error { return d.s.Holds(ctx, r) } +// It is read only: what the sweep holds and removes it asks the registry seats (sweepseats.go). +func (d *door) Address() string { return d.s.Address } +func (d *door) Holds(ctx context.Context, r string) error { return d.s.Holds(ctx, r) } func (d *door) IsIndex(ctx context.Context, r string) (bool, error) { return d.s.IsIndex(ctx, r) } -func (d *door) LetGo(ctx context.Context, r string) error { return d.s.LetGo(ctx, r) } -func (d *door) SpareKept(ctx context.Context, keptImages, touching []string) error { - spare, err := d.s.SpareKeptIndexes(ctx, keptImages, touching) - if err == nil { - d.s.Spare = spare - } - return err -} // DoorAt reads the store's address (host:port) from the file the manifest fills from the seat mesh-artifact-store's // reach, each time, so a store that moved is reached where it is now. diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go index ef742241..2293c86a 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go @@ -71,15 +71,12 @@ type Bus interface { Publish(subject string, body any) error } -// Door is the artifact store's own door: registry.Store, behind an interface so a test needs no store. +// Door is the artifact store's own door, read only: registry.Store, behind an interface so a test needs no store. What +// the sweep holds and removes it asks the registry seats (Seats, sweepseats.go). type Door interface { Address() string - Hold(ctx context.Context, reference string) (bool, error) Holds(ctx context.Context, reference string) error IsIndex(ctx context.Context, reference string) (bool, error) - LetGo(ctx context.Context, reference string) error - // SpareKept sets the platform manifests a kept index names aside before an index is let go of (ADR 0257). - SpareKept(ctx context.Context, keptImages, touching []string) error } // Holder is the seat's holder. @@ -89,8 +86,10 @@ type Holder struct { // Listed is the store's own listing now (Store.View); nil reads none, and then nothing is said gone or collected. Listed func(ctx context.Context) (*View, error) Acts handacts.Recorder - Now func() time.Time - Logf func(string, ...any) + // Seats asks a registry seat's verb, `seat:.` (hold, remove): the runtime's stdio.Ask in main. + Seats func(key string, body any) (json.RawMessage, error) + Now func() time.Time + Logf func(string, ...any) sweep sync.Mutex // one sweep at a time mu sync.Mutex @@ -219,7 +218,7 @@ func KeepSet(builds []Build, wants []Want) map[string]*Ref { out := map[string]*Ref{} for _, b := range builds { for _, a := range b.Artifacts { - reference := catalogue.Recorded(a.Reference) + reference := canonical(catalogue.Recorded(a.Reference)) if _, ours := catalogue.InArtifactStore(reference); !ours || a.Kind == "package" { continue } @@ -513,13 +512,13 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error) return nil, err } a := &CollectAnswer{DryRun: s.DryRun, WouldLetGo: []string{}, LetGo: []string{}} - var keptArchives, keptImages, eligible []string + var keptArchives, eligible []string for _, r := range j.sorted() { switch { - case r.Kept && r.Kind == "archive": + case r.Kept && (r.Kind == "archive" || isCopy(r.Reference)): + // What retention would remove unless held: a bundle or an archive, and a copied base (ADR 0617 §2). keptArchives = append(keptArchives, r.Reference) case r.Kept: - keptImages = append(keptImages, r.Reference) default: if held, known := inStore(j.view, r.Reference); known && !held { continue // collected already @@ -545,33 +544,25 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error) a.Store = door.Address() within, stop := context.WithTimeout(ctx, SweepBudget) defer stop() - // Hold before letting go (ADR 0251 §8): a kept archive that cannot be held stops the sweep before any delete. + // Hold before letting go (ADR 0251 §8, ADR 0617 §2): each kept archive and copied base is held on its seat; one that + // cannot be held stops the sweep before any removal and writes the sweep's finding. var missing []string for _, r := range keptArchives { - wrote, err := door.Hold(within, r) - switch { - case err == nil: - if wrote { - a.HoldersWritten++ - } - case errors.Is(err, registry.Gone): + if held, known := inStore(j.view, r); known && !held { missing = append(missing, r) - case errors.Is(err, registry.ErrNotOurs): - default: + continue + } + if _, err := h.holdOnSeat(r, keptWhy(j.refs[r])); err != nil { a.Stopped = fmt.Sprintf("not every kept archive could be held, so nothing was let go: holding %s: %v", r, err) a.Left = len(eligible) + h.sweepFinding("the hold of "+r, err) return a, nil } + a.HoldersWritten++ } a.Held, a.Missing = len(keptArchives)-len(missing), len(missing) h.sayGone(j, missing, "a kept archive the store does not hold") - if s.Confirmed { - if err := door.SpareKept(within, keptImages, eligible); err != nil { - a.Stopped = fmt.Sprintf("which platform manifests a kept index names could not be read, so nothing was let go: %v", err) - a.Left = len(eligible) - return a, nil - } - } + refused := false for i, r := range eligible { if i >= s.Most || within.Err() != nil { a.Left = len(eligible) - i @@ -595,24 +586,19 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error) continue } } - err := door.LetGo(within, r) - switch { - case err == nil || errors.Is(err, registry.Gone): - case errors.Is(err, registry.ErrNotOurs): - a.Skipped++ - continue - default: - a.Stopped = fmt.Sprintf("the store refused to let go of %s, so nothing more was asked: %v", r, err) - a.Left = len(eligible) - i - } - if a.Stopped != "" { - break - } - // Recorded in the hand-act log, each one (ADR 0565 §10; the operator's "option A"). + // Removed through its seat (ADR 0617 §2), recorded in the hand-act log, each one (ADR 0565 §10; the operator's + // "option A"). why := s.Why if why == "" { why = "eligible: no want names it and it is not among its module's " + fmt.Sprint(KeptBuilds) + " newest builds" } + if err := h.removeOnSeat(r, why); err != nil { + a.Stopped = fmt.Sprintf("a registry seat refused to remove %s, so nothing more was asked: %v", r, err) + a.Left = len(eligible) - i + h.sweepFinding("the removal of "+r, err) + refused = true + break + } if _, err := h.Acts.Record(handacts.Request{What: "let go of " + r + " (" + s.By + ")", Why: why, Cause: "artifact-store-sweep", Verb: "collect"}); err != nil { h.logf("%s was let go of and could not be recorded in the hand-act log: %v", r, err) @@ -620,6 +606,9 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error) a.LetGo = append(a.LetGo, r) } h.sayGone(j, a.LetGo, "let go of") + if !refused { + h.sweepClean() + } return a, nil } @@ -639,7 +628,7 @@ func (h *Holder) sayGone(j *judged, refs []string, how string) { byKey[b.Key] = g keys = append(keys, b.Key) } - g.Gone = append(g.Gone, r) + g.Gone = append(g.Gone, namedBy(b, r)) } } sort.Strings(keys) @@ -654,6 +643,17 @@ func (h *Holder) sayGone(j *judged, refs []string, how string) { } } +// namedBy is the reference b names r by, in the form its key holds (the older one or the seat's): the builder matches +// artifacts-gone against its own key. +func namedBy(b Build, r string) string { + for _, a := range b.Artifacts { + if rec := catalogue.Recorded(a.Reference); canonical(rec) == r { + return rec + } + } + return r +} + // AfterBuild is the sweep at a build's end. func (h *Holder) AfterBuild(key string) { a, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "after the build " + key}) @@ -687,7 +687,7 @@ func (h *Holder) Held(refs []string) (delivery.HeldAnswer, error) { defer stop() a := delivery.HeldAnswer{Store: door.Address(), Held: []string{}, Gone: []string{}} for _, r := range refs { - switch err := door.Holds(asking, r); { + switch err := door.Holds(asking, canonical(r)); { case err == nil: a.Held = append(a.Held, r) case errors.Is(err, registry.Gone): @@ -901,6 +901,9 @@ func (h *Holder) Observe(ctx context.Context) error { return fmt.Errorf("the findings: %w", err) } for _, k := range keys { + if k == sweepFindingKey { + continue // the sweep's own finding, which the sweep clears + } if _, still := now[k]; !still { // A finding of a first view unanswered is the reader's (views.Reader), cleared by its answer, not by a sweep. if raw, found, err := h.Bus.Get(FindingsState, k); err == nil && found { diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go index b97cfd9a..25f8b35d 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go @@ -173,15 +173,32 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) { }); err != nil { t.Fatal(err) } - h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Now: time.Now, Logf: t.Logf} - h.Acts = handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) { + // The registry seats' holder on the bus (CAT-AS1): the sweep holds and removes through their verbs. + for _, seat := range []string{"oci-artifact-registry", "container-image-registry"} { + if _, err := nc.Subscribe("mesh.seat."+seat+".tool.*", func(m *nats.Msg) { + var body map[string]any + _ = json.Unmarshal(m.Data, &body) + mu.Lock() + out, err := door.seat("seat:"+seat+"."+m.Subject[strings.LastIndex(m.Subject, ".")+1:], body) + mu.Unlock() + if err != nil { + out, _ = json.Marshal(map[string]string{"error": err.Error()}) + } + _ = m.Respond(out) + }); err != nil { + t.Fatal(err) + } + } + ask := func(key string, body any) (json.RawMessage, error) { raw, _ := json.Marshal(body) m, err := nc.Request("mesh.seat."+strings.TrimPrefix(strings.Replace(key, ".", ".tool.", 1), "seat:"), raw, 5*time.Second) if err != nil { return nil, err } return m.Data, nil - }} + } + h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Seats: ask, Now: time.Now, Logf: t.Logf} + h.Acts = handacts.Recorder{Ask: ask} // A build's end heard from the builds bucket, as follow hands it over. kv, _ := bus.kv(BuildsRead) diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go index eac183db..37b7d553 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go @@ -3,7 +3,9 @@ package main import ( "context" "encoding/json" + "errors" "fmt" + "slices" "sort" "strings" "sync" @@ -15,6 +17,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/handacts" "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "artifactstoretools/internal/registry" ) @@ -85,21 +89,56 @@ func (b *memBus) Publish(subject string, body any) error { return nil } -// memDoor is the store's door in memory: what it holds, which references are indexes, and what was asked. +// memDoor is the store's door in memory, read only, and the registry seats' holder: what it holds, which references are +// indexes, what the seats were asked (asked: "hold ", "let go "; calls: each seat call as its key and name), the +// holds asked (held), and the calls it refuses (refuse: by " "). type memDoor struct { holds map[string]bool indexes map[string]bool asked []string + calls []string + held map[string]bool + refuse map[string]error +} + +// seat is the registry seats' holder, as Holder.Seats calls it. +func (d *memDoor) seat(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + var in struct{ Name, Digest, Reason string } + _ = json.Unmarshal(raw, &in) + seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".") + ref := catalogue.ArtifactStoreScheme + in.Name + "@" + in.Digest + if seat == "oci-artifact-registry" { + ref = catalogue.ArtifactStoreScheme + in.Name + "/blobs/" + in.Digest + } + d.calls = append(d.calls, key+" "+in.Name+"@"+in.Digest+" "+in.Reason) + if err := d.refuse[verb+" "+ref]; err != nil { + return nil, err + } + if d.held == nil { + d.held = map[string]bool{} + } + switch verb { + case "hold": + d.asked = append(d.asked, "hold "+ref) + d.held[ref] = true + return json.Marshal(map[string]any{"held": true}) + case "release": + released := d.held[ref] // what is not held answers released false, as the seat's definition says + delete(d.held, ref) + return json.Marshal(map[string]any{"released": released}) + case "remove": + if d.held[ref] { + return nil, fmt.Errorf("remove: refused: %s@%s is held; release it first", in.Name, in.Digest) + } + d.asked = append(d.asked, "let go "+ref) + delete(d.holds, ref) + return json.Marshal(map[string]any{"removed": true}) + } + return nil, fmt.Errorf("no verb %s", key) } func (d *memDoor) Address() string { return "10.0.0.5:5000" } -func (d *memDoor) Hold(_ context.Context, r string) (bool, error) { - d.asked = append(d.asked, "hold "+r) - if !d.holds[r] { - return false, registry.Gone - } - return false, nil -} func (d *memDoor) Holds(_ context.Context, r string) error { if !d.holds[r] { return registry.Gone @@ -107,15 +146,6 @@ func (d *memDoor) Holds(_ context.Context, r string) error { return nil } func (d *memDoor) IsIndex(_ context.Context, r string) (bool, error) { return d.indexes[r], nil } -func (d *memDoor) LetGo(_ context.Context, r string) error { - d.asked = append(d.asked, "let go "+r) - delete(d.holds, r) - return nil -} -func (d *memDoor) SpareKept(context.Context, []string, []string) error { - d.asked = append(d.asked, "spare") - return nil -} // mirror is a copy of an upstream base in the store, the n-th. func mirror(n int) string { @@ -194,7 +224,7 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) { want, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp(1), Self: true}) _ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want)) var acts []string - h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil }, + h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil }, Seats: door.seat, Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil }, Now: func() time.Time { return day.Add(48 * time.Hour) }, Acts: handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) { @@ -246,18 +276,43 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T) left int skipped int unlog int + // held is how many kept archives and copies are held (7 when 0); seatForm names letta's 2nd and 3rd builds in the + // seat's form; refuse is the seat call refused, " ". + held int + seatForm bool + refuse string }{ {name: "a build's end", how: "build", letGo: all}, + // ADR 0605 §9: a build that names its artifacts in the seat's form is the same artifact: removed and held under the + // seat's name and digest, and said gone in its own form. + {name: "the seat's form", most: MostPerSweep, letGo: all, seatForm: true}, + // A seat that refuses a removal stops the sweep; the finding sweep-refused is written, and cleared by a clean run. + {name: "a refused removal", most: MostPerSweep, letGo: []string{archiveRef("letta", 2)}, left: 2, refuse: "remove " + imageRef("letta", 2)}, {name: "a confirmed collect", how: "collect", letGo: all}, {name: "a sweep", most: MostPerSweep, letGo: all}, {name: "an index waits", index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1}, {name: "bounded", most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2}, // A kept build the log no longer holds: what it stood on is unknown, so no copy goes. - {name: "a kept build's bases unknown", most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3}, + {name: "a kept build's bases unknown", most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3, held: 8}, } { t.Run(c.name, func(t *testing.T) { h, b, door, acts := aMesh(t) door.indexes[imageRef("letta", 2)] = c.index + if c.held == 0 { + c.held = 7 + } + if c.seatForm { + for _, n := range []int{2, 3} { + key, _ := delivery.BuilderBuildKey("letta", fp(n)) + raw, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []delivery.BuiltArtifact{ + {Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: catalogue.SeatReference("container-image-registry", "letta", "image", digestOf(imageRef("letta", n)))}, + {Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: catalogue.SeatReference("oci-artifact-registry", "letta", "files", digestOf(archiveRef("letta", n)))}}}) + _ = b.Put(BuildsRead, key, json.RawMessage(raw)) + } + } + if c.refuse != "" { + door.refuse = map[string]error{c.refuse: errors.New("the store is read only")} + } if c.unlog > 0 { b.log = append(b.log[:c.unlog-1:c.unlog-1], b.log[c.unlog:]...) } @@ -284,7 +339,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T) if err != nil { t.Fatal(err) } - if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 { + if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != c.held || a.Missing != 0 { t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo) } } @@ -298,8 +353,34 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T) held++ } } - if held != 6 { - t.Errorf("the store was asked %v: the six kept archives are held first", door.asked) + if held != c.held { + t.Errorf("the seats were asked %v: the %d kept archives and copies are held first", door.asked, c.held) + } + // Each removal asks the seat that holds it, by its name and digest, for retention, after a release. + for _, call := range door.calls { + if strings.Contains(call, ".remove ") && !strings.HasSuffix(call, " "+Retention) { + t.Errorf("a removal asked %q, not for retention", call) + } + } + if c.letGo[0] == archiveRef("letta", 2) && !slices.Contains(door.calls, "seat:oci-artifact-registry.remove letta/files@"+digestOf(archiveRef("letta", 2))+" "+Retention) { + t.Errorf("the seats were called %v", door.calls) + } + _, finding, _ := b.Get(FindingsState, sweepFindingKey) + if finding != (c.refuse != "") { + t.Errorf("the sweep's finding is written: %v, want %v", finding, c.refuse != "") + } + if c.refuse != "" { + door.refuse = nil + if _, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "again"}); err != nil { + t.Fatal(err) + } + if _, still, _ := b.Get(FindingsState, sweepFindingKey); still { + t.Error("a clean sweep left the sweep's finding") + } + return + } + if c.seatForm && !strings.Contains(b.published[0], `"container-image-registry/letta/image@`+digestOf(imageRef("letta", 2))+`"`) { + t.Errorf("artifacts-gone does not name the build's own form: %v", b.published) } if len(*acts) != len(c.letGo) || !strings.HasPrefix((*acts)[0], "seat:mesh-hand-acts.hand-act ") || !strings.Contains((*acts)[0], c.letGo[0]) { diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go index 630b78b7..c74935ed 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go @@ -34,7 +34,7 @@ func main() { v.Follow(delivery.AssignerModule, "wants", WantsTool) v.Follow(delivery.IndexerModule, "manifests", ManifestsTool) h := &Holder{Bus: runtimeBus{views: v}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View, - Now: time.Now, Logf: logf} + Seats: stdio.Ask, Now: time.Now, Logf: logf} h.Acts = handacts.Recorder{Ask: stdio.Ask} v.Start(context.Background()) go follow(h, v) diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go index 618543de..1f9b7bec 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go @@ -472,7 +472,10 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) { } // ADR 0571 clause 2: it holds the seat mesh-artifact-sweep and serves its verbs, and calls no verb of the controller's; // it reads no controller state, the lease neither (issue 1126). - want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule, WantsTool, ManifestsTool} + // CAT-AS1: what it holds and removes it asks the two registry seats the artifact store's holder serves. + want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule, WantsTool, ManifestsTool, + "seat:oci-artifact-registry.hold", "seat:oci-artifact-registry.release", "seat:oci-artifact-registry.remove", + "seat:container-image-registry.hold", "seat:container-image-registry.release", "seat:container-image-registry.remove"} if !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes %v", m.Invokes) } diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go b/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go new file mode 100644 index 00000000..ecb4e373 --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go @@ -0,0 +1,147 @@ +package main + +// **The sweep asks the registry seats** (CAT-AS1; novox/hq ADR 0617 §2, design 51): the artifact store's write side is +// its holder's, so the sweep no longer deletes or holds over the store's door. What it keeps from retention it asks the +// seat's `hold` for (a kept bundle or archive on oci-artifact-registry, a kept copy of an upstream base on +// container-image-registry), and what is eligible it asks the seat's `remove` for, with the sweep's reason; both verbs +// are in the manifest's `invokes`, and only this module is on their caller lists. A reference is read in both forms +// (ADR 0605 §9): the older `artifact-store://` and the seat's `//@`; the keep set +// keys each by the older form, so the two meet as one artifact. A call a seat refuses stops the sweep and writes the +// sweep's finding, sweep-refused; the next sweep with no refusal clears it. + +import ( + "encoding/json" + "fmt" + "strings" + + ociseat "git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" + + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" +) + +// SweepRefused is the sweep's finding: a seat refused a hold or a remove the sweep asked. +const SweepRefused = "sweep-refused" + +// sweepFindingKey is the one key of the sweep's finding in the module's findings. +var sweepFindingKey = health.FindingKey(health.ScopeModule, Module, SweepRefused) + +// canonical is a kept reference in the older form, `artifact-store://`, whichever form it came in; another is as +// it was. +func canonical(reference string) string { + path, kept := catalogue.InArtifactStore(reference) + if !kept { + return reference + } + return catalogue.ArtifactStoreScheme + path +} + +// seatOf is the seat, name and digest a kept reference is held under: a bundle or an archive on oci-artifact-registry, +// an image or a copy of an upstream base on container-image-registry. +func seatOf(reference string) (seat, name, digest string, err error) { + path, kept := catalogue.InArtifactStore(reference) + if !kept { + return "", "", "", fmt.Errorf("%s is no reference into the mesh's registries", reference) + } + if name, digest, ok := strings.Cut(path, "/blobs/"); ok { + return ociseat.Seat, name, digest, nil + } + if name, digest, ok := strings.Cut(path, "@"); ok { + return reg.Seat, name, digest, nil + } + return "", "", "", fmt.Errorf("%s names no digest", reference) +} + +// isCopy is whether a kept reference is a copy of an upstream base. +func isCopy(reference string) bool { + path, _ := catalogue.InArtifactStore(reference) + return strings.HasPrefix(path, "upstream/") +} + +// askSeat asks verb of the seat reference is held on, with body built from its name and digest. +func (h *Holder) askSeat(reference, verb string, body func(name, digest string) any, answer any) error { + seat, name, digest, err := seatOf(reference) + if err != nil { + return err + } + if h.Seats == nil { + return fmt.Errorf("this process reaches no registry seat") + } + raw, err := h.Seats("seat:"+seat+"."+verb, body(name, digest)) + if err != nil { + return fmt.Errorf("%s's %s of %s@%s: %w", seat, verb, name, digest, err) + } + if answer != nil && len(raw) > 0 { + if err := json.Unmarshal(raw, answer); err != nil { + return fmt.Errorf("%s's answer to %s: %w", seat, verb, err) + } + } + return nil +} + +// holdOnSeat asks the seat to keep reference from retention, for why. +func (h *Holder) holdOnSeat(reference, why string) (bool, error) { + var a reg.HoldAnswer + err := h.askSeat(reference, reg.VerbHold, func(name, digest string) any { + return reg.HoldInput{Name: name, Digest: digest, Reason: why} + }, &a) + if err == nil && !a.Held { + err = fmt.Errorf("%s: the seat answered it is not held", reference) + } + return a.Held, err +} + +// Retention is the reason of a removal by digest, the sweep's alone (design 51 *Who may call which verb*): the holder +// removes an untagged thing only for it. The seat's definition names it. +const Retention = reg.RemoveInputReasonRetention + +// removeOnSeat lets reference go: a hold an earlier sweep asked while it was kept is released first (the holder +// refuses to remove a held thing; one not held answers released false), then it is removed for retention. why goes with +// the release; the hand act records it. +func (h *Holder) removeOnSeat(reference, why string) error { + if err := h.askSeat(reference, reg.VerbRelease, func(name, digest string) any { + return reg.ReleaseInput{Name: name, Digest: digest, Reason: why} + }, ®.ReleaseAnswer{}); err != nil { + return err + } + return h.askSeat(reference, reg.VerbRemove, func(name, digest string) any { + return reg.RemoveInput{Name: name, Digest: digest, Reason: Retention} + }, ®.RemoveAnswer{}) +} + +// sweepFinding writes the sweep's finding: what the seat refused, and why. +func (h *Holder) sweepFinding(what string, err error) { + at := h.now().UTC() + f := health.Finding{V: health.FindingVersion, Key: sweepFindingKey, Kind: SweepRefused, + Subject: health.Subject{Scope: health.ScopeModule, ID: Module}, + Severity: health.Warning, Resolver: health.ResolverOperator, + Summary: fmt.Sprintf("the artifact sweep stopped: a registry seat refused %s (the evidence names why)", what), + Headline: "The artifact sweep stopped", + Explanation: "The sweep keeps what the mesh still needs and removes the rest through the registry seats. A seat " + + "refused one of its asks, so it stopped before anything more was removed; what it kept stays kept.", + Needs: "read the seat holder's journal for the refusal; the next sweep asks again.", + Resolved: "Resolved: the artifact sweep ran through without a refusal", + Evidence: []health.Evidence{{At: at, Said: fmt.Sprintf("%s: %v", what, err)}}, + Source: "CAT-AS1", Observer: Module, At: at} + if werr := h.Bus.Put(FindingsState, sweepFindingKey, f); werr != nil { + h.logf("the sweep's finding (%s: %v) could not be written: %v", what, err, werr) + } +} + +// sweepClean clears the sweep's finding after a sweep no seat refused. +func (h *Holder) sweepClean() { + if _, found, err := h.Bus.Get(FindingsState, sweepFindingKey); err == nil && found { + if err := h.Bus.Delete(FindingsState, sweepFindingKey); err != nil { + h.logf("the sweep's finding could not be cleared: %v", err) + } + } +} + +// keptWhy is why a kept reference is held, as the seat's hold takes it. +func keptWhy(r *Ref) string { + if r == nil || len(r.Why) == 0 { + return "kept by the artifact sweep" + } + return "kept: " + strings.Join(r.Why, "; ") +} diff --git a/modules/artifact-store-tools/go.mod b/modules/artifact-store-tools/go.mod index f2499370..b0cde750 100644 --- a/modules/artifact-store-tools/go.mod +++ b/modules/artifact-store-tools/go.mod @@ -8,6 +8,8 @@ require ( ) require ( + git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.0 // indirect diff --git a/modules/artifact-store-tools/internal/registry/blob.go b/modules/artifact-store-tools/internal/registry/blob.go deleted file mode 100644 index bedbfa38..00000000 --- a/modules/artifact-store-tools/internal/registry/blob.go +++ /dev/null @@ -1,57 +0,0 @@ -package registry - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "net/http" -) - -// blobBound is the most GetBlob reads: a templates archive is a few kilobytes; anything near this is no archive a -// take-in reads text from. -const blobBound = 64 << 20 - -// GetBlob is the archive a recorded reference names (`artifact-store:///blobs/sha256:`), read whole -// and checked against the digest it names: bytes that hash to another digest are refused, never read (R4e: the -// controller reads a templates archive by the reference of a builder's done key). Gone when the store does not hold -// it; ErrNotOurs for a reference outside the store or one that is not an archive. -func (s Store) GetBlob(ctx context.Context, reference string) ([]byte, error) { - repository, digest, archive, err := s.archive(reference) - if err != nil { - return nil, err - } - if !archive { - return nil, fmt.Errorf("%w: %s is not an archive", ErrNotOurs, reference) - } - request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "blobs", digest), nil) - if err != nil { - return nil, err - } - response, err := s.client().Do(request) - if err != nil { - return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) - } - defer response.Body.Close() - switch response.StatusCode { - case http.StatusOK: - case http.StatusNotFound: - return nil, Gone - default: - return nil, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest) - } - body, err := io.ReadAll(io.LimitReader(response.Body, blobBound+1)) - if err != nil { - return nil, fmt.Errorf("reading %s/blobs/%s: %w", repository, digest, err) - } - if len(body) > blobBound { - return nil, fmt.Errorf("%s/blobs/%s is larger than %d bytes, which no archive read here is", repository, digest, blobBound) - } - sum := sha256.Sum256(body) - if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { - return nil, fmt.Errorf("the artifact store answered %s/blobs/%s with bytes of the digest %s: refused", repository, - digest, got) - } - return body, nil -} diff --git a/modules/artifact-store-tools/internal/registry/blob_test.go b/modules/artifact-store-tools/internal/registry/blob_test.go deleted file mode 100644 index 13487cc7..00000000 --- a/modules/artifact-store-tools/internal/registry/blob_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package registry - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -// A take-in reads a templates archive by its reference, and only bytes of the digest the reference names. -func TestGetBlobReadsOnlyTheBytesOfItsDigest(t *testing.T) { - good := []byte("the archive") - sum := sha256.Sum256(good) - digest := "sha256:" + hex.EncodeToString(sum[:]) - liar := "sha256:" + strings.Repeat("0", 64) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+digest), strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+liar): - _, _ = w.Write(good) - default: - w.WriteHeader(http.StatusNotFound) - } - })) - defer server.Close() - s := Store{Address: strings.TrimPrefix(server.URL, "http://")} - ctx := context.Background() - body, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+digest) - if err != nil || string(body) != string(good) { - t.Fatalf("the archive: %q %v", body, err) - } - if _, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+liar); err == nil || !strings.Contains(err.Error(), "refused") { - t.Fatalf("bytes of another digest are read: %v", err) - } - if _, err := s.GetBlob(ctx, "artifact-store://x/other/blobs/"+digest); !errors.Is(err, Gone) { - t.Fatalf("a blob the store does not hold: %v", err) - } - if _, err := s.GetBlob(ctx, "registry.example/x@"+digest); !errors.Is(err, ErrNotOurs) { - t.Fatalf("a reference outside the store: %v", err) - } -} diff --git a/modules/artifact-store-tools/internal/registry/hold.go b/modules/artifact-store-tools/internal/registry/hold.go deleted file mode 100644 index 391e9c89..00000000 --- a/modules/artifact-store-tools/internal/registry/hold.go +++ /dev/null @@ -1,345 +0,0 @@ -package registry - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "net/http" - "strconv" - "strings" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189). -// -// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and -// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs -// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what -// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a -// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a -// bare blob is unreferenced, so the first real collection would have deleted every archive the -// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The -// collector runs `--dry-run` until this is true. -// -// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty -// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The -// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its -// holder first, which is what lets the bytes go at the next collection. -// -// **Nothing a machine reads changes.** The recorded reference stays -// `artifact-store:////blobs/sha256:…`, and machines fetch the blob exactly as -// before. The holder is the store's bookkeeping, not a second way to reach anything. -// -// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest -// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can -// compute which manifest holds any archive from the reference it already has plus one HEAD for the -// size. No schema change, no second record that could disagree with the store. - -const ( - // mediaManifest is the type a holder is put and asked for as. - mediaManifest = "application/vnd.oci.image.manifest.v1+json" - // mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest - // whose only purpose is to name its layer. - mediaEmpty = "application/vnd.oci.empty.v1+json" - // emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec. - emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" - // mediaArchive is the layer type an archive is held as. Every archive the builder publishes is - // `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant, - // not read from anywhere, because the holder must be recomputable from the reference alone. - mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip" -) - -// emptyConfig is the content emptyDigest names. -var emptyConfig = []byte("{}") - -// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a -// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well -// (measured 2026-09-28; internal/builder/registry.go says how that was found). -var manifestAccept = []string{ - mediaManifest, - "application/vnd.docker.distribution.manifest.v2+json", -} - -type descriptor struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - Size int64 `json:"size"` -} - -type holderManifest struct { - SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType"` - Config descriptor `json:"config"` - Layers []descriptor `json:"layers"` -} - -// Holder is the manifest that holds an archive in the store, and its digest. -// -// A pure function of the archive's digest and size: the same two in give the same bytes out, -// always, because `encoding/json` writes a struct's fields in their declared order and there is -// nothing here that varies by when or where it was composed. -func Holder(digest string, size int64) (body []byte, holder string) { - body, err := json.Marshal(holderManifest{ - SchemaVersion: 2, - MediaType: mediaManifest, - Config: descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))}, - Layers: []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}}, - }) - if err != nil { - // Marshalling a struct of strings and integers cannot fail. - panic(err) - } - sum := sha256.Sum256(body) - return body, "sha256:" + hex.EncodeToString(sum[:]) -} - -// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one. -// -// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so -// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which -// is what lets the sweep run it over every kept archive on every build and so backfill the bare -// blobs published before holders existed (novox/hq issue 253). -// -// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a -// fact the caller reports rather than one this invents a remedy for. -func (s Store) Hold(ctx context.Context, reference string) (bool, error) { - repository, digest, archive, err := s.archive(reference) - if err != nil || !archive { - return false, err - } - size, err := s.blobSize(ctx, repository, digest) - if err != nil { - return false, err - } - return s.HoldBlob(ctx, repository, digest, size) -} - -// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the -// question an operator needs answered with "none unheld" before the collector is let loose. -// -// An image answers true: it is its own manifest. An archive the store does not have answers Gone. -func (s Store) Held(ctx context.Context, reference string) (bool, error) { - repository, digest, archive, err := s.archive(reference) - if err != nil { - return false, err - } - if !archive { - return true, nil - } - size, err := s.blobSize(ctx, repository, digest) - if err != nil { - return false, err - } - _, holder := Holder(digest, size) - return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...) -} - -// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is -// there already. Answers whether it wrote one. -// -// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size -// the store reports. Both arrive at the same holder, which is the point of composing it. -func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) { - if s.Address == "" { - return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest) - } - body, holder := Holder(digest, size) - there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...) - if err != nil { - return false, err - } - if there { - return false, nil - } - - // The config must be in the repository before a manifest naming it is accepted: a registry - // refuses a manifest whose blobs it cannot find there, which is the property that makes a - // holder mean something. - if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil { - return false, err - } - - // **By digest, never by tag.** A tag would be one more name to move and one more thing the - // collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that - // it pins by digest, and an untagged manifest is kept by plain collection. - request, err := http.NewRequestWithContext(ctx, http.MethodPut, - s.url(repository, "manifests", holder), bytes.NewReader(body)) - if err != nil { - return false, err - } - request.Header.Set("Content-Type", mediaManifest) - response, err := s.client().Do(request) - if err != nil { - return false, err - } - defer response.Body.Close() - if response.StatusCode != http.StatusCreated { - said, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) - return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s", - repository, digest, response.Status, strings.TrimSpace(string(said))) - } - return true, nil -} - -// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes. -// -// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the -// collector would keep its bytes for ever on the strength of it — the sweep would record the -// archive collected while the disk said otherwise. Deleting the holder first and failing before -// the link goes leaves an unheld archive that the next sweep still offers, which is safe. -// -// A store that no longer has the blob answers Gone: without its size the holder cannot be named, -// and without the blob there is nothing left for a holder to keep. A store that never had a holder -// for it — an archive published before holders, never backfilled — answers 404 to the delete, and -// that is the outcome wanted. -func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error { - size, err := s.blobSize(ctx, repository, digest) - if err != nil { - return err - } - _, holder := Holder(digest, size) - err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder) - if err == Gone { - return nil - } - return err -} - -// archive reads a recorded reference into its repository and digest, and whether it is an archive -// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store. -func (s Store) archive(reference string) (repository, digest string, archive bool, err error) { - path, kept := catalogue.InArtifactStore(reference) - if !kept { - return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference) - } - if s.Address == "" { - return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference) - } - repository, kind, digest, err := split(path) - if err != nil { - return "", "", false, err - } - return repository, digest, kind == "blobs", nil -} - -// blobSize is how large the store says a blob is; Gone when it does not have it. -func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) { - request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil) - if err != nil { - return 0, err - } - response, err := s.client().Do(request) - if err != nil { - return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) - } - defer response.Body.Close() - switch response.StatusCode { - case http.StatusOK: - case http.StatusNotFound: - return 0, Gone - default: - return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest) - } - // Read from the header rather than ContentLength: a HEAD's ContentLength is what the response - // says it would have sent, which Go reports faithfully, but a proxy in between is free to drop - // it, and the header is what the registry itself wrote. - if length := response.Header.Get("Content-Length"); length != "" { - if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 { - return n, nil - } - } - if response.ContentLength >= 0 { - return response.ContentLength, nil - } - return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest) -} - -// putBlob uploads a small blob unless the repository already has it: ask where, then put it there -// naming the digest — the registry's own two steps, the same the builder takes for an archive. -func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error { - if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil { - return err - } else if there { - return nil - } - start, err := http.NewRequestWithContext(ctx, http.MethodPost, - "http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil) - if err != nil { - return err - } - begun, err := s.client().Do(start) - if err != nil { - return fmt.Errorf("cannot start an upload to %s: %w", repository, err) - } - begun.Body.Close() - if begun.StatusCode != http.StatusAccepted { - return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository) - } - where := begun.Header.Get("Location") - if where == "" { - return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository) - } - if strings.HasPrefix(where, "/") { - where = "http://" + s.Address + where - } - separator := "?" - if strings.Contains(where, "?") { - separator = "&" - } - put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body)) - if err != nil { - return err - } - put.Header.Set("Content-Type", "application/octet-stream") - done, err := s.client().Do(put) - if err != nil { - return err - } - defer done.Body.Close() - if done.StatusCode != http.StatusCreated { - said, _ := io.ReadAll(io.LimitReader(done.Body, 4096)) - return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said))) - } - return nil -} - -// has is whether the store answers 200 for a HEAD at that URL. -func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) { - request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) - if err != nil { - return false, err - } - for _, media := range accept { - request.Header.Add("Accept", media) - } - response, err := s.client().Do(request) - if err != nil { - return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) - } - defer response.Body.Close() - switch response.StatusCode { - case http.StatusOK: - return true, nil - case http.StatusNotFound: - return false, nil - default: - return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url) - } -} - -func (s Store) url(repository, kind, digest string) string { - return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest -} - -func (s Store) client() *http.Client { - if s.HTTP != nil { - return s.HTTP - } - return &http.Client{Timeout: 30 * time.Second} -} diff --git a/modules/artifact-store-tools/internal/registry/hold_test.go b/modules/artifact-store-tools/internal/registry/hold_test.go deleted file mode 100644 index 8a3a16fa..00000000 --- a/modules/artifact-store-tools/internal/registry/hold_test.go +++ /dev/null @@ -1,268 +0,0 @@ -package registry - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strconv" - "strings" - "sync" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189). -// -// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a -// registry refuses — a blob whose digest does not match, a manifest whose digest does not match -// or whose blobs the repository does not have, a manifest asked for without an Accept naming its -// type. What is asserted is this side's decisions; the live test below asserts the registry's. - -type memRegistry struct { - mu sync.Mutex - blobs map[string][]byte // repository + "@" + digest - manifests map[string][]byte // repository + "@" + digest - writes []string // every PUT and DELETE, as "METHOD path" -} - -func digestOf(body []byte) string { - sum := sha256.Sum256(body) - return "sha256:" + hex.EncodeToString(sum[:]) -} - -func (m *memRegistry) serve(t *testing.T) Store { - t.Helper() - m.blobs = map[string][]byte{} - m.manifests = map[string][]byte{} - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - m.mu.Lock() - defer m.mu.Unlock() - path := strings.TrimPrefix(r.URL.Path, "/v2/") - if r.Method == http.MethodPut || r.Method == http.MethodDelete { - m.writes = append(m.writes, r.Method+" "+r.URL.Path) - } - switch { - case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"): - repository := strings.TrimSuffix(path, "/blobs/uploads/") - w.Header().Set("Location", "/upload/"+repository+"?state=x") - w.WriteHeader(http.StatusAccepted) - case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"): - repository := strings.TrimPrefix(r.URL.Path, "/upload/") - body, _ := io.ReadAll(r.Body) - digest := r.URL.Query().Get("digest") - if digest != digestOf(body) { - w.WriteHeader(http.StatusBadRequest) - return - } - m.blobs[repository+"@"+digest] = body - w.WriteHeader(http.StatusCreated) - case strings.Contains(path, "/blobs/"): - repository, digest, _ := strings.Cut(path, "/blobs/") - key := repository + "@" + digest - body, ok := m.blobs[key] - if !ok { - w.WriteHeader(http.StatusNotFound) - return - } - switch r.Method { - case http.MethodHead: - w.Header().Set("Content-Length", strconv.Itoa(len(body))) - w.WriteHeader(http.StatusOK) - case http.MethodDelete: - delete(m.blobs, key) - w.WriteHeader(http.StatusAccepted) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } - case strings.Contains(path, "/manifests/"): - repository, digest, _ := strings.Cut(path, "/manifests/") - key := repository + "@" + digest - switch r.Method { - case http.MethodHead: - if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) { - w.WriteHeader(http.StatusNotFound) - return - } - w.WriteHeader(http.StatusOK) - case http.MethodPut: - body, _ := io.ReadAll(r.Body) - if digest != digestOf(body) { - w.WriteHeader(http.StatusBadRequest) - return - } - var named holderManifest - if err := json.Unmarshal(body, &named); err != nil { - w.WriteHeader(http.StatusBadRequest) - return - } - for _, d := range append([]descriptor{named.Config}, named.Layers...) { - if _, ok := m.blobs[repository+"@"+d.Digest]; !ok { - w.WriteHeader(http.StatusBadRequest) - fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest) - return - } - } - m.manifests[key] = body - w.WriteHeader(http.StatusCreated) - case http.MethodDelete: - if _, ok := m.manifests[key]; !ok { - w.WriteHeader(http.StatusNotFound) - return - } - delete(m.manifests, key) - w.WriteHeader(http.StatusAccepted) - } - default: - w.WriteHeader(http.StatusNotFound) - } - })) - t.Cleanup(server.Close) - return Store{Address: strings.TrimPrefix(server.URL, "http://")} -} - -// bare puts an archive in the store the way the builder did before holders: a blob, nothing more. -func (m *memRegistry) bare(repository string, body []byte) string { - m.mu.Lock() - defer m.mu.Unlock() - digest := digestOf(body) - m.blobs[repository+"@"+digest] = body - return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest -} - -func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) { - // The sweep must arrive at the very manifest the builder wrote, with nothing recorded between - // them. Same inputs, same bytes — and a different size is a different holder, so a holder can - // never be mistaken for one of a different blob. - digest := "sha256:" + strings.Repeat("a", 64) - one, first := Holder(digest, 42) - two, second := Holder(digest, 42) - if !bytes.Equal(one, two) || first != second { - t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two) - } - if _, other := Holder(digest, 43); other == first { - t.Fatal("a different size composed the same holder") - } - want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` + - `"config":{"mediaType":"application/vnd.oci.empty.v1+json",` + - `"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` + - `"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}` - if string(one) != want { - t.Fatalf("the holder is\n%s\nwant\n%s", one, want) - } - if digestOf(emptyConfig) != emptyDigest { - t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest) - } -} - -func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) { - // The archives published before this have no holder. Hold, run over every kept archive on - // every sweep, writes one the first time and nothing after. - m := &memRegistry{} - store := m.serve(t) - ctx := context.Background() - body := []byte("a theme") - reference := m.bare("shell/config", body) - - if held, err := store.Held(ctx, reference); err != nil || held { - t.Fatalf("a bare blob reads as held=%v (%v)", held, err) - } - wrote, err := store.Hold(ctx, reference) - if err != nil { - t.Fatal(err) - } - if !wrote { - t.Fatal("holding a bare archive wrote nothing") - } - _, holder := Holder(digestOf(body), int64(len(body))) - if _, ok := m.manifests["shell/config@"+holder]; !ok { - t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder) - } - if held, err := store.Held(ctx, reference); err != nil || !held { - t.Fatalf("after holding, held=%v (%v)", held, err) - } - - writes := len(m.writes) - wrote, err = store.Hold(ctx, reference) - if err != nil { - t.Fatal(err) - } - if wrote || len(m.writes) != writes { - t.Fatalf("holding again wrote %v", m.writes[writes:]) - } -} - -func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) { - m := &memRegistry{} - store := m.serve(t) - ctx := context.Background() - - // An image is its own manifest: nothing is asked. - wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64)) - if err != nil || wrote || len(m.writes) != 0 { - t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes) - } - // An archive the store does not have is a fact to report, not something to invent a holder for. - _, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64)) - if !errors.Is(err, Gone) { - t.Fatalf("holding a missing archive answered %v, want Gone", err) - } - // And a reference that is not the mesh's is refused as such. - if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) { - t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err) - } -} - -func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) { - // A holder left behind would keep the bytes through every collection while the record said - // collected; the link deleted first and the holder failing after would be that exactly. - m := &memRegistry{} - store := m.serve(t) - ctx := context.Background() - body := []byte("an old theme") - reference := m.bare("shell/config", body) - if _, err := store.Hold(ctx, reference); err != nil { - t.Fatal(err) - } - m.writes = nil - - if err := store.LetGo(ctx, reference); err != nil { - t.Fatal(err) - } - _, holder := Holder(digestOf(body), int64(len(body))) - want := []string{ - "DELETE /v2/shell/config/manifests/" + holder, - "DELETE /v2/shell/config/blobs/" + digestOf(body), - } - if strings.Join(m.writes, "\n") != strings.Join(want, "\n") { - t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n")) - } - if len(m.manifests) != 0 { - t.Fatalf("a holder survived: %v", m.manifests) - } - // Asked again, the archive is already gone, which is the outcome wanted. - if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) { - t.Fatalf("letting go twice answered %v, want Gone", err) - } -} - -func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) { - // An archive published before holders and let go of before any sweep held it: the holder's - // delete answers 404, which is the outcome wanted, and the blob still goes. - m := &memRegistry{} - store := m.serve(t) - reference := m.bare("shell/config", []byte("never held")) - if err := store.LetGo(context.Background(), reference); err != nil { - t.Fatal(err) - } - if len(m.blobs) != 0 { - t.Fatalf("the blob survived: %v", m.blobs) - } -} diff --git a/modules/artifact-store-tools/internal/registry/platforms_test.go b/modules/artifact-store-tools/internal/registry/platforms_test.go deleted file mode 100644 index a335fb2c..00000000 --- a/modules/artifact-store-tools/internal/registry/platforms_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package registry - -import ( - "context" - "errors" - "fmt" - "net/http" - "net/http/httptest" - "slices" - "strings" - "sync" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// An index is let go of with the platform manifests it names, except those a kept index of the same -// repository names too (novox/hq ADR 0257). Each platform manifest is a manifest of the repository in -// its own right, and the store's collector keeps every manifest a repository holds — so an index let go -// of alone frees nothing of the images it names. - -func digestN(n int) string { return fmt.Sprintf("sha256:%064x", n) } - -// indexStore holds indexes and images in one repository, answers GETs with their documents, and records -// every delete. -type indexStore struct { - mu sync.Mutex - indexes map[string][]string // digest → the platform manifests it names - images map[string]bool - deleted []string -} - -func (s *indexStore) serve(t *testing.T) Store { - t.Helper() - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - defer s.mu.Unlock() - digest := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] - switch r.Method { - case http.MethodGet: - if children, ok := s.indexes[digest]; ok { - var named []string - for _, c := range children { - named = append(named, `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"`+c+`","size":1}`) - } - w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json") - _, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` + - strings.Join(named, ",") + `]}`)) - return - } - if s.images[digest] { - w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - _, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`)) - return - } - w.WriteHeader(http.StatusNotFound) - case http.MethodHead: - if _, ok := s.indexes[digest]; ok || s.images[digest] { - w.WriteHeader(http.StatusOK) - return - } - w.WriteHeader(http.StatusNotFound) - case http.MethodDelete: - if _, ok := s.indexes[digest]; !ok && !s.images[digest] { - w.WriteHeader(http.StatusNotFound) - return - } - s.deleted = append(s.deleted, digest) - delete(s.indexes, digest) - delete(s.images, digest) - w.WriteHeader(http.StatusAccepted) - default: - w.WriteHeader(http.StatusBadRequest) - } - })) - t.Cleanup(server.Close) - return Store{Address: strings.TrimPrefix(server.URL, "http://")} -} - -func TestAnIndexGoesWithItsPlatformsAndAKeptIndexKeepsItsOwn(t *testing.T) { - // Two indexes of one image's repository: the old one names platforms 11 and 12, the kept one 12 and 13. - s := &indexStore{ - indexes: map[string][]string{digestN(1): {digestN(11), digestN(12)}, digestN(2): {digestN(12), digestN(13)}}, - images: map[string]bool{digestN(11): true, digestN(12): true, digestN(13): true}, - } - store := s.serve(t) - repository := "upstream/docker.io/library/golang" - old := catalogue.ArtifactStoreScheme + repository + "@" + digestN(1) - kept := catalogue.ArtifactStoreScheme + repository + "@" + digestN(2) - spare, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old}) - if err != nil { - t.Fatal(err) - } - store.Spare = spare - - if err := store.LetGo(context.Background(), old); err != nil { - t.Fatal(err) - } - // Its own platform first, the index last; the platform the kept index names is left in place. - if !slices.Equal(s.deleted, []string{digestN(11), digestN(1)}) { - t.Fatalf("deleted %v; want the old index's own platform, then the index", s.deleted) - } - if !s.images[digestN(12)] || !s.images[digestN(13)] { - t.Fatal("a platform a kept index names was let go of") - } -} - -func TestWithoutSparingAnIndexGoesAloneAndAnImageNamesNoPlatforms(t *testing.T) { - s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true, digestN(5): true}} - store := s.serve(t) - // No Spare: as before ADR 0257, the index alone. - if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1)); err != nil { - t.Fatal(err) - } - if !slices.Equal(s.deleted, []string{digestN(1)}) { - t.Fatalf("without a Spare, deleted %v", s.deleted) - } - // An image is one delete, with a Spare or without. - s.deleted = nil - spare, err := store.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "web/app@" + digestN(5)}) - if err != nil { - t.Fatal(err) - } - store.Spare = spare - if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@"+digestN(5)); err != nil { - t.Fatal(err) - } - if !slices.Equal(s.deleted, []string{digestN(5)}) { - t.Fatalf("an image was let go of as %v", s.deleted) - } - // An index the store no longer holds is Gone, and nothing is deleted on its account. - s.deleted = nil - err = store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(9)) - if !errors.Is(err, Gone) || len(s.deleted) != 0 { - t.Fatalf("an index the store does not hold: %v, deleted %v", err, s.deleted) - } -} - -func TestASpareThatCannotBeReadKeepsTheIndex(t *testing.T) { - s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}} - store := s.serve(t) - store.Spare = func(context.Context, string) (map[string]bool, error) { return nil, fmt.Errorf("the store went away") } - err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1)) - if err == nil || !strings.Contains(err.Error(), "was kept") { - t.Fatalf("a spare that could not be read: %v", err) - } - if len(s.deleted) != 0 { - t.Fatalf("deleted %v without knowing what a kept index names", s.deleted) - } -} - -// The spare list is read whole before the sweep: a kept index that cannot be read is an error before -// anything is deleted, and a repository not read before keeps its index. -func TestASpareListIsReadWholeBeforeAnythingGoes(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - t.Cleanup(server.Close) - store := Store{Address: strings.TrimPrefix(server.URL, "http://")} - kept := catalogue.ArtifactStoreScheme + "web/base@" + digestN(2) - old := catalogue.ArtifactStoreScheme + "web/base@" + digestN(1) - if _, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old}); err == nil { - t.Fatal("a kept index the store would not answer for was taken as naming nothing") - } - - s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}} - good := s.serve(t) - spare, err := good.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "other/repo@" + digestN(3)}) - if err != nil { - t.Fatal(err) - } - good.Spare = spare - if err := good.LetGo(context.Background(), old); err == nil || len(s.deleted) != 0 { - t.Fatalf("an index in a repository not read before the sweep was let go of: %v, deleted %v", err, s.deleted) - } -} diff --git a/modules/artifact-store-tools/internal/registry/read.go b/modules/artifact-store-tools/internal/registry/read.go new file mode 100644 index 00000000..f759c5e5 --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/read.go @@ -0,0 +1,76 @@ +package registry + +import ( + "context" + "fmt" + "net/http" + "strconv" + "time" +) + +// blobSize is how large the store says a blob is; Gone when it does not have it. +func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil) + if err != nil { + return 0, err + } + response, err := s.client().Do(request) + if err != nil { + return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return 0, Gone + default: + return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest) + } + // Read from the header rather than ContentLength: a HEAD's ContentLength is what the response + // says it would have sent, which Go reports faithfully, but a proxy in between is free to drop + // it, and the header is what the registry itself wrote. + if length := response.Header.Get("Content-Length"); length != "" { + if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 { + return n, nil + } + } + if response.ContentLength >= 0 { + return response.ContentLength, nil + } + return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest) +} + +// has is whether the store answers 200 for a HEAD at that URL. +func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) + if err != nil { + return false, err + } + for _, media := range accept { + request.Header.Add("Accept", media) + } + response, err := s.client().Do(request) + if err != nil { + return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound: + return false, nil + default: + return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url) + } +} + +func (s Store) url(repository, kind, digest string) string { + return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest +} + +func (s Store) client() *http.Client { + if s.HTTP != nil { + return s.HTTP + } + return &http.Client{Timeout: 30 * time.Second} +} diff --git a/modules/artifact-store-tools/internal/registry/store.go b/modules/artifact-store-tools/internal/registry/store.go index 8a1fe931..9e7fcdd0 100644 --- a/modules/artifact-store-tools/internal/registry/store.go +++ b/modules/artifact-store-tools/internal/registry/store.go @@ -1,13 +1,6 @@ -// Package registry speaks to the mesh's artifact store over its own door. -// -// Only what the mesh needs that nothing else does: letting go of something it put there -// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's -// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading -// is every machine's, through its runtime. This is the one operation that belongs to the thing -// holding the records, because it is the only one that is a decision rather than a transfer. -// -// Copied from mesh-controller internal/artifacts at e04846e9 (novox/hq ADR 0571 clause 2): the seat mesh-artifact-sweep lets -// go of what it no longer keeps exactly as the controller's sweep did; the controller's copy goes with its lease. +// Package registry reads the mesh's artifact store over its own door: whether it holds what a reference names, and +// whether a manifest is an index. It writes nothing: what the artifact sweep holds and removes it asks the registry +// seats the store's holder serves (CAT-AS1, novox/hq ADR 0617 §2). package registry import ( @@ -29,10 +22,6 @@ type Store struct { Address string // HTTP is the client used; nil is a client with a modest timeout. HTTP *http.Client - // Spare says, for one repository, which manifests a kept index there names: the platform - // manifests an index being let go of must leave in place (novox/hq ADR 0257). Nil spares every - // platform manifest — an index is then let go of alone, as it was before. - Spare func(ctx context.Context, repository string) (map[string]bool, error) } // Gone is the answer when the store does not hold it: the outcome wanted, already true. @@ -46,86 +35,6 @@ var Gone = errors.New("the store does not hold it") // error" is how a cautious loop became one that did nothing while reporting the right number. var ErrNotOurs = errors.New("not a reference into the mesh's artifact store") -// LetGo asks the store to drop one artifact the mesh recorded making. -// -// Takes a reference as the mesh records it — `artifact-store:///@sha256:…` for -// an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses, -// and composes the address here at the moment of use. -// -// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq -// issue 253); an image in one. -// -// Returns Gone when the store answers that it does not have it. That is not a failure: the sweep -// wants the artifact absent, and it is. It is distinguished from success only so a caller can say -// which of the two happened. -func (s Store) LetGo(ctx context.Context, reference string) error { - // **Strict, and deliberately** (novox/hq issue 226). Only a reference the mesh keeps in its - // own vocabulary is addressed here. `Recorded` would read `docker.io/library/registry@sha256:…` - // as the mesh's too — it cannot tell one registry host from another — so normalising belongs - // where the provenance is known, which is the sweep reading its own build records, not here - // where the only job is to refuse anything that is not plainly ours. - path, kept := catalogue.InArtifactStore(reference) - if !kept { - // Nothing the mesh put in its own store. Refused rather than attempted: composing a - // delete for a reference of unknown shape is how a sweep reaches something that is not - // the mesh's. Distinguished from a store that refuses, so a sweep skips this and goes on. - return fmt.Errorf("%w: %s", ErrNotOurs, reference) - } - if s.Address == "" { - return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference) - } - repository, kind, digest, err := split(path) - if err != nil { - return err - } - if kind == "blobs" { - // **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left - // naming the blob would keep its bytes through every collection while the record said - // collected. Gone here means the store has no such blob, so there is nothing to let go. - if err := s.letGoOfHolder(ctx, repository, digest); err != nil { - return err - } - } - if kind == "manifests" && s.Spare != nil { - // **An index's platform manifests go before the index** (novox/hq ADR 0257). Each was put - // under its own digest, so each is a manifest of the repository in its own right, and the - // store's collector keeps every manifest a repository holds: an index let go of alone frees - // no byte of the images it names. Before, not after, so a sweep cut short leaves the index - // to be asked about again, and with it the list of what is still to go. - if err := s.letGoOfPlatforms(ctx, repository, digest, reference); err != nil { - return err - } - } - return s.remove(ctx, s.url(repository, kind, digest), reference) -} - -// letGoOfPlatforms deletes the manifests an index names, except those a kept index of the same -// repository also names. A manifest that is not an index names none. -func (s Store) letGoOfPlatforms(ctx context.Context, repository, digest, reference string) error { - children, err := s.Platforms(ctx, repository, digest) - if err != nil || len(children) == 0 { - // Gone: there is no index to read, and the delete that follows says so. - if errors.Is(err, Gone) { - return nil - } - return err - } - spared, err := s.Spare(ctx, repository) - if err != nil { - return fmt.Errorf("cannot say which platform manifests of %s a kept index names, so %s was kept: %w", - repository, reference, err) - } - for _, child := range children { - if spared[child] { - continue - } - if err := s.remove(ctx, s.url(repository, "manifests", child), reference+" (its "+child+")"); err != nil && !errors.Is(err, Gone) { - return err - } - } - return nil -} - // anyManifest is every manifest the store may hold, indexes included, so it answers with the // document as it is. var anyManifest = []string{ @@ -172,84 +81,6 @@ func (s Store) Platforms(ctx context.Context, repository, digest string) ([]stri return out, nil } -// SpareKeptIndexes is a Spare for a sweep, read whole before it starts: for every repository an -// image reference of `touching` is in, each kept manifest there and every platform manifest a kept -// index there names. Any read that fails is an error, and the sweep lets nothing go. A repository not -// read here is answered with an error, so an index in it is kept. A kept index the store no longer -// holds names nothing. -func (s Store) SpareKeptIndexes(ctx context.Context, kept, touching []string) (func(context.Context, string) (map[string]bool, error), error) { - imageIn := func(reference string) (repository, digest string, ok bool) { - path, ours := catalogue.InArtifactStore(reference) - if !ours { - return "", "", false - } - repository, kind, digest, err := split(path) - if err != nil || kind != "manifests" { - return "", "", false - } - return repository, digest, true - } - read := map[string]map[string]bool{} - for _, reference := range touching { - if repository, _, ok := imageIn(reference); ok { - read[repository] = map[string]bool{} - } - } - for _, reference := range kept { - repository, digest, ok := imageIn(reference) - spared, touched := read[repository] - if !ok || !touched { - continue - } - spared[digest] = true - children, err := s.Platforms(ctx, repository, digest) - if errors.Is(err, Gone) { - continue - } - if err != nil { - return nil, fmt.Errorf("reading the kept %s@%s: %w", repository, digest, err) - } - for _, c := range children { - spared[c] = true - } - } - return func(_ context.Context, repository string) (map[string]bool, error) { - spared, done := read[repository] - if !done { - return nil, fmt.Errorf("%s was not read before the sweep began", repository) - } - return spared, nil - }, nil -} - -// remove asks the store to delete what is at url. Gone when it has no such thing. -func (s Store) remove(ctx context.Context, url, what string) error { - request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil) - if err != nil { - return err - } - response, err := s.client().Do(request) - if err != nil { - return err - } - defer response.Body.Close() - switch response.StatusCode { - case http.StatusAccepted, http.StatusOK, http.StatusNoContent: - return nil - case http.StatusNotFound: - return Gone - case http.StatusMethodNotAllowed: - // The registry was started without deletion enabled. Said plainly, because the remedy is - // a setting on the store's module and not anything about this artifact. - return fmt.Errorf( - "the artifact store refuses deletion: its server was started without it enabled "+ - "(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+ - "module is applied again (novox/hq ADR 0189). Asking about %s", what) - default: - return fmt.Errorf("the artifact store answered %s for %s", response.Status, what) - } -} - // split reads a recorded path into the repository, which endpoint names the thing, and the digest. // // Two shapes, which are the two the mesh records: `@sha256:` is a manifest, and @@ -279,22 +110,6 @@ func (s Store) IsIndex(ctx context.Context, reference string) (bool, error) { return len(children) > 0, err } -// HoldsManifest is whether the store holds the manifest a recorded image reference names. -func (s Store) HoldsManifest(ctx context.Context, reference string) (bool, error) { - path, ours := catalogue.InArtifactStore(reference) - if !ours { - return false, fmt.Errorf("%w: %s", ErrNotOurs, reference) - } - repository, kind, digest, err := split(path) - if err != nil { - return false, err - } - if kind != "manifests" { - return false, fmt.Errorf("%w: %s names a blob, not a manifest", ErrNotOurs, reference) - } - return s.has(ctx, s.url(repository, kind, digest), anyManifest...) -} - // Holds is whether the store holds what a recorded reference names: the blob of an archive, the manifest of an image. // Gone when it does not; asks and changes nothing (ADR 0526 clause 3: a registration by hand is refused at bytes that // are not there). diff --git a/modules/artifact-store-tools/internal/registry/store_test.go b/modules/artifact-store-tools/internal/registry/store_test.go index 334b95a5..094ddbfe 100644 --- a/modules/artifact-store-tools/internal/registry/store_test.go +++ b/modules/artifact-store-tools/internal/registry/store_test.go @@ -11,124 +11,66 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" ) -// Asking the store to let go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108). -// -// A fake store records what it was asked to delete, so what is asserted is the mesh's decision -// and the shape of the request — not the registry's behaviour, which is the registry's to test. - -func fakeStore(t *testing.T, answer int) (Store, *[]string) { - t.Helper() +// The store's door is read only (CAT-AS1: what the sweep holds and removes it asks the registry seats): Holds says +// whether the store holds what a reference names, in either reference form, IsIndex whether it is an index; a reference +// that is not the mesh's own is never asked. A fake store answers by path and records what it was asked. +func TestTheDoorReadsWhatTheStoreHoldsAndWritesNothing(t *testing.T) { + const index = `{"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"digest":"sha256:c1"}]}` + const image = `{"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}` var asked []string - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") { - // An archive's size, asked so its holder can be named (novox/hq issue 253). A store - // that does not have the thing does not have its blob either. - if answer == http.StatusNotFound { - w.WriteHeader(http.StatusNotFound) - return + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + asked = append(asked, r.Method+" "+r.URL.Path) + switch r.URL.Path { + case "/v2/web/app/manifests/sha256:abc", "/v2/web/files/blobs/sha256:def": + if r.Method == http.MethodGet { + _, _ = w.Write([]byte(image)) + } else { + w.Header().Set("Content-Length", "7") } - w.Header().Set("Content-Length", "7") - w.WriteHeader(http.StatusOK) - return + case "/v2/web/app/manifests/sha256:idx": + if r.Method == http.MethodGet { + _, _ = w.Write([]byte(index)) + } + default: + w.WriteHeader(http.StatusNotFound) } - if r.Method != http.MethodDelete { - t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path) - } - asked = append(asked, r.URL.Path) - w.WriteHeader(answer) })) - t.Cleanup(server.Close) - return Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked -} - -func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) { - // The two shapes the mesh records: a manifest by digest, and a blob by digest. They are - // different endpoints, and asking at the wrong one answers 404 — which this would then - // record as collected, leaving the bytes on disk for ever while the record says otherwise. - store, asked := fakeStore(t, http.StatusAccepted) + defer srv.Close() + store := Store{Address: strings.TrimPrefix(srv.URL, "http://")} ctx := context.Background() - - image := catalogue.ArtifactStoreScheme + "web/app@sha256:abc123" - archive := catalogue.ArtifactStoreScheme + "web/config/blobs/sha256:def456" - if err := store.LetGo(ctx, image); err != nil { - t.Fatal(err) - } - if err := store.LetGo(ctx, archive); err != nil { - t.Fatal(err) - } - // The archive's holder goes first, then the archive (novox/hq issue 253). - _, holder := Holder("sha256:def456", 7) - want := []string{ - "/v2/web/app/manifests/sha256:abc123", - "/v2/web/config/manifests/" + holder, - "/v2/web/config/blobs/sha256:def456", - } - if strings.Join(*asked, " ") != strings.Join(want, " ") { - t.Fatalf("the store was asked %v; want %v", *asked, want) - } -} - -func TestAStoreThatDoesNotHaveItAnswersGone(t *testing.T) { - // The outcome wanted, already true. Told apart from success only so the sweep can say which - // happened; both are recorded, because retrying for ever is the thing to avoid. - store, _ := fakeStore(t, http.StatusNotFound) - err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123") - if !errors.Is(err, Gone) { - t.Fatalf("a store that does not hold it answered %v, want Gone", err) - } -} - -func TestAStoreWithDeletionOffSaysSoAndNamesTheRemedy(t *testing.T) { - // The registry answers 405 when it was started without deletion enabled. The remedy is a - // setting on the store's module, and saying "405" would send somebody to the wrong place. - store, _ := fakeStore(t, http.StatusMethodNotAllowed) - err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123") - if err == nil { - t.Fatal("a store that refuses deletion was read as success") - } - if !strings.Contains(err.Error(), "REGISTRY_STORAGE_DELETE_ENABLED") { - t.Fatalf("the refusal does not name the remedy: %v", err) - } -} - -func TestAReferenceThatIsNotTheMeshsOwnIsNeverAsked(t *testing.T) { - // The whole safety of the sweep is that it names only what the mesh recorded putting there. - // A reference of another shape — a vendor's image, a package version — is refused rather - // than composed into a delete somewhere that is not the mesh's store. - store, asked := fakeStore(t, http.StatusAccepted) - for _, reference := range []string{ - "docker.io/library/registry@sha256:abc123", - "registry@sha256:abc123", - "1.4.2", + for name, c := range map[string]struct { + reference string + holds error // nil, Gone or ErrNotOurs + index bool + }{ + "an image": {reference: catalogue.ArtifactStoreScheme + "web/app@sha256:abc"}, + "an image in the seat's form": {reference: catalogue.SeatReference("container-image-registry", "web", "app", "sha256:abc")}, + "an archive": {reference: catalogue.ArtifactStoreScheme + "web/files/blobs/sha256:def"}, + "an archive in the seat's form": {reference: catalogue.SeatReference("oci-artifact-registry", "web", "files", "sha256:def")}, + "an index": {reference: catalogue.ArtifactStoreScheme + "web/app@sha256:idx", index: true}, + "an image the store lacks": {reference: catalogue.ArtifactStoreScheme + "web/app@sha256:999", holds: Gone}, + "an archive the store lacks": {reference: catalogue.ArtifactStoreScheme + "web/files/blobs/sha256:999", holds: Gone}, + "another registry's image": {reference: "docker.io/library/alpine@sha256:abc", holds: ErrNotOurs}, + "a reference naming no digest": {reference: catalogue.ArtifactStoreScheme + "web/app:latest", holds: ErrNotOurs}, } { - if err := store.LetGo(context.Background(), reference); err == nil { - t.Errorf("%s was asked about; it is not a reference into the mesh's store", reference) - } - } - if len(*asked) != 0 { - t.Fatalf("the store was asked about %v", *asked) - } -} - -// A reference this sweep will not address says so as ErrNotOurs, which is a fact about the -// record and not about the store (novox/hq issue 226). -// -// The sweep skips one and abandons itself for the other, so they cannot be the same error. The -// first live run met a reference recorded with the store's old address, read the refusal as "the -// store refuses everything", and collected none of the 1681 it had found. -func TestAReferenceThisSweepWillNotAddressIsToldApartFromAStoreRefusing(t *testing.T) { - store, asked := fakeStore(t, http.StatusAccepted) - for _, reference := range []string{ - "docker.io/library/registry@sha256:abc123", - "127.0.0.1:5100/mesh-tools/build@sha256:abc123", - "1.4.2", - } { - err := store.LetGo(context.Background(), reference) - if !errors.Is(err, ErrNotOurs) { - t.Errorf("%s answered %v; a sweep must be able to skip it and go on", reference, err) - } - } - if len(*asked) != 0 { - t.Fatalf("the store was asked about %v", *asked) + t.Run(name, func(t *testing.T) { + asked = nil + if err := store.Holds(ctx, c.reference); !errors.Is(err, c.holds) && (err != nil || c.holds != nil) { + t.Errorf("Holds = %v, want %v", err, c.holds) + } + if c.holds == nil { + if got, err := store.IsIndex(ctx, c.reference); err != nil || got != c.index { + t.Errorf("IsIndex = %v, %v, want %v", got, err, c.index) + } + } + for _, a := range asked { + if !strings.HasPrefix(a, "HEAD ") && !strings.HasPrefix(a, "GET ") { + t.Errorf("the door asked %s: it writes nothing", a) + } + } + if errors.Is(c.holds, ErrNotOurs) && len(asked) != 0 { + t.Errorf("a reference that is not the mesh's own was asked: %v", asked) + } + }) } } diff --git a/modules/artifact-store-tools/module.json b/modules/artifact-store-tools/module.json index 5fd62c28..4c0161fe 100644 --- a/modules/artifact-store-tools/module.json +++ b/modules/artifact-store-tools/module.json @@ -22,7 +22,13 @@ "seat:mesh-catalog.modules", "seat:mesh-catalog.module", "mesh-delivery-assigner.assigner_read_wants", - "mesh-delivery-indexer.indexer_read_manifests" + "mesh-delivery-indexer.indexer_read_manifests", + "seat:oci-artifact-registry.hold", + "seat:oci-artifact-registry.release", + "seat:oci-artifact-registry.remove", + "seat:container-image-registry.hold", + "seat:container-image-registry.release", + "seat:container-image-registry.remove" ], "reads": [ "mesh-delivery-builder.builds", diff --git a/modules/distribution/cmd/distribution-tools/contract.go b/modules/distribution/cmd/distribution-tools/contract.go new file mode 100644 index 00000000..8e724dd2 --- /dev/null +++ b/modules/distribution/cmd/distribution-tools/contract.go @@ -0,0 +1,71 @@ +package main + +// The seats' contracts (mesh-contracts, generated from the seat definitions; ADR 0620): every verb's input and answer, +// and every event's body, as the two seats distribution holds define them. The verbs common to both seats have one +// shape in every registry seat, so containerimageregistry's types serve oci-artifact-registry's too. The holder's own +// types stay inside; what crosses the bus is the contract's. + +import ( + "encoding/base64" + "fmt" + "time" + + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + + "distribution/internal/holder" +) + +func publishOf(in reg.PublishInput) holder.PublishRequest { + r := holder.PublishRequest{Transfer: in.Transfer, Name: in.Name, Version: in.Version, Tag: in.Tag, Kind: in.Kind, + Fingerprint: in.Fingerprint, Commit: in.Commit, Digest: in.Digest} + for _, p := range in.Parts { + r.Parts = append(r.Parts, holder.Part{Digest: p.Digest, Size: p.Size}) + } + return r +} + +func publishAnswer(a holder.PublishAnswer) reg.PublishAnswer { + return reg.PublishAnswer{Transfer: a.Transfer, Lacks: a.Missing, Held: a.Ended && len(a.Missing) == 0} +} + +func partOf(in reg.PublishPartInput) (holder.PartRequest, error) { + data, err := base64.StdEncoding.DecodeString(in.Data) + if err != nil { + return holder.PartRequest{}, fmt.Errorf("publish-part: data is not base64: %w", err) + } + return holder.PartRequest{Transfer: in.Transfer, Digest: in.Part, Sequence: int(in.Sequence), Last: in.Last, Data: data}, nil +} + +func partAnswer(in reg.PublishPartInput, a holder.PublishAnswer) reg.PublishPartAnswer { + return reg.PublishPartAnswer{Transfer: in.Transfer, Part: in.Part, Received: in.Sequence + 1, Complete: a.Ended} +} + +// versionsAnswer is `versions`: each version with the digest the store holds under it. +func versionsAnswer(v *holder.Verbs, kind string, a holder.VersionsAnswer) reg.VersionsAnswer { + out := reg.VersionsAnswer{Name: a.Name, Versions: []reg.Artifact{}} + for _, ver := range a.Versions { + d, _ := v.Catalog.Digest(a.Name, ver) + out.Versions = append(out.Versions, reg.Artifact{Name: a.Name, Version: ver, Digest: d, Kind: kind}) + } + return out +} + +func tagsAnswer(v *holder.Verbs, a holder.VersionsAnswer) reg.TagsAnswer { + out := reg.TagsAnswer{Name: a.Name, Tags: []reg.TagsAnswerTagsItem{}} + for _, t := range a.Versions { + d, _ := v.Catalog.Digest(a.Name, t) + out.Tags = append(out.Tags, reg.TagsAnswerTagsItem{Tag: t, Digest: d}) + } + return out +} + +// eventBody is an event's body as the seat's contract defines it: an Outcome for published, refused and removed, a +// Hold for held and released. +func eventBody(event string, e holder.Event) any { + switch event { + case holder.EventHeld, holder.EventReleased: + return reg.Hold{Name: e.Name, Digest: e.Digest, Holder: e.Holder, By: e.By, Reason: e.Reason, At: time.Now().UTC().Format(time.RFC3339)} + } + return reg.Outcome{Name: e.Name, Version: e.Version, Kind: e.Kind, Tag: e.Tag, Digest: e.Digest, Fingerprint: e.Fingerprint, + Commit: e.Commit, Holder: e.Holder, Transfer: e.Transfer, Reference: e.Reference, Reason: e.Reason} +} diff --git a/modules/distribution/cmd/distribution-tools/contract_test.go b/modules/distribution/cmd/distribution-tools/contract_test.go new file mode 100644 index 00000000..0cc8fb8f --- /dev/null +++ b/modules/distribution/cmd/distribution-tools/contract_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + + "distribution/internal/holder" +) + +func jsonFields(v any) map[string]bool { + out := map[string]bool{} + t := reflect.TypeOf(v) + for i := 0; i < t.NumField(); i++ { + name, _, _ := strings.Cut(t.Field(i).Tag.Get("json"), ",") + out[name] = true + } + return out +} + +// TestTheVerbsSpeakTheSeatsContract holds distribution's seat verbs to the contracts mesh-contracts generates from the +// two seats' definitions (ADR 0620 *How it is checked*): every argument a verb's tool takes is a field of the +// contract's input, and every event body decodes into the contract's type with no field left over. +func TestTheVerbsSpeakTheSeatsContract(t *testing.T) { + inputs := map[string]any{reg.VerbPublish: reg.PublishInput{}, reg.VerbPublishPart: reg.PublishPartInput{}, + reg.VerbVersions: reg.VersionsInput{}, reg.VerbRemove: reg.RemoveInput{}, reg.VerbHold: reg.HoldInput{}, + reg.VerbRelease: reg.ReleaseInput{}, reg.VerbMirror: reg.MirrorInput{}, reg.VerbTags: reg.TagsInput{}} + seen := 0 + for _, tool := range tools(&holder.Verbs{}, kept[holder.Window]{}, func() (holder.Callers, error) { return holder.DefaultCallers(), nil }) { + seat, verb, ok := strings.Cut(tool.Name, ".") + in, isVerb := inputs[verb] + if !ok || !isVerb || (seat != holder.ImageSeat && seat != holder.ArtifactSeat) { + continue + } + seen++ + fields := jsonFields(in) + props, _ := tool.Input["properties"].(map[string]any) + for p := range props { + if !fields[p] { + t.Errorf("%s takes %q, which the contract's input does not name", tool.Name, p) + } + } + } + if seen != 14 { + t.Errorf("%d seat verbs served, want 14 (8 on the image seat, 6 on the artifact seat)", seen) + } + for event, want := range map[string]any{holder.EventPublished: ®.Outcome{}, holder.EventHeld: ®.Hold{}, holder.EventReleased: ®.Hold{}} { + b, _ := json.Marshal(eventBody(event, holder.Event{Name: "m/a", Kind: holder.KindArchive, Digest: "sha256:aa", Holder: holder.Self, By: "x.artifact-store-tools"})) + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + if err := dec.Decode(want); err != nil { + t.Errorf("%s's body %s is not the contract's: %v", event, b, err) + } + } + // The caller list is read on each call: one that cannot be read answers no verb, and registry_callers says it as read. + unread := func() (holder.Callers, error) { return nil, errors.New("config refused") } + for _, tool := range tools(&holder.Verbs{}, kept[holder.Window]{}, unread) { + if tool.Name == holder.ImageSeat+"."+holder.VerbVersions { + if _, err := tool.Run(map[string]any{"name": "m/a"}); err == nil || !strings.Contains(err.Error(), "caller list") { + t.Errorf("versions with an unread caller list = %v, want refused naming the caller list", err) + } + } + } + answered := false + for _, tool := range tools(&holder.Verbs{}, kept[holder.Window]{}, func() (holder.Callers, error) { return holder.Callers{holder.VerbPublish: {"mesh-build-go"}}, nil }) { + if tool.Name == holder.CallersTool { + answered = true + got, err := tool.Run(map[string]any{}) + if b, _ := json.Marshal(got); err != nil || string(b) != `{"callers":{"publish":["mesh-build-go"]}}` { + t.Errorf("%s = %s, %v", holder.CallersTool, b, err) + } + } + } + if !answered { + t.Errorf("no tool %s: the program could not read the caller list", holder.CallersTool) + } + if a := publishAnswer(holder.PublishAnswer{Transfer: "t", Missing: []string{"sha256:bb"}}); !reflect.DeepEqual(a.Lacks, []string{"sha256:bb"}) || a.Held { + t.Errorf("publish answers %+v", a) + } + if r, err := partOf(reg.PublishPartInput{Transfer: "t", Part: "sha256:bb", Data: "aGk=", Last: true}); err != nil || string(r.Data) != "hi" || r.Digest != "sha256:bb" { + t.Errorf("publish-part read %+v, %v", r, err) + } +} diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go new file mode 100644 index 00000000..2385b3e9 --- /dev/null +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -0,0 +1,492 @@ +// Command distribution-tools serves distribution's seat verbs and tools (novox/hq design 51): the verbs of +// container-image-registry and oci-artifact-registry, each as `.`, and registry_incoming, registry_holds, +// registry_old_name and registry_direct_writes. The runtime launches it on distribution's own account; the caller of +// every verb is the one the tool runner proved (stdio.CallerOf), never a name in the arguments. stdout is the MCP +// channel; what it says, it says on stderr. +package main + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "strings" + "sync" + "time" + + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/parts" + + "distribution/internal/holder" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, "[distribution-tools] "+format+"\n", a...) } + +// kept is one state of distribution's, through the runtime. +type kept[T any] struct { + state string + key func(T) string +} + +func (k kept[T]) Get(key string) (T, bool, error) { + var v T + e, err := stdio.State(k.state).Get(key) + if err != nil || e == nil { + return v, false, err + } + return v, true, json.Unmarshal(e.Value, &v) +} +func (k kept[T]) Put(v T) error { + _, err := stdio.State(k.state).Put(k.key(v), v) + return err +} +func (k kept[T]) Delete(key string) error { return stdio.State(k.state).Delete(key) } +func (k kept[T]) List() ([]T, error) { + keys, err := stdio.State(k.state).Keys() + if err != nil { + return nil, err + } + var out []T + for _, key := range keys { + if v, ok, err := k.Get(key); err == nil && ok { + out = append(out, v) + } + } + return out, nil +} + +func say(seat, event string, body holder.Event) error { + return stdio.SeatPublish("mesh.seat."+seat+".event."+event, eventBody(event, body), body.ID) +} + +func finding(kind, subject, words string, raised bool) { + key := health.FindingKey(health.ScopeSeat, "distribution", kind+"."+strings.NewReplacer(".", "_", "@", "_", "/", "_").Replace(subject)) + if !raised { + _ = stdio.State(health.FindingsState).Delete(key) + return + } + f := health.Finding{V: health.FindingVersion, Key: key, Kind: kind, Ref: key[strings.LastIndex(key, ".")+1:], + Subject: health.Subject{Scope: health.ScopeSeat, ID: "distribution"}, Severity: health.Warning, + Summary: words, Headline: words, Explanation: words, Resolved: "cleared once it no longer holds", + Source: kind, Observer: "distribution", At: time.Now().UTC()} + if _, err := stdio.State(health.FindingsState).Put(key, f); err != nil { + logf("the finding %s could not be written: %v", key, err) + } +} + +// closeBootstrap asks the program to discard the bootstrap's one-off credential once a publish is answered (design 51 +// *On a fresh mesh*): the window bootstrap-push closed in the state windows, which the program follows and then removes +// the file. The tools bundle runs in the tool runner's account, which cannot remove a file in distribution's state +// directory. Once per process; the window's record says it to the next. +var closeBootstrap func() + +func bootstrapCloser(windows kept[holder.Window]) func() { + return sync.OnceFunc(func() { + cur, _, err := windows.Get(holder.BootstrapWindow) + if err == nil && !cur.Closed.IsZero() { + return + } + w := holder.Window{Name: holder.BootstrapWindow, Closed: time.Now().UTC()} + if err := windows.Put(w); err != nil { + logf("the bootstrap's one-off credential could not be asked away: %v", err) + return + } + logf("the bootstrap's one-off credential is asked away: the holders serve publish") + }) +} + +// learned are the proved callers whose build this process recorded as publishing through the seats. +var learned sync.Map + +// learnPublishing records the fingerprint the proved caller's node runs of its build module as one that publishes +// through the seats (holder.PublishingKey), once per caller and process: the windows' driver in the program counts a +// node's have running it as publishing, closing the direct-write window when every node's build modules do. +func learnPublishing(windows kept[holder.Window], caller string) { + node, module, ok := strings.Cut(caller, ".") + if !ok || strings.HasPrefix(caller, "person.") { + return + } + if _, done := learned.Load(caller); done { + return + } + raw, err := stdio.Ask("mesh-delivery-view.node_haves", map[string]any{"node": node, "module": module}) + if err != nil { + logf("the build of %s that published cannot be read (%v): the direct-write window counts it at its next publish", caller, err) + return + } + var a struct { + Keys map[string]struct { + FP string `json:"fp"` + } `json:"keys"` + } + if json.Unmarshal(raw, &a) != nil || a.Keys[caller].FP == "" { + logf("node_haves names no build of %s: the direct-write window counts it at its next publish", caller) + return + } + learnt, err := learnInto(stdio.State("windows"), module, a.Keys[caller].FP, learnAttempts) + if err != nil { + logf("the publishing build of %s is not kept: %v", caller, err) + return + } + if learnt { + logf("%s's build %s publishes through the seats", module, a.Keys[caller].FP) + } + learned.Store(caller, true) +} + +// learnAttempts bounds learnInto's tries against other writers of the key. +const learnAttempts = 8 + +// casState is the part of a kept state learnInto writes through: a read with its revision, a create, an update at a +// revision (stdio.KeptState). +type casState interface { + Get(key string) (*stdio.StateEntry, error) + Create(key string, value any) (uint64, error) + Update(key string, value any, revision uint64) (uint64, error) +} + +// learnInto adds fp for module to the publishing builds kept at holder.PublishingKey with compare-and-set on the key's +// revision: two first publishes at once each read, add and write only if nobody wrote meanwhile, else read again, so +// neither is lost. It answers whether fp was new. +func learnInto(state casState, module, fp string, attempts int) (bool, error) { + for i := 0; i < attempts; i++ { + e, err := state.Get(holder.PublishingKey) + if err != nil { + return false, err + } + var cur holder.Publishing + if e != nil { + if err := json.Unmarshal(e.Value, &cur); err != nil { + return false, fmt.Errorf("%s holds no publishing builds: %w", holder.PublishingKey, err) + } + } + cur.Name = holder.PublishingKey + if !cur.Learn(module, fp) { + return false, nil + } + if e == nil { + _, err = state.Create(holder.PublishingKey, cur) + } else { + _, err = state.Update(holder.PublishingKey, cur, e.Revision) + } + if err == nil { + return true, nil + } + if !errors.Is(err, stdio.ErrChanged) { + return false, err + } + } + return false, fmt.Errorf("%s was written by another writer first %d times: %w", holder.PublishingKey, attempts, stdio.ErrChanged) +} + +// mirrors is the state mirrors: an ask is created once; one already kept is the same copy, taken on already. +type mirrors struct{} + +func (mirrors) Ask(key string, r holder.MirrorRequest) error { + if _, err := stdio.State(holder.MirrorsState).Create(key, r); err != nil && !errors.Is(err, stdio.ErrChanged) { + return err + } + return nil +} +func (mirrors) Claim(key string, r holder.MirrorRequest, revision uint64) (bool, error) { + _, err := stdio.State(holder.MirrorsState).Update(key, r, revision) + if errors.Is(err, stdio.ErrChanged) { + return false, nil + } + return err == nil, err +} +func (mirrors) Delete(key string) error { return stdio.State(holder.MirrorsState).Delete(key) } + +// followMirrors runs every copy `mirror` took on, on the state's watch: the asks kept when the bundle starts (a copy a +// restart cut) and each new one, one at a time. Each is claimed first (Verbs.Take), so a second follower copies none +// a first one took on. +func followMirrors(v *holder.Verbs) { + id := make([]byte, 8) + _, _ = rand.Read(id) + self := "distribution-tools-" + hex.EncodeToString(id) + <-stdio.Served() + stdio.State(holder.MirrorsState).Follow("", func(ch stdio.StateChange) error { + if ch.Op != stdio.OpPut { + return nil + } + var r holder.MirrorRequest + if err := json.Unmarshal(ch.Value, &r); err != nil { + logf("the mirror ask %s is no ask (%v): dropped", ch.Key, err) + return stdio.State(holder.MirrorsState).Delete(ch.Key) + } + return v.Take(ch.Key, r, ch.Revision, ch.Current, self) + }) +} + +func arg[T any](args map[string]any) (T, error) { + var v T + clean := map[string]any{} + for k, x := range args { + if k != "_meta" { + clean[k] = x + } + } + b, _ := json.Marshal(clean) + return v, json.Unmarshal(b, &v) +} + +func str(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + +// tools are distribution's tools over v; each call that checks a caller reads the caller list on that call (callers: +// the config value as mesh-config resolves it now), so an operator's change of it holds from the next call on. +func tools(v *holder.Verbs, windows kept[holder.Window], callers func() (holder.Callers, error)) []stdio.Tool { + now := func() (*holder.Verbs, error) { + c, err := callers() + if err != nil { + return nil, fmt.Errorf("the caller list cannot be read, so no verb is answered: %w", err) + } + return v.WithCallers(c), nil + } + obj := func(props map[string]any, req ...string) map[string]any { + return map[string]any{"type": "object", "required": req, "properties": props} + } + s := func(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + var out []stdio.Tool + for _, seat := range holder.Seats { + out = append(out, + stdio.Tool{Name: seat + "." + holder.VerbPublish, Description: "Open a transfer of a publish (ADR 0611): its name, version, kind, digest and parts; answers the parts the holder lacks.", + Input: obj(map[string]any{"transfer": s("the transfer's id"), "name": s("/"), "version": s("the builder's version"), "kind": s("image, bundle, archive or upstream"), "digest": s("sha256:…"), "parts": map[string]any{"type": "array"}}, "transfer", "name", "kind", "digest", "parts"), + Run: func(args map[string]any) (any, error) { + v, err := now() + if err != nil { + return nil, err + } + in, err := arg[reg.PublishInput](args) + if err != nil { + return nil, err + } + _, _ = v.DropIdle() + a, err := v.Publish(seat, stdio.CallerOf(args), publishOf(in)) + if err == nil && closeBootstrap != nil { + closeBootstrap() + } + if err == nil { + learnPublishing(windows, stdio.CallerOf(args)) + } + return publishAnswer(a), err + }}, + stdio.Tool{Name: seat + "." + holder.VerbPublishPart, Description: "Send one chunk (at most 1 MiB) of one part of an open transfer, from the caller that opened it.", + Input: obj(map[string]any{"transfer": s("its id"), "part": s("the part's digest"), "sequence": map[string]any{"type": "integer"}, "last": map[string]any{"type": "boolean"}, "data": s("base64")}, "transfer", "part", "sequence", "last", "data"), + Run: func(args map[string]any) (any, error) { + v, err := now() + if err != nil { + return nil, err + } + in, err := arg[reg.PublishPartInput](args) + if err != nil { + return nil, err + } + r, err := partOf(in) + if err != nil { + return nil, err + } + a, err := v.PublishPart(seat, stdio.CallerOf(args), r) + return partAnswer(in, a), err + }}, + stdio.Tool{Name: seat + "." + holder.VerbVersions, Description: "The versions the seat holds of a name.", Input: obj(map[string]any{"name": s("/")}, "name"), + Run: func(args map[string]any) (any, error) { + v, err := now() + if err != nil { + return nil, err + } + a, err := v.Versions(seat, stdio.CallerOf(args), str(args, "name")) + if err != nil { + return nil, err + } + return versionsAnswer(v, str(args, "kind"), a), nil + }}, + stdio.Tool{Name: seat + "." + holder.VerbRemove, Description: "Remove a tagged version (the builder), or for retention an untagged one by its digest (the artifact sweep; an index with the platform manifests no tagged or held index names, and never a manifest such an index names); says removed.", + Input: obj(map[string]any{"name": s("name"), "version": s("a tagged version"), "tag": s("its tag"), "digest": s("for retention"), "reason": s("why")}, "name", "reason"), + Run: func(args map[string]any) (any, error) { + v, err := now() + if err != nil { + return nil, err + } + in, err := arg[reg.RemoveInput](args) + if err != nil { + return nil, err + } + r := holder.RemoveRequest{Name: in.Name, Version: in.Version, Digest: in.Digest, Reason: in.Reason} + if err := v.Remove(seat, stdio.CallerOf(args), r); err != nil { + return nil, err + } + return reg.RemoveAnswer{Removed: true, Artifact: ®.Artifact{Name: in.Name, Version: in.Version, Tag: in.Tag, Digest: in.Digest}}, nil + }}, + stdio.Tool{Name: seat + "." + holder.VerbHold, Description: "Keep one untagged bundle, archive or copied base from retention (the artifact sweep alone); says held.", + Input: obj(map[string]any{"name": s("name"), "digest": s("sha256:…"), "reason": s("which of the sweep's rules keeps it")}, "name", "digest", "reason"), + Run: func(args map[string]any) (any, error) { + v, err := now() + if err != nil { + return nil, err + } + in, err := arg[reg.HoldInput](args) + if err != nil { + return nil, err + } + if err := v.HoldOne(seat, stdio.CallerOf(args), holder.HoldRequest{Name: in.Name, Digest: in.Digest, Why: in.Reason}); err != nil { + return nil, err + } + return reg.HoldAnswer{Held: true, Since: time.Now().UTC().Format(time.RFC3339)}, nil + }}, + stdio.Tool{Name: seat + "." + holder.VerbRelease, Description: "Let a hold go (the artifact sweep, or the operator's session through registry_holds); says released.", + Input: obj(map[string]any{"name": s("name"), "digest": s("sha256:…"), "reason": s("why it is let go")}, "name", "digest", "reason"), + Run: func(args map[string]any) (any, error) { + v, err := now() + if err != nil { + return nil, err + } + in, err := arg[reg.ReleaseInput](args) + if err != nil { + return nil, err + } + released, err := v.Release(seat, stdio.CallerOf(args), holder.HoldRequest{Name: in.Name, Digest: in.Digest, Why: in.Reason}) + if err != nil { + return nil, err + } + return reg.ReleaseAnswer{Released: released}, nil + }}, + ) + } + out = append(out, + stdio.Tool{Name: holder.ImageSeat + "." + holder.VerbTags, Description: "The tags of an image.", Input: obj(map[string]any{"name": s("name")}, "name"), + Run: func(args map[string]any) (any, error) { + v, err := now() + if err != nil { + return nil, err + } + a, err := v.Tags(holder.ImageSeat, stdio.CallerOf(args), str(args, "name")) + if err != nil { + return nil, err + } + return tagsAnswer(v, a), nil + }}, + stdio.Tool{Name: holder.ImageSeat + "." + holder.VerbMirror, Description: "Copy an outside image, pinned by its digest, into the store as upstream// (every platform, only the parts it lacks): answers at once with the reference the copy will have, and says published (or refused, with why) on the seat's events when the copy ends; wait on that event (mesh-build-image alone).", + Input: obj(map[string]any{"source": s("/"), "digest": s("sha256:…")}, "source", "digest"), + Run: func(args map[string]any) (any, error) { + v, err := now() + if err != nil { + return nil, err + } + in, err := arg[reg.MirrorInput](args) + if err != nil { + return nil, err + } + ref, err := v.Mirror(holder.ImageSeat, stdio.CallerOf(args), holder.MirrorRequest{Source: in.Source, Digest: in.Digest}) + if err != nil { + return nil, err + } + return reg.MirrorAnswer{Digest: in.Digest, Reference: ref}, nil + }}, + stdio.Tool{Name: holder.CallersTool, Description: "Who may call which verb of distribution's seats: the trusted config value callers as mesh-config resolves it on this machine now (design 51's list where unset). The program asks it at each wake of the windows, as it has no config of its own.", + Input: obj(map[string]any{}), + Run: func(map[string]any) (any, error) { + c, err := callers() + if err != nil { + return nil, err + } + return map[string]any{"callers": c}, nil + }}, + stdio.Tool{Name: "registry_health", Description: "distribution's health check, run by the runtime each minute: versions of each seat asked through the store's read side (the program's port 5000); fails naming the program when it does not answer.", + Input: obj(map[string]any{}), + Run: func(map[string]any) (any, error) { + front, _ := url.Parse("http://127.0.0.1:5000") + return holder.Health(&holder.OCI{Base: front}, finding) + }}, + stdio.Tool{Name: "registry_incoming", Description: "Open transfers of incoming and its fill; with drop, drops one by hand (a hand act, why required).", + Input: obj(map[string]any{"drop": s("a transfer id to drop"), "why": s("why it is dropped")}), + Run: func(args map[string]any) (any, error) { + v, err := now() + if err != nil { + return nil, err + } + idle, _ := v.DropIdle() + if id := str(args, "drop"); id != "" { + if str(args, "why") == "" { + return nil, errors.New("a drop is a hand act: say why") + } + t, err := v.Drop(id) + logf("transfer %s dropped by hand by %s: %s", id, stdio.CallerOf(args), str(args, "why")) + return map[string]any{"dropped": t.ID}, err + } + all, err := v.Transfers.List() + used, _ := v.Objects.Used() + return map[string]any{"transfers": all, "used": used, "cap": holder.IncomingCap, "dropped_idle": idle}, err + }}, + stdio.Tool{Name: "registry_holds", Description: "What is held, since when and for whom; with release, lets one go through the seat's release as the operator's session (a hand act).", + Input: obj(map[string]any{"seat": s("the seat"), "release": s("the name to release"), "digest": s("its digest"), "why": s("why")}), + Run: func(args map[string]any) (any, error) { + v, err := now() + if err != nil { + return nil, err + } + if name := str(args, "release"); name != "" { + if str(args, "why") == "" { + return nil, errors.New("a release by hand is a hand act: say why") + } + // The caller proved on this call (the operator's session) is carried on; distribution's own never. + released, err := v.Release(str(args, "seat"), stdio.CallerOf(args), holder.HoldRequest{Name: name, Digest: str(args, "digest"), Why: str(args, "why")}) + if err == nil && !released { + return map[string]any{"released": false, "note": name + " is not held: nothing was let go"}, nil + } + return map[string]any{"released": name}, err + } + return v.HeldNow() + }}, + ) + for _, w := range []string{holder.OldNameWindow, holder.DirectWritesWindow} { + name := "registry_old_name" + if w == holder.DirectWritesWindow { + name = "registry_direct_writes" + } + out = append(out, stdio.Tool{Name: name, Description: "Whether the " + w + " window is open and the nodes holding it, since when; with drop, drops a retired node from it by hand (a hand act, why required).", + Input: obj(map[string]any{"drop": s("a retired node"), "why": s("why")}), + Run: func(args map[string]any) (any, error) { + cur, _, err := windows.Get(w) + if err != nil { + return nil, err + } + cur.Name = w + if node := str(args, "drop"); node != "" { + if cur, err = cur.Drop(node, stdio.CallerOf(args), str(args, "why"), time.Now().UTC()); err != nil { + return nil, err + } + if err := windows.Put(cur); err != nil { + return nil, err + } + } + b, _, err := windows.Get(holder.BootstrapWindow) + if err != nil { + return nil, err + } + // The program removes the file when this window closes and writes it closed again with its removal's time. + return map[string]any{"window": cur, "bootstrap_credential_discarded": !b.Closed.IsZero(), "bootstrap": b}, nil + }}) + } + return out +} + +func main() { + base, _ := url.Parse("http://127.0.0.1:5001") + store := &holder.OCI{Base: base, Upstream: &holder.Upstream{}} + h := &holder.Holder{Objects: parts.BusObjects(holder.IncomingObjects), Builder: delivery.BuilderVersions{Ask: stdio.Ask}, Store: store, Say: say, Finding: finding, + Transfers: parts.BusTransfers[holder.Opening](holder.TransfersState)} + v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{}} + go followMirrors(v) + windows := kept[holder.Window]{state: "windows", key: func(w holder.Window) string { return w.Name }} + closeBootstrap = bootstrapCloser(windows) + if err := stdio.Serve("", tools(v, windows, holder.ConfiguredCallers)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/distribution/cmd/distribution-tools/publishing_test.go b/modules/distribution/cmd/distribution-tools/publishing_test.go new file mode 100644 index 00000000..5a208579 --- /dev/null +++ b/modules/distribution/cmd/distribution-tools/publishing_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "encoding/json" + "reflect" + "testing" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "distribution/internal/holder" +) + +// casMem is a kept state with revisions; before writes it lets another writer write first, once. +type casMem struct { + value []byte + rev uint64 + before func() +} + +func (m *casMem) Get(string) (*stdio.StateEntry, error) { + if m.value == nil { + return nil, nil + } + return &stdio.StateEntry{Value: m.value, Revision: m.rev}, nil +} +func (m *casMem) write(v any, at uint64) (uint64, error) { + if f := m.before; f != nil { + m.before = nil + f() + } + if m.rev != at { + return 0, stdio.ErrChanged + } + m.value, _ = json.Marshal(v) + m.rev++ + return m.rev, nil +} +func (m *casMem) Create(_ string, v any) (uint64, error) { return m.write(v, 0) } +func (m *casMem) Update(_ string, v any, rev uint64) (uint64, error) { return m.write(v, rev) } + +// TestTwoFirstPublishesAtOnceAreBothKept: the publishing builds are written with compare-and-set, so a writer that lost +// the race reads again and neither build is lost (A3f finding 2). +func TestTwoFirstPublishesAtOnceAreBothKept(t *testing.T) { + m := &casMem{} + m.before = func() { + // Another tools process learns mesh-build-go's build first. + m.value, _ = json.Marshal(holder.Publishing{Name: holder.PublishingKey, Modules: map[string][]string{"mesh-build-go": {"fp-go"}}}) + m.rev = 1 + } + if learnt, err := learnInto(m, "mesh-build-node", "fp-node", 3); err != nil || !learnt { + t.Fatalf("learnInto = %v, %v", learnt, err) + } + var got holder.Publishing + _ = json.Unmarshal(m.value, &got) + want := map[string][]string{"mesh-build-go": {"fp-go"}, "mesh-build-node": {"fp-node"}} + if !reflect.DeepEqual(got.Modules, want) { + t.Errorf("kept %v, want both builds %v", got.Modules, want) + } + if learnt, err := learnInto(m, "mesh-build-node", "fp-node", 3); err != nil || learnt { + t.Errorf("learning a kept build again = %v, %v, want not new and no write", learnt, err) + } + m.before = func() { m.rev++ } + if _, err := learnInto(m, "mesh-build-oci", "fp-oci", 1); err == nil { + t.Error("a write lost on every attempt answered no error") + } +} diff --git a/modules/distribution/cmd/distribution/main.go b/modules/distribution/cmd/distribution/main.go new file mode 100644 index 00000000..242cc0b4 --- /dev/null +++ b/modules/distribution/cmd/distribution/main.go @@ -0,0 +1,163 @@ +// Command distribution is distribution's program (novox/hq design 51): it serves the artifact store's port in front of +// the registry container, which listens on its node's loopback only. Reads (GET, HEAD) go through to every node's +// container runtime; a write is refused but while the direct-write window is open, never over a tag the seat holds +// published, or with the bootstrap's one-off credential (holder.Front). The windows are distribution's state windows: +// the program moves them (windows.go) and follows the direct-write window's record on each change; it holds no timer. +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "log" + "net/http" + "net/url" + "os" + "os/signal" + "strings" + "sync/atomic" + "syscall" + "time" + + "git.novox.be/novox/mesh-sdk/go/process" + "github.com/nats-io/nats.go/jetstream" + + "distribution/internal/holder" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := run(ctx, os.Getenv); err != nil { + fmt.Fprintln(os.Stderr, "distribution:", err) + os.Exit(1) + } +} + +func run(ctx context.Context, getenv func(string) string) error { + logger := log.New(os.Stderr, "distribution: ", log.LstdFlags) + store, err := url.Parse(or(getenv("MESH_REGISTRY_LOOPBACK"), "http://127.0.0.1:5001")) + if err != nil { + return err + } + oci := &holder.OCI{Base: store} + var open atomic.Bool + open.Store(true) // until the window's state says it closed: no wider than today's store + cred, err := process.ReadBrokerAccount(getenv) + if err != nil { + return err + } + conn, js, err := cred.Dial() + if err != nil { + return err + } + defer conn.Close() + // The direct-write window's record, written by the tools bundle on distribution's own account. + windows := process.OptionalBucket(ctx, js, "distribution_windows", "the direct-write window is read as open until the program starts again", logger.Printf) + if windows != nil { + go follow(ctx, windows, &open, logger) + go followBootstrap(ctx, windows, logger) + calls, err := process.Connect(cred) + if err != nil { + return err + } + defer calls.Close() + findings := process.OptionalBucket(ctx, js, "distribution_findings", "a window held for seven days is said in the log only", logger.Printf) + go drive(ctx, js, cred, &holder.Driver{Delivery: busDelivery{calls}, State: kvWindows{windows}, Callers: busCallers{calls}.Read, + Closed: sayClosed(js, cred), Finding: findingsKV(findings, logger.Printf)}, logger) + } + front := &holder.Front{Store: store, Window: &open, Logf: logger.Printf, + Published: func(name, tag string) (bool, error) { + d, err := oci.Digest(name, tag) + return d != "", err + }, + Bootstrap: func() (string, bool) { + b, err := os.ReadFile(holder.BootstrapFile) + if err != nil { + return "", false + } + return strings.TrimSpace(string(b)), true + }} + srv := &http.Server{Addr: or(getenv("MESH_REGISTRY_LISTEN"), ":5000"), Handler: front, ReadHeaderTimeout: 30 * time.Second} + errs := make(chan error, 1) + go func() { errs <- srv.ListenAndServe() }() + logger.Printf("serving the store's port %s in front of %s", srv.Addr, store) + select { + case <-ctx.Done(): + shut, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + return srv.Shutdown(shut) + case err := <-errs: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + } +} + +// follow keeps open as the direct-write window's record says, on every change of it (a watch, no timer). +func follow(ctx context.Context, kv jetstream.KeyValue, open *atomic.Bool, logger *log.Logger) { + w, err := kv.Watch(ctx, holder.DirectWritesWindow) + if err != nil { + logger.Printf("the direct-write window cannot be watched (%v): read as open", err) + return + } + for e := range w.Updates() { + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + var win holder.Window + if json.Unmarshal(e.Value(), &win) != nil { + continue + } + closed := !win.Open && !win.Closed.IsZero() + if closed && open.Swap(false) { + logger.Printf("the direct-write window is closed: every write but distribution's own is refused, naming publish") + } + } +} + +// followBootstrap removes the bootstrap's one-off credential (holder.BootstrapFile) when the tools bundle closes the +// window bootstrap-push at its first answered publish (design 51 *On a fresh mesh*): the program runs as distribution's +// account, which owns the file. A file already gone is no fault; one that cannot be removed is said each time the +// window is written. +func followBootstrap(ctx context.Context, kv jetstream.KeyValue, logger *log.Logger) { + w, err := kv.Watch(ctx, holder.BootstrapWindow) + if err != nil { + logger.Printf("the bootstrap's credential window cannot be watched (%v): its file stays until the program starts again", err) + return + } + for e := range w.Updates() { + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + var win holder.Window + if json.Unmarshal(e.Value(), &win) != nil || win.Closed.IsZero() { + continue + } + if why := removeBootstrap(holder.BootstrapFile); why != "" { + logger.Printf("%s", why) + } + } +} + +// removeBootstrap removes the credential's file; it answers what went wrong, or "" when the file is gone. +func removeBootstrap(path string) string { + switch err := os.Remove(path); { + case err == nil: + return "the bootstrap's one-off credential is discarded: the holders serve publish" + case errors.Is(err, fs.ErrNotExist): + return "" + default: + return fmt.Sprintf("the bootstrap's one-off credential could not be discarded, so it is still accepted: %v", err) + } +} + +func or(v, d string) string { + if strings.TrimSpace(v) == "" { + return d + } + return v +} diff --git a/modules/distribution/cmd/distribution/main_test.go b/modules/distribution/cmd/distribution/main_test.go new file mode 100644 index 00000000..f9ef3acf --- /dev/null +++ b/modules/distribution/cmd/distribution/main_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The program, which runs as distribution's account, removes the bootstrap's one-off credential when its window +// closes (design 51 *On a fresh mesh*; a3-publish-contract): removed once, a file already gone is no fault, and one +// that cannot be removed is said, since the front still accepts it. +func TestTheProgramDiscardsTheBootstrapCredential(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "bootstrap-push") + if err := os.WriteFile(file, []byte("token\n"), 0o600); err != nil { + t.Fatal(err) + } + if why := removeBootstrap(file); !strings.Contains(why, "discarded") { + t.Fatalf("first removal = %q, want it said discarded", why) + } + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatalf("the file is still there: %v", err) + } + if why := removeBootstrap(file); why != "" { + t.Fatalf("a file already gone = %q, want nothing said", why) + } + stuck := filepath.Join(dir, "stuck") + if err := os.MkdirAll(filepath.Join(stuck, "child"), 0o700); err != nil { + t.Fatal(err) + } + if why := removeBootstrap(stuck); !strings.Contains(why, "still accepted") { + t.Fatalf("an unremovable file = %q, want it said still accepted", why) + } +} diff --git a/modules/distribution/cmd/distribution/whole_integration_test.go b/modules/distribution/cmd/distribution/whole_integration_test.go new file mode 100644 index 00000000..c60bb57b --- /dev/null +++ b/modules/distribution/cmd/distribution/whole_integration_test.go @@ -0,0 +1,233 @@ +//go:build integration + +package main + +// distribution's program whole against a real bus (CAT-D7; design 51 *Acceptance tests*, the holder's cases): its +// run() as the node starts it, its broker account on a nats-server of this test alone, the registry container a fake +// OCI store on the loopback, the status view's node_haves and the node-engine seat's manifest-reads answered on their +// call subjects. An away node holds the old name open; its return (manifest-reads-changed) closes it, said once on +// distribution.events, after which a direct write is refused naming publish; reads stay open; a waiter on a seat's +// `published` hears none of the windows' events; and the build modules counted are publish's callers as the tools +// bundle answers the config value, not design 51's defaults. + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + neturl "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/testbus" + + "distribution/internal/holder" +) + +func TestDistributionWhole(t *testing.T) { + url := testbus.URL(t) + nc, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, _ := jetstream.New(nc) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + for _, b := range []string{"distribution_windows", "distribution_findings"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "DISTRIBUTION_EVENTS", Subjects: []string{"mesh.stream.distribution.event.>"}, Duplicates: time.Hour}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "novox_distribution", AckPolicy: jetstream.AckExplicitPolicy, + FilterSubjects: []string{"mesh.mod.mesh-delivery-view.event.delivered", "mesh.seat.node-engine.event.manifest-reads-changed.*"}}); err != nil { + t.Fatal(err) + } + + // The status view and the node-engines: g14 away (its node-engine answers nothing) and running the old build. + var mu sync.Mutex + g14 := false + answer := func(m *nats.Msg, result any) { + b, _ := json.Marshal(map[string]any{"result": result, "node": "novox"}) + _ = m.Respond(b) + } + if _, err := nc.Subscribe("mesh.mod.mesh-delivery-view.>", func(m *nats.Msg) { + mu.Lock() + defer mu.Unlock() + fp := "fp-old" + if g14 { + fp = "fp-new" + } + answer(m, map[string]any{"keys": map[string]any{ + "ace.mesh-build-go": map[string]any{"fp": "fp-new"}, "g14.mesh-build-go": map[string]any{"fp": fp}, + "ace.gitea": map[string]any{"fp": "x"}, "g14.gitea": map[string]any{"fp": "x"}, + // A build module the operator took off publish's list: it holds no window (the caller list as configured). + "g14.mesh-build-node": map[string]any{"fp": "fp-node-old"}, + }}) + }); err != nil { + t.Fatal(err) + } + // distribution's own tools bundle answers the config value callers as mesh-config resolved it: publish by + // mesh-build-go alone. The program reads it there, never from an environment variable nothing renders. + if _, err := nc.Subscribe("mesh.mod.distribution.>", func(m *nats.Msg) { + if !strings.Contains(m.Subject, holder.CallersTool) { + return + } + answer(m, map[string]any{"callers": map[string]any{"publish": []string{"mesh-build-go"}}}) + }); err != nil { + t.Fatal(err) + } + engine := func(node string) { + for _, subj := range []string{"mesh.seat.node-engine.call.manifest-reads." + node + ".>", "mesh.seat.node-engine.tool.manifest-reads." + node} { + if _, err := nc.Subscribe(subj, func(m *nats.Msg) { + answer(m, map[string]any{"v": 1, "manifest-reads": []string{holder.NewNamesCapability}}) + }); err != nil { + t.Fatal(err) + } + } + _ = nc.Flush() + } + engine("ace") // g14 is away: nothing answers its node-engine, so it states nothing + // A waiter on a seat's published: it must hear none of the windows' events. + var woke []string + if _, err := nc.Subscribe("mesh.seat.container-image-registry.event.published", func(m *nats.Msg) { woke = append(woke, m.Subject) }); err != nil { + t.Fatal(err) + } + pub, _ := json.Marshal(holder.Publishing{Name: holder.PublishingKey, Modules: map[string][]string{"mesh-build-go": {"fp-new"}}}) + windows, _ := js.KeyValue(ctx, "distribution_windows") + if _, err := windows.Put(ctx, holder.PublishingKey, pub); err != nil { + t.Fatal(err) + } + + // The registry container on the loopback, and the program's port. + store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/tags/list") { + _, _ = io.WriteString(w, `{"tags":[]}`) + return + } + if r.Method == http.MethodHead { + w.WriteHeader(http.StatusNotFound) // no tag published through the seat + return + } + w.WriteHeader(http.StatusCreated) + })) + defer store.Close() + l, _ := net.Listen("tcp", "127.0.0.1:0") + listen := l.Addr().String() + l.Close() + dir := t.TempDir() + acct, _ := json.Marshal(map[string]any{"url": url, "node": "novox", "module": "distribution", "user": "novox.distribution", "password": "unused"}) + _ = os.WriteFile(filepath.Join(dir, "broker"), acct, 0o600) + env := map[string]string{"MESH_BROKER_FILE": filepath.Join(dir, "broker"), "MESH_REGISTRY_LOOPBACK": store.URL, "MESH_REGISTRY_LISTEN": listen} + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- run(runCtx, func(k string) string { return env[k] }) }() + defer func() { stop(); <-done }() + + window := func(name string) holder.Window { + var w holder.Window + if e, err := windows.Get(ctx, name); err == nil { + _ = json.Unmarshal(e.Value(), &w) + } + return w + } + eventually := func(what string, ok func() bool) { + t.Helper() + for end := time.Now().Add(20 * time.Second); time.Now().Before(end); time.Sleep(50 * time.Millisecond) { + if ok() { + return + } + select { + case err := <-done: + t.Fatalf("the program ended: %v", err) + default: + } + } + t.Fatalf("not within 20s: %s (old-name %+v, direct-writes %+v)", what, window(holder.OldNameWindow), window(holder.DirectWritesWindow)) + } + write := func() int { + req, _ := http.NewRequest(http.MethodPut, "http://"+listen+"/v2/m/i/manifests/0.1.9", nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return 0 + } + resp.Body.Close() + return resp.StatusCode + } + eventually("the start's wake: g14 holds both windows", func() bool { + o, d := window(holder.OldNameWindow), window(holder.DirectWritesWindow) + return o.Open && fmt.Sprint(o.Holding) == "[g14]" && d.Open && fmt.Sprint(d.Holding) == "[g14]" + }) + if got := write(); got != http.StatusCreated { + t.Errorf("a direct write while the window is open = %d, want it through", got) + } + + // g14 returns with the new node-engine and the new build: its manifest-reads-changed closes both windows. + mu.Lock() + g14 = true + mu.Unlock() + engine("g14") + if _, err := js.Publish(ctx, "mesh.seat.node-engine.event.manifest-reads-changed.g14", []byte(`{"v":1,"node":"g14","why":"returned"}`)); err != nil { + t.Fatal(err) + } + eventually("g14's return closes both windows", func() bool { + return !window(holder.OldNameWindow).Closed.IsZero() && !window(holder.DirectWritesWindow).Closed.IsZero() + }) + // A delivered after the close says nothing again. + if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-view.event.delivered", []byte(`{}`)); err != nil { + t.Fatal(err) + } + eventually("a direct write refused naming publish once closed", func() bool { return write() == http.StatusForbidden }) + resp, err := http.Get("http://" + listen + "/v2/m/i/tags/list") + if err != nil || resp.StatusCode != http.StatusOK { + t.Errorf("a read once the window closed: %v %v", resp, err) + } + if _, err := holder.Health(&holder.OCI{Base: mustURL("http://" + listen)}, nil); err != nil { + t.Errorf("the health check through the program's port: %v", err) + } + s, _ := js.Stream(ctx, "DISTRIBUTION_EVENTS") + eventually("the program heard every waking event, the delivered after the close included", func() bool { + c, err := js.Consumer(ctx, "EVENTS", "novox_distribution") + if err != nil { + return false + } + i, err := c.Info(ctx) + return err == nil && i.NumPending == 0 && i.NumAckPending == 0 && i.Delivered.Consumer >= 2 + }) + info, _ := s.Info(ctx) + if info.State.Msgs != 2 { + t.Errorf("distribution.events holds %d events, want old-name-closed and direct-writes-closed once each", info.State.Msgs) + } + for _, ev := range []string{holder.OldNameClosedEvent, holder.DirectWritesClosedEvent} { + if _, err := s.GetLastMsgForSubject(ctx, "mesh.stream.distribution.event."+ev); err != nil { + t.Errorf("%s not said: %v", ev, err) + } + } + if len(woke) != 0 { + t.Errorf("a waiter on published woke on %v", woke) + } +} + +func mustURL(s string) *neturl.URL { + u, err := neturl.Parse(s) + if err != nil { + panic(err) + } + return u +} diff --git a/modules/distribution/cmd/distribution/windows.go b/modules/distribution/cmd/distribution/windows.go new file mode 100644 index 00000000..03734c82 --- /dev/null +++ b/modules/distribution/cmd/distribution/windows.go @@ -0,0 +1,260 @@ +package main + +// The windows' driver (design 51 *The move*): at the program's start, on each `delivered` of the status view and on each +// node-engine's `manifest-reads-changed` (ADR 0618 §4) it reads every node through node_haves and the node-engine +// seat's manifest-reads, and steps the old-name and the direct-write windows in distribution's state windows. The +// program does it, not the tools bundle: it holds distribution's own account, its durable consumer of the events +// stream and its own stream for the closing events. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/process" + + "distribution/internal/holder" +) + +// The addresses the driver asks (manifest invokes). +const ( + havesAsk = "mesh-delivery-view.node_haves" + readsAsk = "seat:node-engine.manifest-reads" + // callersAsk is distribution's own tool answering its config value callers as resolved. + callersAsk = holder.Self + "." + holder.CallersTool +) + +// caller is a process's call of a tool or a seat verb. +type caller interface { + Call(key string, body any) (json.RawMessage, error) + CallOn(node, key string, body any) (json.RawMessage, error) +} + +// busCallers reads the caller list from distribution's own tools bundle (holder.CallersTool, manifest invokes): a +// program has no runtime that resolves its module's config values, the tools bundle has (stdio.Config), so the program +// asks it on each wake rather than read an environment variable nothing renders. +type busCallers struct{ c caller } + +func (b busCallers) Read() (holder.Callers, error) { + raw, err := b.c.Call(callersAsk, map[string]any{}) + if err != nil { + return nil, fmt.Errorf("%s: %w", callersAsk, err) + } + var a struct { + Callers json.RawMessage `json:"callers"` + } + if err := json.Unmarshal(unwrap(raw), &a); err != nil || len(a.Callers) == 0 { + return nil, fmt.Errorf("%s answered what is not a caller list: %s", callersAsk, unwrap(raw)) + } + return holder.ParseCallers(string(a.Callers)) +} + +// busDelivery answers holder.Delivery through the bus. +type busDelivery struct{ c caller } + +func (d busDelivery) Haves() ([]holder.NodeHave, error) { + raw, err := d.c.Call(havesAsk, map[string]any{}) + if err != nil { + return nil, err + } + var a struct { + Keys map[string]struct { + FP string `json:"fp"` + } `json:"keys"` + Cut bool `json:"cut"` + } + if err := json.Unmarshal(unwrap(raw), &a); err != nil { + return nil, fmt.Errorf("node_haves answered what is not its listing: %w", err) + } + if a.Cut { + return nil, errors.New("node_haves cut its answer, so not every node is read") + } + out := make([]holder.NodeHave, 0, len(a.Keys)) + for key, h := range a.Keys { + node, module, ok := strings.Cut(key, ".") + if ok { + out = append(out, holder.NodeHave{Node: node, Module: module, FP: h.FP}) + } + } + return out, nil +} + +func (d busDelivery) Reads(node string) (bool, []string, error) { + raw, err := d.c.CallOn(node, readsAsk, map[string]any{}) + if err != nil { + if notServed(err) { + return false, nil, nil // a node-engine not serving manifest-reads states nothing (design 51) + } + return false, nil, err + } + var a struct { + Reads []string `json:"manifest-reads"` + } + if err := json.Unmarshal(unwrap(raw), &a); err != nil { + return true, nil, fmt.Errorf("%s's manifest-reads answered what is not its statement: %w", node, err) + } + return true, a.Reads, nil +} + +// notServed is whether a call's error says nothing serves the address (an away node, an older node-engine). +func notServed(err error) bool { + if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrTimeout) || errors.Is(err, context.DeadlineExceeded) { + return true + } + m := strings.ToLower(err.Error()) + for _, w := range []string{"no responders", "serves no verb", "not served", "unknown tool", "timeout"} { + if strings.Contains(m, w) { + return true + } + } + return false +} + +// unwrap takes a tool's answer out of the MCP content form the tool runner may answer in. +func unwrap(raw json.RawMessage) json.RawMessage { + var s string + if json.Unmarshal(raw, &s) == nil { + return unwrap(json.RawMessage(s)) + } + var c struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + } + if json.Unmarshal(raw, &c) == nil && len(c.Content) > 0 { + return json.RawMessage(c.Content[0].Text) + } + return raw +} + +// kvWindows answers holder.WindowState over the state windows. +type kvWindows struct{ kv jetstream.KeyValue } + +func (k kvWindows) get(key string, v any) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + e, err := k.kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil + } + if err != nil { + return err + } + return json.Unmarshal(e.Value(), v) +} + +func (k kvWindows) Window(name string) (holder.Window, error) { + var w holder.Window + return w, k.get(name, &w) +} + +func (k kvWindows) PutWindow(w holder.Window) error { + b, _ := json.Marshal(w) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := k.kv.Put(ctx, w.Name, b) + return err +} + +func (k kvWindows) Publishing() (holder.Publishing, error) { + var p holder.Publishing + return p, k.get(holder.PublishingKey, &p) +} + +// findingsKV writes the driver's findings into distribution's state findings, as the tools bundle writes its own. +func findingsKV(kv jetstream.KeyValue, logf func(string, ...any)) func(kind, subject, words string, raised bool) { + return func(kind, subject, words string, raised bool) { + if kv == nil { + return + } + key := health.FindingKey(health.ScopeSeat, holder.Self, kind+"."+strings.NewReplacer(".", "_", "@", "_", "/", "_").Replace(subject)) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if !raised { + _ = kv.Delete(ctx, key) + return + } + f := health.Finding{V: health.FindingVersion, Key: key, Kind: kind, Ref: key[strings.LastIndex(key, ".")+1:], + Subject: health.Subject{Scope: health.ScopeSeat, ID: holder.Self}, Severity: health.Warning, + Summary: words, Headline: words, Explanation: words + "; registry_old_name or registry_direct_writes drops a retired node by hand", + Resolved: "cleared when the window closes", Source: kind, Observer: holder.Self, At: time.Now().UTC()} + b, _ := json.Marshal(f) + if _, err := kv.Put(ctx, key, b); err != nil { + logf("the finding %s could not be written: %v", key, err) + } + } +} + +// sayClosed says a window's closing on distribution's own family distribution.events, once (its id is the window's). +func sayClosed(js jetstream.JetStream, cred process.Account) func(holder.Closed) error { + return func(c holder.Closed) error { + event := holder.OldNameClosedEvent + if c.Window == holder.DirectWritesWindow { + event = holder.DirectWritesClosedEvent + } + b, _ := json.Marshal(c) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := js.Publish(ctx, cred.StreamSubject(event), b, jetstream.WithMsgID(holder.Self+"."+event)) + return err + } +} + +// wakes says which events wake the driver, by subject: the status view's delivered, a node-engine's +// manifest-reads-changed (its node the subject's last token). +func wakes(subject string) (string, bool) { + switch { + case strings.HasSuffix(subject, ".mesh-delivery-view.event.delivered"): + return "delivered", true + case strings.Contains(subject, ".node-engine.event.manifest-reads-changed."): + return "manifest-reads-changed on " + subject[strings.LastIndex(subject, ".")+1:], true + } + return "", false +} + +// drive wakes the driver at start and on each waking event of the program's durable consumer of the events stream. +// A wake that could not judge is said and its event asked again a minute later. +func drive(ctx context.Context, js jetstream.JetStream, cred process.Account, d *holder.Driver, logger *log.Logger) { + wake := func(why string) error { + said, err := d.Wake(why) + for _, s := range said { + logger.Printf("%s", s) + } + if err != nil { + logger.Printf("the windows were not judged (%s): %v", why, err) + } + return err + } + _ = wake("distribution started") + cons, err := cred.EventConsumer(ctx, js, process.EventsStream) + if err != nil { + logger.Printf("the events stream cannot be heard (%v): the windows move only at the next start", err) + return + } + cc, err := process.HearEvents(cons, func(m jetstream.Msg) { + why, ok := wakes(m.Subject()) + if !ok { + _ = m.Ack() + return + } + if wake(why) != nil { + _ = m.NakWithDelay(time.Minute) + return + } + _ = m.Ack() + }, logger.Printf) + if err != nil { + logger.Printf("the events stream cannot be heard (%v): the windows move only at the next start", err) + return + } + <-ctx.Done() + cc.Stop() +} diff --git a/modules/distribution/go.mod b/modules/distribution/go.mod new file mode 100644 index 00000000..a54291c7 --- /dev/null +++ b/modules/distribution/go.mod @@ -0,0 +1,23 @@ +module distribution + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/distribution/go.sum b/modules/distribution/go.sum new file mode 100644 index 00000000..7679a5dc --- /dev/null +++ b/modules/distribution/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/distribution/internal/holder/callers.go b/modules/distribution/internal/holder/callers.go new file mode 100644 index 00000000..c2731811 --- /dev/null +++ b/modules/distribution/internal/holder/callers.go @@ -0,0 +1,130 @@ +// Package holder is distribution's holding of the two image seats, `container-image-registry` and +// `oci-artifact-registry`, and of the old seat `mesh-artifact-store` until its window closes (novox/hq design 51 *The +// four registry seats*, ADR 0605, ADR 0611, ADR 0617 §2). A build module publishes into a seat through `publish` and +// `publish-part`: the parts land in distribution's own object store `incoming`, are hashed, checked against the +// builder's `builder_versions` and stored in the artifact store, which listens on its node's loopback; the store's port +// is the program's, reads open, writes from its own process alone (front.go). +package holder + +import ( + "fmt" + "slices" + "sort" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The seats distribution holds, and the old one it holds until its window closes. +const ( + ImageSeat = "container-image-registry" + ArtifactSeat = "oci-artifact-registry" + OldSeat = "mesh-artifact-store" +) + +// The verbs of the seats (design 26 *A seat's definition*; design 51 *One contract*). +const ( + VerbPublish = "publish" + VerbPublishPart = "publish-part" + VerbVersions = "versions" + VerbRemove = "remove" + VerbMirror = "mirror" + VerbTags = "tags" + VerbHold = "hold" + VerbRelease = "release" +) + +// Seats are the seats distribution answers verbs on. +var Seats = []string{ImageSeat, ArtifactSeat} + +// VerbsOf are a seat's verbs: mirror and tags on the image seat alone; hold and release on both of distribution's. +func VerbsOf(seat string) []string { + v := []string{VerbPublish, VerbPublishPart, VerbVersions, VerbRemove, VerbHold, VerbRelease} + if seat == ImageSeat { + v = append(v, VerbMirror, VerbTags) + } + sort.Strings(v) + return v +} + +// Callers is the holder's trusted config value `callers`: per verb, the callers that may call it, read as mesh-sdk's +// seat-neutral list per verb reads it (ADR 0627 §2). The holder checks the caller the tool runner proved and stamped on +// the request (design 25 § 12.4), never a name in the request's body. +type Callers = stdio.VerbCallers + +// DefaultCallers is the list design 51 *Who may call which verb* names: publish and publish-part the build modules; +// mirror mesh-build-image; remove the builder (a tagged version) and the artifact sweep's holder (retention); hold the +// artifact sweep's holder alone; release that holder, and the operator's session only through registry_holds (Operator +// below). versions is read by anyone on the mesh, as the store's reads are. +func DefaultCallers() Callers { + builds := []string{"mesh-build-go", "mesh-build-image", "mesh-build-node"} + return Callers{ + VerbPublish: builds, + VerbPublishPart: builds, + VerbMirror: {"mesh-build-image"}, + VerbRemove: {"mesh-delivery-builder", SweepHolder}, + VerbHold: {SweepHolder}, + VerbRelease: {SweepHolder, Operator}, + VerbVersions: {Anyone}, + VerbTags: {Anyone}, + } +} + +// SweepHolder is the module holding the seat `mesh-artifact-sweep`, the artifact sweep. +const SweepHolder = "artifact-store-tools" + +// The entries of a list that stand for a class, and the caller the runtime names for a call it could not prove. +const ( + Anyone = stdio.AnyCaller + Operator = stdio.AnyPerson + Unproven = stdio.Unproven +) + +// Self is distribution's own account: never on release's list, so registry_holds cannot release in its own name. +const Self = "distribution" + +// ParseCallers reads the config value; an empty value is DefaultCallers. A verb no seat has is refused, and Self on +// release's list is refused (design 51). +func ParseCallers(raw string) (Callers, error) { + c, err := stdio.ParseVerbCallers(raw, VerbsOf(ImageSeat), DefaultCallers()) + if err != nil { + return nil, err + } + return selfNotReleasing(c) +} + +// ConfiguredCallers is the config value `callers` as mesh-config resolved it for this machine, read through the tools +// runtime's stdio.Config on each call (never an environment variable: nothing renders a config value into a +// program's environment); unset is DefaultCallers, and Self on release's list is refused as ParseCallers refuses it. +func ConfiguredCallers() (Callers, error) { + c, err := stdio.ConfiguredVerbCallers(CallersConfig, VerbsOf(ImageSeat), DefaultCallers()) + if err != nil { + return nil, err + } + return selfNotReleasing(c) +} + +// CallersConfig is the config value's key, and CallersTool the tools bundle's tool answering it as resolved: the +// program, which has no tools runtime, asks that tool over the bus (manifest invokes). +const ( + CallersConfig = "callers" + CallersTool = "registry_callers" +) + +// WithCallers is v with the caller list c: a copy for one call, so a list read on that call never changes another's. +func (v *Verbs) WithCallers(c Callers) *Verbs { + h := *v.Holder + h.Callers = c + out := *v + out.Holder = &h + return &out +} + +func selfNotReleasing(c Callers) (Callers, error) { + if slices.Contains(c[VerbRelease], Self) { + return nil, fmt.Errorf("callers puts %s on release's list: registry_holds would release in its own name", Self) + } + return c, nil +} + +// CallerModule is the module of a proved module caller, which the runtime stamps `.`. +func CallerModule(proved string) string { return stdio.ModuleOf(proved) } diff --git a/modules/distribution/internal/holder/driver.go b/modules/distribution/internal/holder/driver.go new file mode 100644 index 00000000..d682923d --- /dev/null +++ b/modules/distribution/internal/holder/driver.go @@ -0,0 +1,195 @@ +package holder + +import ( + "errors" + "fmt" + "slices" + "sort" + "strings" + "time" +) + +// PublishingKey is the key of distribution's state windows that records, per build module, the fingerprints of the +// builds heard publishing through the seats: the tools bundle adds the have's fingerprint of the proved caller of each +// accepted publish. A node's have of a build module running one of them counts as publishing through the seats, so a +// node whose build module runs the same build but builds nothing (a paused build agent) does not hold the window. +const PublishingKey = "publishing" + +// Publishing is that record: module -> fingerprints. +type Publishing struct { + Name string `json:"name"` + Modules map[string][]string `json:"modules"` +} + +// Learn adds fp for module; it answers whether it was new. +func (p *Publishing) Learn(module, fp string) bool { + if module == "" || fp == "" { + return false + } + if p.Modules == nil { + p.Modules = map[string][]string{} + } + if slices.Contains(p.Modules[module], fp) { + return false + } + p.Modules[module] = append(p.Modules[module], fp) + sort.Strings(p.Modules[module]) + return true +} + +// NodeHave is one key of the status view's node_haves: a node, a module, the fingerprint it runs. +type NodeHave struct { + Node, Module, FP string +} + +// Delivery is what the windows' driver asks of the delivery parts and the node-engine seat, through their tools and +// verbs on the bus (ADR 0596): every node's haves, away ones included, and one node-engine's stated reader +// capabilities. Reads answers served false when the node-engine does not serve `manifest-reads` (it states nothing). +type Delivery interface { + Haves() ([]NodeHave, error) + Reads(node string) (served bool, stated []string, err error) +} + +// WindowState is distribution's state windows. +type WindowState interface { + Window(name string) (Window, error) + PutWindow(w Window) error + Publishing() (Publishing, error) +} + +// Driver moves the old-name and the direct-write windows: at distribution's start, on each `delivered` and on each +// node-engine's `manifest-reads-changed` (ADR 0618 §4), never on a timer. Each wake reads every node, since a window +// closes only when no node holds it. +type Driver struct { + Delivery Delivery + State WindowState + // Callers is the caller list as it holds on this wake: publish's callers are the build modules counted. + Callers func() (Callers, error) + // Closed says a window's closing on distribution.events. + Closed func(c Closed) error + Finding func(kind, subject, words string, raised bool) + Now func() time.Time +} + +// FindingWindowHeld is the finding of a node holding a window open for seven days. +const FindingWindowHeld = "window-held" + +// Read reads every node: its haves, its node-engine's stated capabilities, and whether each build module's have runs a +// build heard publishing through the seats. +func (d *Driver) Read() ([]NodeRead, error) { + haves, err := d.Delivery.Haves() + if err != nil { + return nil, fmt.Errorf("node_haves cannot be read, so no window moves: %w", err) + } + pub, err := d.State.Publishing() + if err != nil { + return nil, fmt.Errorf("the publishing builds cannot be read, so no window moves: %w", err) + } + callers, err := d.Callers() + if err != nil { + return nil, fmt.Errorf("the caller list cannot be read, so no window moves: %w", err) + } + builds := callers[VerbPublish] + byNode := map[string]*NodeRead{} + var order []string + for _, h := range haves { + if h.Module == "node" || h.Node == "" { + continue + } + n := byNode[h.Node] + if n == nil { + n = &NodeRead{Node: h.Node, NamesArtifact: true} + byNode[h.Node] = n + order = append(order, h.Node) + } + if slices.Contains(builds, h.Module) { + if n.BuildModules == nil { + n.BuildModules = map[string]bool{} + } + n.BuildModules[h.Module] = slices.Contains(pub.Modules[h.Module], h.FP) + } + } + sort.Strings(order) + out := make([]NodeRead, 0, len(order)) + for _, node := range order { + n := byNode[node] + served, stated, err := d.Delivery.Reads(node) + if err != nil && served { + return nil, fmt.Errorf("%s's manifest-reads cannot be read, so no window moves: %w", node, err) + } + n.Serves, n.Stated = served, stated + out = append(out, *n) + } + return out, nil +} + +// Wake reads every node and steps both windows; it answers a line for each thing it did, and an error when nothing +// could be judged. +func (d *Driver) Wake(why string) ([]string, error) { + nodes, err := d.Read() + if err != nil { + return nil, err + } + now := d.now() + var said, errs []string + for _, c := range []struct { + name, event, seat string + holding func([]NodeRead, []DroppedNode) []string + }{ + {OldNameWindow, OldNameClosedEvent, OldSeat, HoldingOldName}, + {DirectWritesWindow, DirectWritesClosedEvent, "", HoldingDirectWrites}, + } { + cur, err := d.State.Window(c.name) + if err != nil { + errs = append(errs, err.Error()) + continue + } + if cur.Name == "" { + cur = Window{Name: c.name, Open: true} + } + next, closed, overdue := cur.Step(c.holding(nodes, cur.Dropped), now) + if !windowsEqual(cur, next) { + if err := d.State.PutWindow(next); err != nil { + errs = append(errs, fmt.Sprintf("the %s window is not kept: %v", c.name, err)) + continue + } + } + for _, n := range cur.Holding { + if !slices.Contains(overdue, n) && d.Finding != nil { + d.Finding(FindingWindowHeld, c.name+"."+n, "", false) + } + } + for _, n := range overdue { + if d.Finding != nil { + d.Finding(FindingWindowHeld, c.name+"."+n, fmt.Sprintf("%s has held distribution's %s window open since %s", n, c.name, next.Since[n].Format(time.RFC3339)), true) + } + } + if closed != nil { + closed.Seat = c.seat + if d.Closed != nil { + if err := d.Closed(*closed); err != nil { + errs = append(errs, fmt.Sprintf("%s was not said: %v", c.event, err)) + } + } + said = append(said, fmt.Sprintf("the %s window is closed (%s); it was held last by %v", c.name, why, closed.LastHolding)) + } else if next.Open { + said = append(said, fmt.Sprintf("the %s window stays open (%s): held by %v", c.name, why, next.Holding)) + } + } + if len(errs) > 0 { + return said, errors.New(strings.Join(errs, "; ")) + } + return said, nil +} + +func (d *Driver) now() time.Time { + if d.Now != nil { + return d.Now() + } + return time.Now().UTC() +} + +func windowsEqual(a, b Window) bool { + return a.Open == b.Open && slices.Equal(a.Holding, b.Holding) && a.Closed.Equal(b.Closed) && + len(a.Since) == len(b.Since) && len(a.Dropped) == len(b.Dropped) +} diff --git a/modules/distribution/internal/holder/front.go b/modules/distribution/internal/holder/front.go new file mode 100644 index 00000000..eb42ae13 --- /dev/null +++ b/modules/distribution/internal/holder/front.go @@ -0,0 +1,101 @@ +package holder + +import ( + "crypto/subtle" + "fmt" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "sync/atomic" +) + +// Front is the store's port, served by distribution's program in front of the registry container, which listens on its +// node's loopback only (design 51 *No build module holds a registry's write credential*): reads (the OCI distribution +// protocol's GET and HEAD) stay open to every node's container runtime; a write reaches the store only from +// distribution's own process, which writes to the loopback directly and never through the front. While the direct-write +// window is open, a build module's old build still pushes through the front as today, but never to a tag the seat holds +// published. +type Front struct { + // Store is the registry container's loopback address. + Store *url.URL + // Window says whether the direct-write window is open. + Window *atomic.Bool + // Published says whether name holds a tag published through the seat. + Published func(name, tag string) (bool, error) + // Bootstrap is the bootstrap's one-off credential for the holders' first images (design 51 *On a fresh mesh*): + // read on each write, never kept, so the file's removal (BootstrapFile) refuses the next write that carries it. + Bootstrap func() (token string, ok bool) + Logf func(string, ...any) + proxy *httputil.ReverseProxy +} + +// ServeHTTP answers one request of the store's port. +func (f *Front) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if f.proxy == nil { + f.proxy = httputil.NewSingleHostReverseProxy(f.Store) + } + switch r.Method { + case http.MethodGet, http.MethodHead: + f.proxy.ServeHTTP(w, r) + return + } + bootstrap := false + if f.Bootstrap != nil { + if token, ok := f.Bootstrap(); ok && token != "" { + got, has := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ") + bootstrap = has && subtle.ConstantTimeCompare([]byte(got), []byte(token)) == 1 + } + } + if !bootstrap && (f.Window == nil || !f.Window.Load()) { + refuse(w, http.StatusForbidden, "DENIED", "the store takes writes from distribution's own process alone: publish through the seat's publish") + return + } + if name, tag, ok := manifestPut(r); ok && f.Published != nil { + held, err := f.Published(name, tag) + if err != nil { + refuse(w, http.StatusServiceUnavailable, "UNAVAILABLE", "whether "+name+":"+tag+" is published cannot be read, so nothing is written: "+err.Error()) + return + } + if held { + refuse(w, http.StatusForbidden, "DENIED", fmt.Sprintf("%s:%s is published through the seat; a direct write cannot overwrite it: publish through the seat's publish", name, tag)) + return + } + } + if bootstrap { + if f.Logf != nil { + f.Logf("a write %s %s from %s with the bootstrap's one-off credential", r.Method, r.URL.Path, r.RemoteAddr) + } + } else if f.Logf != nil { + f.Logf("a direct write %s %s from %s, accepted while the direct-write window is open", r.Method, r.URL.Path, r.RemoteAddr) + } + f.proxy.ServeHTTP(w, r) +} + +// BootstrapFile is where the node-engine's bootstrap gives distribution its one-off credential, in distribution's +// state directory (0600, distribution's account): one line, the token. The tools bundle removes it when it answers its +// first publish, so the holders serve publish and the credential is gone (RemoveBootstrap). +const BootstrapFile = "/var/lib/mesh/registry/bootstrap-push" + +// manifestPut is a PUT of /v2//manifests/, a tag and not a digest. +func manifestPut(r *http.Request) (name, tag string, ok bool) { + if r.Method != http.MethodPut { + return "", "", false + } + p := strings.TrimPrefix(r.URL.Path, "/v2/") + i := strings.LastIndex(p, "/manifests/") + if i <= 0 { + return "", "", false + } + name, tag = p[:i], p[i+len("/manifests/"):] + if strings.HasPrefix(tag, "sha256:") || tag == "" { + return "", "", false + } + return name, tag, true +} + +func refuse(w http.ResponseWriter, code int, kind, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + fmt.Fprintf(w, `{"errors":[{"code":%q,"message":%q}]}`, kind, msg) +} diff --git a/modules/distribution/internal/holder/health.go b/modules/distribution/internal/holder/health.go new file mode 100644 index 00000000..1473e563 --- /dev/null +++ b/modules/distribution/internal/holder/health.go @@ -0,0 +1,34 @@ +package holder + +import "fmt" + +// FindingReadSide is the finding of a store whose read side does not answer through the program. +const FindingReadSide = "read-side-down" + +// ProbeName is the name the health check lists on each seat: any name answers, an empty list included. +const ProbeName = Self + "/program" + +// Health is distribution's health check (design 51 *While the program is down*; ADR 0240), run by the runtime each +// minute outside the program's process: it asks `versions` of each seat it holds through the store's read side, the +// program's port (front), and answers what failed, naming the program. The finding is written while it fails and +// cleared once it answers. +func Health(front Catalog, finding func(kind, subject, words string, raised bool)) (map[string]any, error) { + answered := map[string]int{} + for _, seat := range Seats { + got, err := front.Versions(ProbeName) + if err != nil { + words := fmt.Sprintf("the store's read side does not answer %s's versions through distribution's program (port 5000): %v; is the program distribution running?", seat, err) + if finding != nil { + finding(FindingReadSide, seat, words, true) + } + return nil, fmt.Errorf("%s", words) + } + answered[seat] = len(got) + } + if finding != nil { + for _, seat := range Seats { + finding(FindingReadSide, seat, "", false) + } + } + return map[string]any{"healthy": true, "versions": answered}, nil +} diff --git a/modules/distribution/internal/holder/holder_test.go b/modules/distribution/internal/holder/holder_test.go new file mode 100644 index 00000000..5e15abfa --- /dev/null +++ b/modules/distribution/internal/holder/holder_test.go @@ -0,0 +1,733 @@ +package holder + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "os" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +type memObjects map[string][]byte + +func (m memObjects) Put(n string, d []byte) error { m[n] = append([]byte(nil), d...); return nil } +func (m memObjects) Get(n string) ([]byte, error) { + if d, ok := m[n]; ok { + return d, nil + } + return nil, errors.New("no object " + n) +} +func (m memObjects) Delete(n string) error { delete(m, n); return nil } +func (m memObjects) Used() (uint64, error) { + var u uint64 + for _, d := range m { + u += uint64(len(d)) + } + return u, nil +} + +type memTransfers map[string]Transfer + +func (m memTransfers) Get(id string) (Transfer, bool, error) { t, ok := m[id]; return t, ok, nil } +func (m memTransfers) Put(t Transfer) error { m[t.ID] = t; return nil } +func (m memTransfers) Delete(id string) error { delete(m, id); return nil } +func (m memTransfers) List() ([]Transfer, error) { + var out []Transfer + for _, t := range m { + out = append(out, t) + } + return out, nil +} + +type fakeBuilder struct { + ok bool + err error +} + +func (b fakeBuilder) Recorded(string, string, string) (bool, string, error) { + return b.ok, "not numbered", b.err +} + +type fakeStore struct { + has map[string]bool + put []map[string][]byte +} + +func (s *fakeStore) Has(_, d string) (bool, error) { return s.has[d], nil } +func (s *fakeStore) Put(t PublishRequest, parts map[string][]byte) (string, error) { + s.put = append(s.put, parts) + return t.Name + "@" + t.Digest, nil +} +func (s *fakeStore) Held(string, string, string) (string, bool, error) { return "", false, nil } + +type said struct { + seat, event string + body Event +} + +type bed struct { + h *Holder + v *Verbs + store *fakeStore + said []said + findings map[string]bool + now time.Time + cat *fakeCatalog +} + +type fakeCatalog struct { + tags map[string]string // name:version -> digest + removed []string + mirrorErr error + indexes map[string][]string // name@digest -> the platform manifests it names +} + +func (c *fakeCatalog) Versions(name string) ([]string, error) { + var out []string + for k := range c.tags { + if n, v, _ := strings.Cut(k, ":"); n == name { + out = append(out, v) + } + } + return out, nil +} +func (c *fakeCatalog) Digest(name, version string) (string, error) { + return c.tags[name+":"+version], nil +} +func (c *fakeCatalog) Remove(name, version string) error { + c.removed = append(c.removed, name+":"+version) + delete(c.tags, name+":"+version) + return nil +} +func (c *fakeCatalog) RemoveDigest(name, digest string) error { + c.removed = append(c.removed, name+"@"+digest) + return nil +} +func (c *fakeCatalog) Platforms(name, digest string) ([]string, error) { + return c.indexes[name+"@"+digest], nil +} +func (c *fakeCatalog) Mirror(source, digest string) (string, error) { + if c.mirrorErr != nil { + return "", c.mirrorErr + } + src, _ := ParseSource(source) + return c.Reference(src.Name(), digest), nil +} +func (c *fakeCatalog) Reference(name, digest string) string { + return name + "@" + digest +} + +type memMirrors map[string]MirrorRequest + +func (m memMirrors) Ask(k string, r MirrorRequest) error { + if _, ok := m[k]; !ok { + m[k] = r + } + return nil +} +func (m memMirrors) Claim(k string, r MirrorRequest, rev uint64) (bool, error) { + if cur, ok := m[k]; !ok || mirrorRev(cur) != rev { + return false, nil + } + m[k] = r + return true, nil +} +func (m memMirrors) Delete(k string) error { delete(m, k); return nil } + +// mirrorRev is a kept ask's revision in memMirrors: 1 unclaimed, 2 once claimed. +func mirrorRev(r MirrorRequest) uint64 { + if r.Claim == "" { + return 1 + } + return 2 +} + +type memHolds map[string]Hold + +func (m memHolds) Get(k string) (Hold, bool, error) { h, ok := m[k]; return h, ok, nil } +func (m memHolds) Put(h Hold) error { m[h.Key()] = h; return nil } +func (m memHolds) Delete(k string) error { delete(m, k); return nil } +func (m memHolds) List() ([]Hold, error) { + var out []Hold + for _, h := range m { + out = append(out, h) + } + return out, nil +} + +func newBed(builderOK bool) *bed { + b := &bed{store: &fakeStore{has: map[string]bool{}}, findings: map[string]bool{}, now: time.Unix(1000, 0), cat: &fakeCatalog{tags: map[string]string{}}} + b.h = &Holder{Callers: DefaultCallers(), Objects: memObjects{}, Transfers: memTransfers{}, Builder: fakeBuilder{ok: builderOK}, + Store: b.store, Now: func() time.Time { return b.now }, + Say: func(seat, event string, body Event) error { + b.said = append(b.said, said{seat, event, body}) + return nil + }, + Finding: func(kind, subject, _ string, raised bool) { b.findings[kind+" "+subject] = raised }} + b.v = &Verbs{Holder: b.h, Catalog: b.cat, Holds: memHolds{}, Mirrors: memMirrors{}} + return b +} + +func keys[V any](m map[string]V) []string { + var out []string + for k := range m { + out = append(out, k) + } + return out +} + +func digestOf(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) } + +const builder = "ace.mesh-build-go" + +// TestTheHolderOfTheTwoImageSeats holds design 51's holder cases at the unit level (CAT-D1, D2, D3, D3b, D4, D4b, D5): +// one table or case per rule, the caller list and the store front's. +func TestTheHolderOfTheTwoImageSeats(t *testing.T) { + blob := []byte(strings.Repeat("a", 3000)) + part := Part{Digest: digestOf(blob), Size: int64(len(blob))} + req := func(id string) PublishRequest { + return PublishRequest{Transfer: id, Name: "m/program", Version: "0.1.4", Kind: KindBundle, Digest: part.Digest, Parts: []Part{part}} + } + + t.Run("a transfer in parts, cut and resumed, hashed, numbered by the builder, published", func(t *testing.T) { + b := newBed(true) + a, err := b.h.Publish(ArtifactSeat, builder, req("t1")) + if err != nil || !reflect.DeepEqual(a.Missing, []string{part.Digest}) { + t.Fatalf("publish = %+v, %v", a, err) + } + if _, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t1", Digest: part.Digest, Sequence: 0, Data: blob[:1000]}); err != nil { + t.Fatal(err) + } + // Cut: the build module opens again and is told what is still missing; its next chunk goes on from there. + if a, _ := b.h.Publish(ArtifactSeat, builder, req("t1")); !reflect.DeepEqual(a.Missing, []string{part.Digest}) { + t.Errorf("resumed publish = %+v", a) + } + if _, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t1", Digest: part.Digest, Sequence: 0, Data: blob[:1000]}); err == nil { + t.Error("a chunk sent twice was taken: the transfer goes on from the chunks held") + } + a, err = b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t1", Digest: part.Digest, Sequence: 1, Last: true, Data: blob[1000:]}) + if err != nil || !a.Ended || len(b.said) != 1 || b.said[0].event != EventPublished || b.said[0].body.Reference == "" { + t.Fatalf("last chunk = %+v, %v; said %+v", a, err, b.said) + } + // The reference's registry part is the seat's name, as a node-engine reads it back (ADR 0605 §9); nothing set on + // the process names a reach. + if seat, path, ok := catalogue.InRegistrySeat(b.said[0].body.Reference); !ok || seat != ArtifactSeat || path != "m/program@"+part.Digest { + t.Errorf("the published reference %q is not the seat's form", b.said[0].body.Reference) + } + if len(b.h.Objects.(memObjects)) != 0 || len(b.h.Transfers.(memTransfers)) != 0 { + t.Error("the transfer's chunks or key stayed after its end") + } + }) + t.Run("refusals", func(t *testing.T) { + for _, c := range []struct { + name string + do func(b *bed) error + want string + }{ + {"a caller not on the list", func(b *bed) error { _, err := b.h.Publish(ArtifactSeat, "ace.mesh-provisioner", req("t")); return err }, "not on the holder's caller list"}, + {"an unproven caller", func(b *bed) error { _, err := b.h.Publish(ArtifactSeat, Unproven, req("t")); return err }, "no caller the tool runner proved"}, + {"a part from another caller than the opener", func(b *bed) error { + _, _ = b.h.Publish(ArtifactSeat, builder, req("t")) + _, err := b.h.PublishPart(ArtifactSeat, "ace.mesh-build-node", PartRequest{Transfer: "t", Digest: part.Digest, Last: true, Data: blob}) + return err + }, "opened by another caller"}, + {"a part whose bytes do not match its digest", func(b *bed) error { + _, _ = b.h.Publish(ArtifactSeat, builder, req("t")) + _, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t", Digest: part.Digest, Last: true, Data: []byte("other")}) + return err + }, "hash to"}, + {"a chunk over 1 MiB", func(b *bed) error { + _, _ = b.h.Publish(ArtifactSeat, builder, req("t")) + _, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t", Digest: part.Digest, Data: make([]byte, MaxPart+1)}) + return err + }, "over the"}, + {"a new transfer at the cap", func(b *bed) error { b.h.Cap = 100; _, err := b.h.Publish(ArtifactSeat, builder, req("t")); return err }, "at their cap"}, + {"a version the builder did not number", func(b *bed) error { + b.h.Builder = fakeBuilder{ok: false} + b.store.has[part.Digest] = true // held already: the publish ends at once + _, err := b.h.Publish(ArtifactSeat, builder, req("t")) + return err + }, "the builder did not number it"}, + {"an image on the artifact seat", func(b *bed) error { + r := req("t") + r.Kind = KindImage + _, err := b.h.Publish(ArtifactSeat, builder, r) + return err + }, "holds bundles and archives"}, + {"remove of a version with no tag", func(b *bed) error { + return b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/program", Version: "0.1.4", Reason: "branch ended"}) + }, "has no tag"}, + {"a retention removal from another caller than the sweep", func(b *bed) error { + return b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/program", Digest: part.Digest, Reason: ReasonRetention}) + }, "artifact sweep's alone"}, + {"a retention removal of a held archive", func(b *bed) error { + _ = b.v.HoldOne(ArtifactSeat, "novox.artifact-store-tools", HoldRequest{Name: "m/a", Kind: KindArchive, Digest: part.Digest}) + return b.v.Remove(ArtifactSeat, "novox.artifact-store-tools", RemoveRequest{Name: "m/a", Digest: part.Digest, Reason: ReasonRetention}) + }, "is held"}, + {"a hold of an image", func(b *bed) error { + return b.v.HoldOne(ImageSeat, "novox.artifact-store-tools", HoldRequest{Name: "m/i", Kind: KindImage, Digest: part.Digest}) + }, "never held"}, + {"a hold from another caller", func(b *bed) error { + return b.v.HoldOne(ArtifactSeat, "novox.mesh-build-go", HoldRequest{Name: "m/a", Kind: KindArchive, Digest: part.Digest}) + }, "not on the holder's caller list"}, + {"a release from a caller not on the list", func(b *bed) error { + _, err := b.v.Release(ArtifactSeat, "novox.mesh-build-go", HoldRequest{Name: "m/a", Digest: part.Digest}) + return err + }, "not on the holder's caller list"}, + {"a release in distribution's own name", func(b *bed) error { + _, err := b.v.Release(ArtifactSeat, "novox.distribution", HoldRequest{Name: "m/a", Digest: part.Digest}) + return err + }, "in its own name"}, + {"a mirror by no digest", func(b *bed) error { + _, err := b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "docker.io/library/nats:2"}) + return err + }, "pinned digest"}, + } { + b := newBed(true) + if err := c.do(b); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: err = %v, want one saying %q", c.name, err, c.want) + } + } + if _, err := ParseCallers(`{"release":["distribution"]}`); err == nil { + t.Error("a caller list putting distribution on release's list is taken") + } + }) + t.Run("only the missing layers are sent, and an idle transfer is dropped", func(t *testing.T) { + b := newBed(true) + other := []byte("layer two") + r := req("t2") + r.Kind, r.Parts = KindImage, []Part{part, {Digest: digestOf(other), Size: int64(len(other))}} + b.store.has[part.Digest] = true + a, err := b.h.Publish(ImageSeat, builder, r) + if err != nil || !reflect.DeepEqual(a.Missing, []string{digestOf(other)}) { + t.Fatalf("publish = %+v, %v; want only the layer the store lacks", a, err) + } + b.now = b.now.Add(TransferIdle) + if dropped, _ := b.h.DropIdle(); !reflect.DeepEqual(dropped, []string{"t2"}) { + t.Errorf("DropIdle = %v, want t2 after 24 hours with no part", dropped) + } + }) + t.Run("incoming's finding at 80 %, cleared below", func(t *testing.T) { + b := newBed(true) + b.h.Cap = 3500 + _, _ = b.h.Publish(ArtifactSeat, builder, req("t3")) + _, _ = b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t3", Digest: part.Digest, Data: blob[:2800]}) + if !b.findings["incoming-full incoming"] { + t.Error("2800 of 3500 bytes wrote no finding") + } + _, _ = b.h.Drop("t3") + if b.findings["incoming-full incoming"] { + t.Error("the finding stayed after the transfer was dropped") + } + // A refusal for want of room stays loud on incoming until the holder takes a publish again. + b.h.Cap = 100 + _, _ = b.h.Publish(ArtifactSeat, builder, req("t4")) + if !b.findings["publish-refused incoming"] { + t.Errorf("a refusal at the cap raised %v, want publish-refused on incoming", b.findings) + } + b.h.Cap = 1 << 20 + b.store.has[part.Digest] = true + if _, err := b.h.Publish(ArtifactSeat, builder, req("t5")); err != nil || b.findings["publish-refused incoming"] { + t.Errorf("the next publish taken = %v, findings %v: want the refusal at the cap cleared", err, b.findings) + } + }) + t.Run("the sweep's removal, hold and release", func(t *testing.T) { + b := newBed(true) + sweep := "novox.artifact-store-tools" + if err := b.v.Remove(ArtifactSeat, sweep, RemoveRequest{Name: "m/a", Digest: part.Digest, Kind: KindArchive, Reason: ReasonRetention}); err != nil { + t.Fatal(err) + } + if err := b.v.HoldOne(ArtifactSeat, sweep, HoldRequest{Name: "m/b", Kind: KindArchive, Digest: part.Digest}); err != nil { + t.Fatal(err) + } + if released, err := b.v.Release(ArtifactSeat, "person.jochen", HoldRequest{Name: "m/b", Digest: part.Digest, Why: "by hand"}); err != nil || !released { + t.Errorf("the operator's session releasing through registry_holds: %v, %v", released, err) + } + // A release of what is not held answers released false, never an error, and says nothing (the seat's definition). + if released, err := b.v.Release(ArtifactSeat, sweep, HoldRequest{Name: "m/b", Digest: part.Digest, Why: "eligible"}); err != nil || released { + t.Errorf("a release of what is not held: %v, %v; want false and no error", released, err) + } + var events []string + for _, s := range b.said { + events = append(events, s.event+" "+s.body.By) + } + if want := []string{"removed " + sweep, "held " + sweep, "released person.jochen"}; !reflect.DeepEqual(events, want) { + t.Errorf("said %v, want %v", events, want) + } + b.cat.tags["m/p:0.1.5-feat-x.1"] = part.Digest + if err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/p", Version: "0.1.5-feat-x.1", Reason: "branch merged"}); err != nil || b.cat.tags["m/p:0.1.5-feat-x.1"] != "" { + t.Errorf("the builder's removal of a tagged version: %v", err) + } + }) + // CAT-AS1: what the sweep did over the store's door is the holder's: a removal for retention of an index removes + // its platform manifests first, but those a tagged or held index of the repository names; a manifest such an index + // names is refused; OCI.Platforms reads an index's from the store. + t.Run("an index's platforms go with it, but those a kept index names", func(t *testing.T) { + d := func(s string) string { return "sha256:" + strings.Repeat(s, 64) } + sweep := "novox.artifact-store-tools" + name := "upstream/docker.io/library/alpine" + for _, c := range []struct { + case_, remove string + tagged, held string // the kept index, by a tag or a hold; none when empty + removed []string + refused string + }{ + {case_: "no kept index", remove: d("1"), removed: []string{d("a"), d("b"), d("1")}}, + {case_: "a tagged index shares a platform", remove: d("1"), tagged: d("2"), removed: []string{d("a"), d("1")}}, + {case_: "a held index shares a platform", remove: d("1"), held: d("2"), removed: []string{d("a"), d("1")}}, + {case_: "a platform of a held index", remove: d("b"), held: d("2"), refused: "is kept by the index " + d("2")}, + {case_: "an image that is no index", remove: d("c"), removed: []string{d("c")}}, + } { + b := newBed(true) + b.cat.indexes = map[string][]string{name + "@" + d("1"): {d("a"), d("b")}, name + "@" + d("2"): {d("b")}} + if c.tagged != "" { + b.cat.tags[name+":3.20"] = c.tagged + } + if c.held != "" { + if err := b.v.HoldOne(ImageSeat, sweep, HoldRequest{Name: name, Digest: c.held}); err != nil { + t.Fatal(err) + } + } + err := b.v.Remove(ImageSeat, sweep, RemoveRequest{Name: name, Digest: c.remove, Reason: ReasonRetention}) + var removed []string + for _, r := range b.cat.removed { + removed = append(removed, strings.TrimPrefix(r, name+"@")) + } + if c.refused != "" { + if err == nil || !strings.Contains(err.Error(), c.refused) || len(removed) != 0 { + t.Errorf("%s: %v, removed %v; want refused saying %q and nothing removed", c.case_, err, removed, c.refused) + } + continue + } + if err != nil || !reflect.DeepEqual(removed, c.removed) { + t.Errorf("%s: %v, removed %v, want %v", c.case_, err, removed, c.removed) + } + } + store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/" + name + "/manifests/" + d("1"): + _, _ = w.Write([]byte(`{"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"digest":"` + d("a") + `"},{"digest":"` + d("b") + `"}]}`)) + case "/v2/" + name + "/manifests/" + d("c"): + _, _ = w.Write([]byte(`{"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer store.Close() + u, _ := url.Parse(store.URL) + o := &OCI{Base: u} + for digest, want := range map[string][]string{d("1"): {d("a"), d("b")}, d("c"): nil, d("9"): nil} { + if got, err := o.Platforms(name, digest); err != nil || !reflect.DeepEqual(got, want) { + t.Errorf("Platforms(%s) = %v, %v, want %v", digest[:10], got, err, want) + } + } + }) + t.Run("the store's port: reads open, writes from distribution alone, and the direct-write window", func(t *testing.T) { + store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) })) + defer store.Close() + u, _ := url.Parse(store.URL) + var open atomic.Bool + token, kept := "s3cret", true + f := &Front{Store: u, Window: &open, Published: func(name, tag string) (bool, error) { return name == "m/i" && tag == "0.1.3", nil }, + Bootstrap: func() (string, bool) { return token, kept }} + auth := "" + do := func(method, path string) int { + rec := httptest.NewRecorder() + req := httptest.NewRequest(method, path, nil) + if auth != "" { + req.Header.Set("Authorization", "Bearer "+auth) + } + f.ServeHTTP(rec, req) + return rec.Code + } + for _, c := range []struct { + open bool + method, path string + want int + }{ + {false, http.MethodGet, "/v2/m/i/manifests/0.1.3", http.StatusCreated}, + {false, http.MethodHead, "/v2/m/i/blobs/sha256:aa", http.StatusCreated}, + {false, http.MethodPut, "/v2/m/i/manifests/0.1.9", http.StatusForbidden}, + {false, http.MethodPost, "/v2/m/i/blobs/uploads/", http.StatusForbidden}, + {true, http.MethodPost, "/v2/m/i/blobs/uploads/", http.StatusCreated}, + {true, http.MethodPut, "/v2/m/i/manifests/0.1.9", http.StatusCreated}, + {true, http.MethodPut, "/v2/m/i/manifests/0.1.3", http.StatusForbidden}, // published through the seat + } { + open.Store(c.open) + if got := do(c.method, c.path); got != c.want { + t.Errorf("window open %v: %s %s = %d, want %d", c.open, c.method, c.path, got, c.want) + } + } + // The bootstrap's one-off credential writes with the window closed, never over a published tag, and not once + // its file is gone; a wrong token writes nothing. + open.Store(false) + for _, c := range []struct { + auth string + kept bool + path string + want int + }{ + {"s3cret", true, "/v2/gitea/program/manifests/0.1.0", http.StatusCreated}, + {"s3cret", true, "/v2/m/i/manifests/0.1.3", http.StatusForbidden}, + {"wrong", true, "/v2/gitea/program/manifests/0.1.0", http.StatusForbidden}, + {"s3cret", false, "/v2/gitea/program/manifests/0.1.0", http.StatusForbidden}, + } { + auth, kept = c.auth, c.kept + if got := do(http.MethodPut, c.path); got != c.want { + t.Errorf("bootstrap %q kept %v: PUT %s = %d, want %d", c.auth, c.kept, c.path, got, c.want) + } + } + }) + t.Run("mirror copies an outside image by its pinned digest, every platform, only the parts the store lacks", func(t *testing.T) { + up, store := newFakeRegistry(), newFakeRegistry() + ups, sts := httptest.NewServer(up), httptest.NewServer(store) + defer ups.Close() + defer sts.Close() + up.token, up.realm = "anon", ups.URL+"/token" + host := strings.TrimPrefix(ups.URL, "http://") + repo := "library/nats" + layer, config := []byte(strings.Repeat("l", 5000)), []byte(`{"os":"linux"}`) + up.blobs[repo+"@"+digestOf(layer)], up.blobs[repo+"@"+digestOf(config)] = layer, config + img, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": mediaManifest, + "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": digestOf(config), "size": len(config)}, + "layers": []map[string]any{{"mediaType": mediaLayer, "digest": digestOf(layer), "size": len(layer)}}}) + up.manifests[repo+":"+digestOf(img)], up.types[repo+":"+digestOf(img)] = img, mediaManifest + idx, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": []map[string]any{{"mediaType": mediaManifest, "digest": digestOf(img), "size": len(img)}}}) + up.manifests[repo+":"+digestOf(idx)] = idx + base, _ := url.Parse(sts.URL) + o := &OCI{Base: base, Upstream: &Upstream{Scheme: "http"}} + name := "upstream/" + host + "/" + repo + store.blobs[name+"@"+digestOf(config)] = config // held already: not fetched again + ref, err := o.Mirror(host+"/"+repo+":2", digestOf(idx)) + if err != nil { + t.Fatal(err) + } + if want := name + "@" + digestOf(idx); ref != want { + t.Errorf("reference %s, want %s", ref, want) + } + if store.manifests[name+":"+digestOf(idx)] == nil || store.manifests[name+":"+digestOf(img)] == nil || store.blobs[name+"@"+digestOf(layer)] == nil { + t.Errorf("the store holds %v and %v, want the index, its image and the layer", keys(store.manifests), keys(store.blobs)) + } + if !reflect.DeepEqual(up.gets, []string{digestOf(layer)}) { + t.Errorf("blobs fetched %v, want only the layer the store lacked", up.gets) + } + up.manifests[repo+":"+digestOf(idx)] = []byte(`{"tampered":true}`) + if _, err := o.Mirror(host+"/"+repo, digestOf(idx)); err != nil { + t.Errorf("a copy held whole is answered at once, without asking the upstream: %v", err) + } + delete(store.manifests, name+":"+digestOf(idx)) + if _, err := o.Mirror(host+"/"+repo, digestOf(idx)); err == nil || !strings.Contains(err.Error(), "does not hash") { + t.Errorf("a manifest whose bytes are not its digest: %v", err) + } + for in, want := range map[string]string{"nats:2": "upstream/docker.io/library/nats", "ghcr.io/o/p@sha256:ab": "upstream/ghcr.io/o/p", "localhost:5000/x": "upstream/localhost:5000/x"} { + if s, _ := ParseSource(in); s.Name() != want { + t.Errorf("ParseSource(%q) = %s, want %s", in, s.Name(), want) + } + } + }) + t.Run("a late publish names a held bundle with no size: tagged at the store's size, nothing sent", func(t *testing.T) { + store := newFakeRegistry() + sts := httptest.NewServer(store) + defer sts.Close() + base, _ := url.Parse(sts.URL) + o := &OCI{Base: base} + blob := []byte(strings.Repeat("b", 4321)) + store.blobs["m/tools@"+digestOf(blob)] = blob + ref, err := o.Put(PublishRequest{Name: "m/tools", Version: "0.1.4", Kind: KindBundle, Digest: digestOf(blob), + Parts: []Part{{Digest: digestOf(blob)}}}, map[string][]byte{}) + if err != nil || ref != "m/tools@"+digestOf(blob) { + t.Fatalf("Put = %q, %v", ref, err) + } + var man struct { + Layers []struct { + Size int64 `json:"size"` + } `json:"layers"` + } + if err := json.Unmarshal(store.manifests["m/tools:0.1.4"], &man); err != nil || len(man.Layers) != 1 || man.Layers[0].Size != int64(len(blob)) { + t.Errorf("the version's manifest %s, want one layer of %d bytes", store.manifests["m/tools:0.1.4"], len(blob)) + } + if _, err := o.Put(PublishRequest{Name: "m/tools", Version: "0.1.5", Kind: KindBundle, Digest: "sha256:" + strings.Repeat("e", 64), + Parts: []Part{{Digest: "sha256:" + strings.Repeat("e", 64)}}}, map[string][]byte{}); err == nil { + t.Error("a digest the store does not hold was tagged") + } + }) + t.Run("mirror answers at once and says the copy's end on the seat's events", func(t *testing.T) { + b := newBed(true) + d := "sha256:" + strings.Repeat("c", 64) + ref, err := b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "docker.io/library/nats:2", Digest: d}) + want := "container-image-registry/upstream/docker.io/library/nats@" + d + key := MirrorsKey("upstream/docker.io/library/nats", d) + asked := b.v.Mirrors.(memMirrors) + if err != nil || ref != want || len(b.said) != 0 || asked[key].By != "novox.mesh-build-image" { + t.Fatalf("mirror = %q, %v, said %v, asks %v: want the reference at once, the copy asked, nothing said yet", ref, err, b.said, asked) + } + if strings.ContainsAny(key, ":@") { + t.Errorf("the ask's key %q holds characters a key may not", key) + } + if err := b.v.Mirrored(asked[key]); err != nil || len(b.said) != 1 || b.said[0].event != EventPublished || + b.said[0].body.Reference != want || b.said[0].body.Kind != KindUpstream || len(asked) != 0 { + t.Fatalf("the copy's end = %v, said %+v, asks %v: want published with the reference and the ask gone", err, b.said, asked) + } + // One copy, said once: two followers on the same ask; the second sees it claimed and passes, and the outcome + // carries the ask's id as its event id. + _, _ = b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "nats", Digest: d}) + ask := asked[key] + copies := len(b.said) + for _, follower := range []string{"one", "two"} { + if err := b.v.Take(key, ask, 1, false, follower); err != nil { + t.Fatalf("Take by %s: %v", follower, err) + } + } + if len(b.said) != copies+1 || ask.ID == "" || b.said[copies].body.ID != ask.ID || len(asked) != 0 { + t.Fatalf("two followers on one ask said %+v (ask id %q), asks %v: want one copy, said once under the ask's id", b.said[copies:], ask.ID, asked) + } + b.said = b.said[:copies] + b.cat.mirrorErr = errors.New("the upstream said 404") + _, _ = b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "nats", Digest: d}) + if err := b.v.Mirrored(asked[key]); err != nil || b.said[1].event != EventRefused || !strings.Contains(b.said[1].body.Reason, "404") || len(asked) != 0 { + t.Errorf("a failed copy = %v, said %+v: want refused with why", err, b.said) + } + }) + t.Run("the windows", func(t *testing.T) { + nodes := []NodeRead{ + {Node: "ace", Serves: true, Stated: []string{NewNamesCapability}, NamesArtifact: true, BuildModules: map[string]bool{"mesh-build-go": true}}, + {Node: "g14", NamesArtifact: true, BuildModules: map[string]bool{"mesh-build-go": false}}, // away, old node-engine + {Node: "novox", Serves: true, Stated: nil, NamesArtifact: false}, + } + if got := HoldingOldName(nodes, nil); !reflect.DeepEqual(got, []string{"g14"}) { + t.Errorf("old name held by %v, want g14 (no manifest-reads: it states nothing)", got) + } + if got := HoldingDirectWrites(nodes, nil); !reflect.DeepEqual(got, []string{"g14"}) { + t.Errorf("direct writes held by %v, want g14", got) + } + t0 := time.Unix(0, 0) + w, closed, _ := Window{Name: OldNameWindow}.Step([]string{"g14"}, t0) + if closed != nil || !w.Open { + t.Fatalf("Step = %+v, %+v", w, closed) + } + if _, _, overdue := w.Step([]string{"g14"}, t0.Add(WindowFindingAfter)); !reflect.DeepEqual(overdue, []string{"g14"}) { + t.Errorf("seven days: overdue %v", overdue) + } + w, _ = w.Drop("g14", "person.jochen", "retired", t0) + w, closed, _ = w.Step(HoldingOldName(nodes, w.Dropped), t0.Add(time.Hour)) + if closed == nil || closed.Window != OldNameWindow || !reflect.DeepEqual(closed.LastHolding, []string{"g14"}) || len(closed.Dropped) != 1 { + t.Errorf("closing = %+v", closed) + } + if _, again, _ := w.Step([]string{"g14"}, t0.Add(2*time.Hour)); again != nil { + t.Error("a closed window closed again") + } + }) + t.Run("the windows' driver: every node read on each wake, closed once, a paused build agent's same build counted", func(t *testing.T) { + del := &fakeDelivery{haves: []NodeHave{ + {"ace", "mesh-build-go", "fp-new"}, {"novox", "mesh-build-go", "fp-new"}, {"g14", "mesh-build-go", "fp-old"}, + {"g14", "node", ""}, {"ace", "gitea", "x"}, {"novox", "gitea", "x"}, + }, stated: map[string][]string{"ace": {NewNamesCapability}, "novox": {NewNamesCapability}}} + st := &memWindows{w: map[string]Window{}} + st.pub.Learn("mesh-build-go", "fp-new") + var closed []string + found := map[string]bool{} + now := time.Unix(0, 0) + d := &Driver{Delivery: del, State: st, Callers: func() (Callers, error) { return DefaultCallers(), nil }, Now: func() time.Time { return now }, + Closed: func(c Closed) error { closed = append(closed, c.Window+" "+c.Seat); return nil }, + Finding: func(kind, subject, _ string, raised bool) { found[kind+" "+subject] = raised }} + if _, err := d.Wake("start"); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(st.w[OldNameWindow].Holding, []string{"g14"}) || !reflect.DeepEqual(st.w[DirectWritesWindow].Holding, []string{"g14"}) { + t.Errorf("windows = %+v, want g14 holding both (away: states nothing; runs the old build)", st.w) + } + now = now.Add(WindowFindingAfter) + _, _ = d.Wake("delivered") + if !found[FindingWindowHeld+" "+OldNameWindow+".g14"] { + t.Errorf("findings %v, want g14 named after seven days", found) + } + del.haves[2].FP, del.stated["g14"] = "fp-new", []string{NewNamesCapability} + if _, err := d.Wake("manifest-reads-changed on g14"); err != nil { + t.Fatal(err) + } + _, _ = d.Wake("delivered") + if want := []string{OldNameWindow + " " + OldSeat, DirectWritesWindow + " "}; !reflect.DeepEqual(closed, want) { + t.Errorf("closed %v, want each window said once, %v", closed, want) + } + if found[FindingWindowHeld+" "+OldNameWindow+".g14"] { + t.Error("the finding stays after the window closed") + } + }) + t.Run("the health check: versions of each seat through the program's port; down names the program, then clears", func(t *testing.T) { + front := &downCatalog{fakeCatalog: fakeCatalog{tags: map[string]string{}}, down: true} + found := map[string]bool{} + f := func(kind, subject, _ string, raised bool) { found[kind+" "+subject] = raised } + if _, err := Health(front, f); err == nil || !strings.Contains(err.Error(), "program distribution") || !found[FindingReadSide+" "+ImageSeat] { + t.Errorf("program stopped: %v, findings %v", err, found) + } + front.down = false + if _, err := Health(front, f); err != nil || found[FindingReadSide+" "+ImageSeat] { + t.Errorf("program back: %v, findings %v", err, found) + } + }) + t.Run("the manifest", func(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Provides []struct{ Name string } `json:"provides"` + Claims []struct { + Name string `json:"name"` + Serves []string `json:"serves"` + } `json:"claims"` + Resources []map[string]any `json:"resources"` + Consumes []string `json:"consumes"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + // E15: artifact-store is still offered until the old name closes, so a revert needs no edge. + if len(m.Provides) != 1 || m.Provides[0].Name != "artifact-store" { + t.Errorf("provides = %+v, want artifact-store kept", m.Provides) + } + claims := map[string][]string{} + for _, c := range m.Claims { + claims[c.Name] = c.Serves + } + if !reflect.DeepEqual(claims[ImageSeat], VerbsOf(ImageSeat)) || !reflect.DeepEqual(claims[ArtifactSeat], VerbsOf(ArtifactSeat)) { + t.Errorf("claims = %v, want each seat serving its verbs", claims) + } + if _, ok := claims[OldSeat]; !ok { + t.Error("the old seat mesh-artifact-store is no longer claimed while its window is open") + } + if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-view.delivered", "node-engine.manifest-reads-changed"}) { + t.Errorf("consumes = %v, want delivered and manifest-reads-changed (the windows' wakes)", m.Consumes) + } + // E14: the registry container, on the loopback, is listed before the program, so the port is free when it starts. + store, program := -1, -1 + for i, r := range m.Resources { + switch r["id"] { + case "store": + store = i + if ports, _ := r["ports"].([]any); len(ports) != 1 || ports[0] != "127.0.0.1:5001:5000" { + t.Errorf("the store's ports = %v, want the loopback only", r["ports"]) + } + case "program": + program = i + if h, _ := r["health"].(map[string]any); h["kind"] != "tool" || h["tool"] != "registry_health" { + t.Errorf("the program's health = %v, want the tool registry_health, run outside its process", r["health"]) + } + } + } + if store < 0 || program < 0 || store > program { + t.Errorf("store at %d, program at %d: want the store listed first", store, program) + } + }) +} diff --git a/modules/distribution/internal/holder/mirror.go b/modules/distribution/internal/holder/mirror.go new file mode 100644 index 00000000..1e35f6da --- /dev/null +++ b/modules/distribution/internal/holder/mirror.go @@ -0,0 +1,303 @@ +package holder + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" +) + +// Upstream reads an outside registry by the OCI distribution protocol, anonymously: on a 401 it takes the bearer +// token the registry's WWW-Authenticate names, scoped to a pull of the one repository, and asks again (the form +// Docker Hub, ghcr.io and quay.io answer). Scheme is "https" but in tests. +type Upstream struct { + Client *http.Client + Scheme string +} + +func (u *Upstream) client() *http.Client { + if u != nil && u.Client != nil { + return u.Client + } + return http.DefaultClient +} + +func (u *Upstream) scheme() string { + if u != nil && u.Scheme != "" { + return u.Scheme + } + return "https" +} + +// accepted are the manifest forms a mirror copies. +var accepted = []string{ + "application/vnd.oci.image.index.v1+json", mediaManifest, + "application/vnd.docker.distribution.manifest.list.v2+json", "application/vnd.docker.distribution.manifest.v2+json", +} + +// Source is an outside image's place: its registry host (as reached) and its repository path, from `/` +// with a tag or digest dropped. A name with no host is Docker Hub's, and a one-part path there is under library/. +type Source struct { + Host, Path string +} + +// ParseSource reads an outside image's name. +func ParseSource(s string) (Source, error) { + s = strings.TrimSpace(s) + if i := strings.Index(s, "@"); i >= 0 { + s = s[:i] + } + if i := strings.LastIndex(s, ":"); i > strings.LastIndex(s, "/") { + s = s[:i] + } + if s == "" || strings.Contains(s, "..") { + return Source{}, fmt.Errorf("mirror: %q is not an image's name", s) + } + host, path, ok := strings.Cut(s, "/") + if !ok || !(strings.ContainsAny(host, ".:") || host == "localhost") { + host, path = "docker.io", s + } + if host == "docker.io" && !strings.Contains(path, "/") { + path = "library/" + path + } + return Source{Host: host, Path: path}, nil +} + +// Name is the copy's name in the store, `upstream//` (design 51). +func (s Source) Name() string { return "upstream/" + s.Host + "/" + s.Path } + +func (s Source) reach() string { + if s.Host == "docker.io" { + return "registry-1.docker.io" + } + return s.Host +} + +// get asks the upstream for path, taking an anonymous token once when the upstream asks for one. +func (u *Upstream) get(src Source, path string, accept []string) (*http.Response, error) { + target := u.scheme() + "://" + src.reach() + "/v2/" + src.Path + path + token := "" + for try := 0; try < 2; try++ { + req, err := http.NewRequest(http.MethodGet, target, nil) + if err != nil { + return nil, err + } + for _, a := range accept { + req.Header.Add("Accept", a) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := u.client().Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusUnauthorized || token != "" { + return resp, nil + } + challenge := resp.Header.Get("WWW-Authenticate") + resp.Body.Close() + if token, err = u.token(challenge, src); err != nil { + return nil, err + } + } + return nil, fmt.Errorf("mirror: %s refused the anonymous token", src.reach()) +} + +// token takes an anonymous pull token from a Bearer challenge. +func (u *Upstream) token(challenge string, src Source) (string, error) { + rest, ok := strings.CutPrefix(challenge, "Bearer ") + if !ok { + return "", fmt.Errorf("mirror: %s asks a credential the mesh does not hold (%q); only anonymous reads are copied", src.reach(), challenge) + } + params := map[string]string{} + for _, kv := range strings.Split(rest, ",") { + k, v, _ := strings.Cut(strings.TrimSpace(kv), "=") + params[k] = strings.Trim(v, `"`) + } + realm, err := url.Parse(params["realm"]) + if err != nil || params["realm"] == "" { + return "", fmt.Errorf("mirror: %s names no token realm", src.reach()) + } + q := realm.Query() + if params["service"] != "" { + q.Set("service", params["service"]) + } + q.Set("scope", "repository:"+src.Path+":pull") + realm.RawQuery = q.Encode() + resp, err := u.client().Get(realm.String()) + if err != nil { + return "", err + } + defer resp.Body.Close() + var t struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + } + if resp.StatusCode != http.StatusOK || json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&t) != nil { + return "", fmt.Errorf("mirror: %s gave no anonymous token (%d)", realm.Host, resp.StatusCode) + } + if t.Token == "" { + t.Token = t.AccessToken + } + return t.Token, nil +} + +// manifest reads one manifest by its digest and checks its bytes against it. +func (u *Upstream) manifest(src Source, digest string) ([]byte, string, error) { + resp, err := u.get(src, "/manifests/"+digest, accepted) + if err != nil { + return nil, "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("mirror: %s/%s@%s answered %d", src.Host, src.Path, digest, resp.StatusCode) + } + b, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) + if err != nil { + return nil, "", err + } + if "sha256:"+sum(b) != digest { + return nil, "", fmt.Errorf("mirror: the manifest %s answered does not hash to its digest", digest) + } + mt := resp.Header.Get("Content-Type") + var probe struct { + MediaType string `json:"mediaType"` + } + if json.Unmarshal(b, &probe) == nil && probe.MediaType != "" { + mt = probe.MediaType + } + return b, mt, nil +} + +// copyBlob streams one blob from the upstream into the store, its digest checked as it passes; a blob held is +// not fetched. +func (o *OCI) copyBlob(up *Upstream, src Source, name string, desc descriptor) error { + if has, err := o.Has(name, desc.Digest); err != nil { + return err + } else if has { + return nil + } + resp, err := up.get(src, "/blobs/"+desc.Digest, nil) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("mirror: the blob %s answered %d", desc.Digest, resp.StatusCode) + } + h := sha256.New() + body := io.TeeReader(io.LimitReader(resp.Body, desc.Size+1), h) + if err := o.putBlobStream(name, desc.Digest, desc.Size, body); err != nil { + return err + } + if got := "sha256:" + hex.EncodeToString(h.Sum(nil)); got != desc.Digest { + return fmt.Errorf("mirror: the blob %s answered hashes to %s", desc.Digest, got) + } + return nil +} + +// putBlobStream stores a blob of a known size from a reader, monolithically. +func (o *OCI) putBlobStream(name, digest string, size int64, body io.Reader) error { + resp, b, err := o.do(http.MethodPost, "/v2/"+name+"/blobs/uploads/", "", nil) + if err != nil { + return err + } + if resp.StatusCode != http.StatusAccepted { + return fmt.Errorf("the store did not open an upload of %s (%d): %s", digest, resp.StatusCode, b) + } + loc, err := url.Parse(resp.Header.Get("Location")) + if err != nil { + return err + } + q := loc.Query() + q.Set("digest", digest) + loc.RawQuery = q.Encode() + req, err := http.NewRequest(http.MethodPut, o.url(loc.Path+"?"+loc.RawQuery), body) + if err != nil { + return err + } + req.ContentLength = size + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) + put, err := o.client().Do(req) + if err != nil { + return err + } + defer put.Body.Close() + if put.StatusCode != http.StatusCreated { + b, _ := io.ReadAll(io.LimitReader(put.Body, 4096)) + return fmt.Errorf("the store did not take %s (%d): %s", digest, put.StatusCode, b) + } + return nil +} + +type descriptor struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +// copyManifest copies one manifest and all it names: an index's every child first (the store refuses an index whose +// children it lacks), an image's config and layers; then the manifest itself under its digest. +func (o *OCI) copyManifest(up *Upstream, src Source, name, digest string, depth int) error { + if depth > 2 { + return fmt.Errorf("mirror: %s nests indexes deeper than an image does", digest) + } + if d, err := o.Digest(name, digest); err == nil && d == digest { + return nil // held whole already + } + b, mt, err := up.manifest(src, digest) + if err != nil { + return err + } + var m struct { + Manifests []descriptor `json:"manifests"` + Config descriptor `json:"config"` + Layers []descriptor `json:"layers"` + } + if err := json.Unmarshal(b, &m); err != nil { + return fmt.Errorf("mirror: the manifest %s does not read: %w", digest, err) + } + for _, c := range m.Manifests { + if err := o.copyManifest(up, src, name, c.Digest, depth+1); err != nil { + return err + } + } + if m.Config.Digest != "" { + if err := o.copyBlob(up, src, name, m.Config); err != nil { + return err + } + } + for _, l := range m.Layers { + if err := o.copyBlob(up, src, name, l); err != nil { + return err + } + } + _, err = o.putManifest(name, digest, mt, b) + return err +} + +// Mirror copies an outside image, pinned by its digest, into the store as `upstream//` (design 51: an +// upstream image is copied by `mirror` with its pinned digest), every platform of an index included, only the parts +// the store lacks; it answers the copy's reference. A copy held already is answered at once. +func (o *OCI) Mirror(source, digest string) (string, error) { + src, err := ParseSource(source) + if err != nil { + return "", err + } + name := src.Name() + if err := o.copyManifest(o.Upstream, src, name, digest, 0); err != nil { + return "", err + } + return o.Reference(name, digest), nil +} + +// Reference is the path of the seat's answer for name at digest: `@`, the seat's name put before it by +// the holder (SeatReference). +func (o *OCI) Reference(name, digest string) string { return name + "@" + digest } diff --git a/modules/distribution/internal/holder/mirror_fakes_test.go b/modules/distribution/internal/holder/mirror_fakes_test.go new file mode 100644 index 00000000..6e51ebd0 --- /dev/null +++ b/modules/distribution/internal/holder/mirror_fakes_test.go @@ -0,0 +1,124 @@ +package holder + +import ( + "errors" + "io" + "net/http" + "strconv" + "strings" + "sync" +) + +// fakeRegistry is an OCI registry in memory, enough for a mirror's copy: blobs, uploads and manifests by +// reference. With token set it answers 401 with a Bearer challenge until a request carries it. +type fakeRegistry struct { + mu sync.Mutex + blobs map[string][]byte // name@digest + manifests map[string][]byte // name:ref + types map[string]string + token string + realm string + gets []string +} + +func newFakeRegistry() *fakeRegistry { + return &fakeRegistry{blobs: map[string][]byte{}, manifests: map[string][]byte{}, types: map[string]string{}} +} + +func (f *fakeRegistry) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + if r.URL.Path == "/token" { + _, _ = io.WriteString(w, `{"token":"`+f.token+`"}`) + return + } + if f.token != "" && r.Header.Get("Authorization") != "Bearer "+f.token { + w.Header().Set("WWW-Authenticate", `Bearer realm="`+f.realm+`",service="fake"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + p := strings.TrimPrefix(r.URL.Path, "/v2/") + switch { + case strings.Contains(p, "/blobs/uploads/"): + name := p[:strings.Index(p, "/blobs/uploads/")] + if r.Method == http.MethodPost { + w.Header().Set("Location", "/v2/"+name+"/blobs/uploads/u1") + w.WriteHeader(http.StatusAccepted) + return + } + b, _ := io.ReadAll(r.Body) + d := r.URL.Query().Get("digest") + if digestOf(b) != d { + w.WriteHeader(http.StatusBadRequest) + return + } + f.blobs[name+"@"+d] = b + w.WriteHeader(http.StatusCreated) + case strings.Contains(p, "/blobs/"): + i := strings.Index(p, "/blobs/") + b, ok := f.blobs[p[:i]+"@"+p[i+7:]] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + if r.Method == http.MethodGet { + f.gets = append(f.gets, p[i+7:]) + } + w.Header().Set("Content-Length", strconv.Itoa(len(b))) // a HEAD's too, as a registry answers it + _, _ = w.Write(b) + case strings.Contains(p, "/manifests/"): + i := strings.Index(p, "/manifests/") + key := p[:i] + ":" + p[i+11:] + if r.Method == http.MethodPut { + b, _ := io.ReadAll(r.Body) + f.manifests[key] = b + f.types[key] = r.Header.Get("Content-Type") + w.WriteHeader(http.StatusCreated) + return + } + b, ok := f.manifests[key] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", f.types[key]) + w.Header().Set("Docker-Content-Digest", digestOf(b)) + _, _ = w.Write(b) + default: + w.WriteHeader(http.StatusNotFound) + } +} + +// fakeDelivery answers the driver's reads: haves, and per node whether its node-engine serves manifest-reads. +type fakeDelivery struct { + haves []NodeHave + stated map[string][]string // a node absent does not serve manifest-reads +} + +func (f *fakeDelivery) Haves() ([]NodeHave, error) { return f.haves, nil } +func (f *fakeDelivery) Reads(node string) (bool, []string, error) { + s, ok := f.stated[node] + return ok, s, nil +} + +type memWindows struct { + w map[string]Window + pub Publishing +} + +func (m *memWindows) Window(n string) (Window, error) { return m.w[n], nil } +func (m *memWindows) PutWindow(w Window) error { m.w[w.Name] = w; return nil } +func (m *memWindows) Publishing() (Publishing, error) { return m.pub, nil } + +// downCatalog is a store whose read side does not answer, while down is set. +type downCatalog struct { + fakeCatalog + down bool +} + +func (d *downCatalog) Versions(name string) ([]string, error) { + if d.down { + return nil, errors.New("connection refused") + } + return d.fakeCatalog.Versions(name) +} diff --git a/modules/distribution/internal/holder/oci.go b/modules/distribution/internal/holder/oci.go new file mode 100644 index 00000000..c722d03b --- /dev/null +++ b/modules/distribution/internal/holder/oci.go @@ -0,0 +1,275 @@ +package holder + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" +) + +// OCI is the artifact store as the OCI distribution protocol answers it on its node's loopback, written by +// distribution's own process alone: the Store and the Catalog the verbs use. It answers a reference's path, +// `@sha256:`; the holder puts the seat's name before it (SeatReference, ADR 0605 §9). +type OCI struct { + Base *url.URL + Client *http.Client + // Upstream reads the outside registries `mirror` copies from. + Upstream *Upstream +} + +const ( + mediaManifest = "application/vnd.oci.image.manifest.v1+json" + mediaEmptyConfig = "application/vnd.oci.empty.v1+json" + mediaLayer = "application/vnd.oci.image.layer.v1.tar" + emptyConfig = "{}" +) + +func (o *OCI) client() *http.Client { + if o.Client != nil { + return o.Client + } + return http.DefaultClient +} + +func (o *OCI) url(path string) string { return strings.TrimSuffix(o.Base.String(), "/") + path } + +func (o *OCI) do(method, path, contentType string, body []byte, accept ...string) (*http.Response, []byte, error) { + req, err := http.NewRequest(method, o.url(path), bytes.NewReader(body)) + if err != nil { + return nil, nil, err + } + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + for _, a := range accept { + req.Header.Add("Accept", a) + } + resp, err := o.client().Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<20)) + return resp, b, nil +} + +// Has is whether name holds a blob. +func (o *OCI) Has(name, digest string) (bool, error) { + resp, _, err := o.do(http.MethodHead, "/v2/"+name+"/blobs/"+digest, "", nil) + if err != nil { + return false, err + } + return resp.StatusCode == http.StatusOK, nil +} + +func (o *OCI) putBlob(name, digest string, data []byte) error { + resp, b, err := o.do(http.MethodPost, "/v2/"+name+"/blobs/uploads/", "", nil) + if err != nil { + return err + } + if resp.StatusCode != http.StatusAccepted { + return fmt.Errorf("the store did not open an upload of %s (%d): %s", digest, resp.StatusCode, b) + } + loc, err := url.Parse(resp.Header.Get("Location")) + if err != nil { + return err + } + q := loc.Query() + q.Set("digest", digest) + loc.RawQuery = q.Encode() + path := loc.Path + if loc.RawQuery != "" { + path += "?" + loc.RawQuery + } + resp, b, err = o.do(http.MethodPut, path, "application/octet-stream", data) + if err != nil { + return err + } + if resp.StatusCode != http.StatusCreated { + return fmt.Errorf("the store did not take %s (%d): %s", digest, resp.StatusCode, b) + } + return nil +} + +func (o *OCI) putManifest(name, ref, mediaType string, data []byte) (string, error) { + resp, b, err := o.do(http.MethodPut, "/v2/"+name+"/manifests/"+ref, mediaType, data) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusCreated { + return "", fmt.Errorf("the store did not take the manifest %s:%s (%d): %s", name, ref, resp.StatusCode, b) + } + return "sha256:" + sum(data), nil +} + +// Put stores a transfer: an image's layers and config as blobs and its manifest (the part whose media type is a +// manifest's) under its version; a bundle's or an archive's one blob under an artifact manifest of an empty config and +// one layer (the shape the artifact sweep recognises), under its version, or by its digest alone when unnumbered. +func (o *OCI) Put(t PublishRequest, parts map[string][]byte) (string, error) { + var manifest *Part + digests := make([]string, 0, len(parts)) + for d := range parts { + digests = append(digests, d) + } + sort.Strings(digests) + for i, p := range t.Parts { + // An image's manifest is the part whose digest is the publish's (the seat's contract names no media type); a + // part naming a manifest's media type is one too. + if (t.Kind == KindImage || t.Kind == KindUpstream) && (p.Digest == t.Digest || strings.Contains(p.MediaType, "manifest")) { + manifest = &t.Parts[i] + continue + } + if data, ok := parts[p.Digest]; ok { + if err := o.putBlob(t.Name, p.Digest, data); err != nil { + return "", err + } + } + } + ref := t.Version + switch t.Kind { + case KindImage, KindUpstream: + if manifest == nil { + return "", fmt.Errorf("an image's publish names no manifest part") + } + data, ok := parts[manifest.Digest] + if !ok { + // Held already: fetch it to tag it under the version. + resp, b, err := o.do(http.MethodGet, "/v2/"+t.Name+"/manifests/"+manifest.Digest, "", nil, accepted...) + if err != nil || resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("the held manifest %s cannot be read: %v", manifest.Digest, err) + } + data = b + } + if ref == "" { + ref = manifest.Digest + } + mt := manifest.MediaType + var probe struct { + MediaType string `json:"mediaType"` + } + if mt == "" && json.Unmarshal(data, &probe) == nil { + mt = probe.MediaType + } + if mt == "" { + mt = mediaManifest + } + d, err := o.putManifest(t.Name, ref, mt, data) + if err != nil { + return "", err + } + return t.Name + "@" + d, nil + default: + if err := o.putBlob(t.Name, "sha256:"+sum([]byte(emptyConfig)), []byte(emptyConfig)); err != nil { + return "", err + } + size, err := o.layerSize(t, parts) + if err != nil { + return "", err + } + m, _ := json.Marshal(map[string]any{ + "schemaVersion": 2, "mediaType": mediaManifest, + "config": map[string]any{"mediaType": mediaEmptyConfig, "digest": "sha256:" + sum([]byte(emptyConfig)), "size": len(emptyConfig)}, + "layers": []map[string]any{{"mediaType": mediaLayer, "digest": t.Digest, "size": size}}, + }) + if ref == "" { + ref = "sha256:" + sum(m) + } + if _, err := o.putManifest(t.Name, ref, mediaManifest, m); err != nil { + return "", err + } + return t.Name + "@" + t.Digest, nil + } +} + +// layerSize is the size of a bundle's or an archive's one blob: its bytes' when they travelled, else the size its part +// names, else (a late publish of a held digest names it with no size, its bytes having left the build node) the store's. +func (o *OCI) layerSize(t PublishRequest, parts map[string][]byte) (int64, error) { + if data, ok := parts[t.Digest]; ok { + return int64(len(data)), nil + } + for _, p := range t.Parts { + if p.Digest == t.Digest && p.Size > 0 { + return p.Size, nil + } + } + resp, _, err := o.do(http.MethodHead, "/v2/"+t.Name+"/blobs/"+t.Digest, "", nil) + if err != nil { + return 0, err + } + if resp.StatusCode != http.StatusOK || resp.ContentLength < 0 { + return 0, fmt.Errorf("the store holds no blob %s of %s to tag (%d)", t.Digest, t.Name, resp.StatusCode) + } + return resp.ContentLength, nil +} + +// Held is the reference of name at version when the store holds digest under it. +func (o *OCI) Held(name, version, digest string) (string, bool, error) { + d, err := o.Digest(name, version) + if err != nil || d == "" { + return "", false, err + } + if d != digest { + return "", false, nil + } + return name + "@" + d, true, nil +} + +// Versions are name's tags. +func (o *OCI) Versions(name string) ([]string, error) { + resp, b, err := o.do(http.MethodGet, "/v2/"+name+"/tags/list", "", nil) + if err != nil { + return nil, err + } + if resp.StatusCode == http.StatusNotFound { + return []string{}, nil + } + var out struct { + Tags []string `json:"tags"` + } + if resp.StatusCode != http.StatusOK || json.Unmarshal(b, &out) != nil { + return nil, fmt.Errorf("the store's tags of %s (%d): %s", name, resp.StatusCode, b) + } + sort.Strings(out.Tags) + return out.Tags, nil +} + +// Digest is the manifest digest name holds at version, empty when none. +func (o *OCI) Digest(name, version string) (string, error) { + resp, _, err := o.do(http.MethodHead, "/v2/"+name+"/manifests/"+version, "", nil, mediaManifest, + "application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.v2+json") + if err != nil { + return "", err + } + if resp.StatusCode == http.StatusNotFound { + return "", nil + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("the store answered %d for %s:%s", resp.StatusCode, name, version) + } + return resp.Header.Get("Docker-Content-Digest"), nil +} + +// Remove deletes the manifest name holds at version. +func (o *OCI) Remove(name, version string) error { + d, err := o.Digest(name, version) + if err != nil || d == "" { + return err + } + return o.RemoveDigest(name, d) +} + +// RemoveDigest deletes a manifest by its digest; the store's nightly collector frees its blobs. +func (o *OCI) RemoveDigest(name, digest string) error { + resp, b, err := o.do(http.MethodDelete, "/v2/"+name+"/manifests/"+digest, "", nil) + if err != nil { + return err + } + if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusNotFound { + return fmt.Errorf("the store did not delete %s@%s (%d): %s", name, digest, resp.StatusCode, b) + } + return nil +} diff --git a/modules/distribution/internal/holder/platforms.go b/modules/distribution/internal/holder/platforms.go new file mode 100644 index 00000000..8cc0bf2e --- /dev/null +++ b/modules/distribution/internal/holder/platforms.go @@ -0,0 +1,117 @@ +package holder + +// **An index's platform manifests go with it, unless a kept index names them** (CAT-AS1: what the artifact sweep did +// over the store's door, ADR 0257, is the holder's now that the sweep asks `remove`). A removal for retention of an +// index removes first every platform manifest it names that no other kept index of the repository names, then the +// index; a kept index is one a tag names or one held (`hold`). A manifest a kept index names is never removed for +// retention, so holding an index keeps its platforms too. + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" +) + +// spared is every manifest of name a kept index keeps on seat, but the one being removed (except): each tagged +// manifest and held digest, and the platform manifests each of those names. +func (v *Verbs) spared(seat, name, except string) (map[string]string, error) { + keepers := map[string]string{} // digest -> why it is kept + tags, err := v.Catalog.Versions(name) + if err != nil { + return nil, err + } + for _, t := range tags { + d, err := v.Catalog.Digest(name, t) + if err != nil { + return nil, err + } + if d != "" { + keepers[d] = "the tag " + t + } + } + holds, err := v.Holds.List() + if err != nil { + return nil, err + } + for _, h := range holds { + if h.Seat == seat && h.Name == name { + keepers[h.Digest] = "the hold of " + h.Digest + } + } + out := map[string]string{} + for d, why := range keepers { + if d == except { + continue + } + out[d] = why + children, err := v.Catalog.Platforms(name, d) + if err != nil { + return nil, fmt.Errorf("reading the kept %s@%s: %w", name, d, err) + } + for _, c := range children { + out[c] = "the index " + d + ", kept by " + why + } + } + return out, nil +} + +// removeForRetention removes name@digest and, for an index, the platform manifests no kept index names; a manifest a +// kept index names is refused. Any read that fails removes nothing. +func (v *Verbs) removeForRetention(seat, name, digest string) error { + children, err := v.Catalog.Platforms(name, digest) + if err != nil { + return fmt.Errorf("remove: the store cannot read %s@%s, so nothing is removed: %w", name, digest, err) + } + spared, err := v.spared(seat, name, digest) + if err != nil { + return fmt.Errorf("remove: which manifests a kept index of %s names cannot be read, so nothing is removed: %w", name, err) + } + if why, kept := spared[digest]; kept { + return fmt.Errorf("remove: refused: %s@%s is kept by %s", name, digest, why) + } + for _, c := range children { + if _, kept := spared[c]; kept { + continue + } + if err := v.Catalog.RemoveDigest(name, c); err != nil { + return fmt.Errorf("remove: the store did not remove %s@%s, a platform of %s: %w", name, c, digest, err) + } + } + if err := v.Catalog.RemoveDigest(name, digest); err != nil { + return fmt.Errorf("remove: the store did not remove %s@%s: %w", name, digest, err) + } + return nil +} + +// Platforms is the manifests an index names, by digest: none for a manifest that is not an index, or one the store +// does not hold. +func (o *OCI) Platforms(name, digest string) ([]string, error) { + resp, b, err := o.do(http.MethodGet, "/v2/"+name+"/manifests/"+digest, "", nil, + "application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json", + mediaManifest, "application/vnd.docker.distribution.manifest.v2+json") + if err != nil { + return nil, err + } + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("the store answered %d for %s@%s", resp.StatusCode, name, digest) + } + var doc struct { + Manifests []struct { + Digest string `json:"digest"` + } `json:"manifests"` + } + if err := json.Unmarshal(b, &doc); err != nil { + return nil, fmt.Errorf("%s@%s is not a manifest: %w", name, digest, err) + } + var out []string + for _, m := range doc.Manifests { + if strings.HasPrefix(m.Digest, "sha256:") { + out = append(out, m.Digest) + } + } + return out, nil +} diff --git a/modules/distribution/internal/holder/transfer.go b/modules/distribution/internal/holder/transfer.go new file mode 100644 index 00000000..6df69f08 --- /dev/null +++ b/modules/distribution/internal/holder/transfer.go @@ -0,0 +1,311 @@ +package holder + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "slices" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/parts" +) + +// The bounds of incoming (design 51 *Publishing*, ADR 0611): a part is at most 1 MiB, the store 20 GiB, a transfer +// with no part for 24 hours is dropped, and past 80 % of the cap the holder writes a finding. +const ( + MaxPart = 1 << 20 + IncomingCap = 20 << 30 + TransferIdle = 24 * time.Hour + IncomingFullPct = 80 + // IncomingObjects and TransfersState are the object store and the state distribution asks as bus state. + IncomingObjects = "incoming" + TransfersState = "transfers" +) + +// The kinds a seat holds. +const ( + KindImage = "image" + KindBundle = "bundle" + KindArchive = "archive" + KindUpstream = "upstream" +) + +// Part is one file of a publish: an image's layer, config or manifest, a bundle's or an archive's blob, by its digest. +type Part struct { + Digest string `json:"digest"` + Size int64 `json:"size"` + // MediaType says what the part is to the store; an image's manifest names its own. + MediaType string `json:"media_type,omitempty"` +} + +// PublishRequest is the body of `publish` (ADR 0611 §1). +type PublishRequest struct { + Transfer string `json:"transfer"` + Name string `json:"name"` + Version string `json:"version,omitempty"` + Tag string `json:"tag,omitempty"` + Kind string `json:"kind"` + Fingerprint string `json:"fingerprint,omitempty"` + Commit string `json:"commit,omitempty"` + // Digest is the digest the build recorded: an image's manifest, a bundle's or an archive's blob. + Digest string `json:"digest"` + Parts []Part `json:"parts"` +} + +// PublishAnswer is what `publish` answers: the parts the holder lacks, which the build module sends; none when it +// holds them all and the publish ended (Published names how). +type PublishAnswer struct { + Transfer string `json:"transfer"` + Missing []string `json:"missing"` + // Ended says the publish ended at once: everything was held, `published` or `refused` is said. + Ended bool `json:"ended,omitempty"` +} + +// PartRequest is the body of `publish-part` (ADR 0611 §2): one chunk of one part. +type PartRequest struct { + Transfer string `json:"transfer"` + Digest string `json:"digest"` + Sequence int `json:"sequence"` + Last bool `json:"last"` + Data []byte `json:"data"` +} + +// Opening is what distribution keeps of a publish beside the receiving side's own fields: the request and its seat. +type Opening struct { + PublishRequest + Seat string `json:"seat"` +} + +// Transfer is one open publish, kept in the state transfers; its chunks are objects of incoming. The receiving side is +// mesh-sdk's seat-neutral parts.Receiver (ADR 0627 §2); distribution binds it to the seats it holds. +type Transfer = parts.Transfer[Opening] + +// Event is the body of `published`, `refused` and `removed` on `.events` (design 51 *One contract*): the name, +// version, kind, tag, digest, fingerprint, commit and holder; `refused` its reason, `removed` its reason too. +type Event struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Kind string `json:"kind"` + Tag string `json:"tag,omitempty"` + Digest string `json:"digest"` + Fingerprint string `json:"fingerprint,omitempty"` + Commit string `json:"commit,omitempty"` + Holder string `json:"holder"` + Transfer string `json:"transfer,omitempty"` + // Reference is the seat's answer: `//@sha256:` (ADR 0605 §9). + Reference string `json:"reference,omitempty"` + Reason string `json:"reason,omitempty"` + // By is who asked a removal, hold or release: the caller the tool runner proved. + By string `json:"by,omitempty"` + // ID is the event's id on the bus when the outcome has one of its own (a mirror ask's): the same outcome said + // twice, by a redelivery or a second follower, is one message. Empty: the runtime makes one. + ID string `json:"-"` +} + +// The events of the seats. +const ( + EventPublished = "published" + EventRefused = "refused" + EventRemoved = "removed" + EventHeld = "held" + EventReleased = "released" +) + +// Objects is distribution's own object store incoming. +type Objects = parts.Objects + +// TransferState is the state transfers. +type TransferState = parts.Transfers[Opening] + +// Builder is the builder's tool builder_versions, asked before anything is stored (ADR 0605 §4). +type Builder interface { + Recorded(name, version, digest string) (ok bool, why string, err error) +} + +// Store is the artifact store behind the loopback, written by distribution's own process alone. +type Store interface { + // Has says whether the store holds a blob of name. + Has(name, digest string) (bool, error) + // Put stores a publish's parts under its name and version and answers the reference's path (SeatReference). + Put(r PublishRequest, parts map[string][]byte) (reference string, err error) + // Held is the reference's path of name at version when the store already holds digest under it: a rebuild of a held + // fingerprint answers it and stores nothing. + Held(name, version, digest string) (reference string, ok bool, err error) +} + +// SeatReference is a reference in the form the seat answers (novox/hq ADR 0605 §9): the seat's name, then the path the +// store answers (`@sha256:`), as catalogue.InRegistrySeat reads it back. Nothing set on the process names +// a reach: the seat's name is the registry part. +func SeatReference(seat, path string) string { return seat + "/" + path } + +// Say says an event on a seat's family `.events`. +type Say func(seat, event string, body Event) error + +// Finding writes (raised true) or clears distribution's finding of a kind for a subject. +type Finding func(kind, subject, words string, raised bool) + +// Holder answers the verbs. +type Holder struct { + Callers Callers + Objects Objects + Transfers TransferState + Builder Builder + Store Store + Say Say + Finding Finding + Now func() time.Time + // Cap is incoming's cap; IncomingCap when zero. + Cap uint64 +} + +func (h *Holder) now() time.Time { + if h.Now != nil { + return h.Now().UTC() + } + return time.Now().UTC() +} + +func (h *Holder) cap() uint64 { + if h.Cap > 0 { + return h.Cap + } + return IncomingCap +} + +// receiver binds mesh-sdk's receiving side to distribution's incoming, its transfers state, the builder's check, its +// store and its seats' events. +func (h *Holder) receiver() *parts.Receiver[Opening] { + return &parts.Receiver[Opening]{Objects: h.Objects, Transfers: h.Transfers, Cap: h.cap(), MaxChunk: MaxPart, + Idle: TransferIdle, Full: IncomingFullPct, Now: h.now, + // Only the missing layers are sent: a part the store already holds under the name counts as held. + Holds: func(t Transfer, digest string) bool { + ok, err := h.Store.Has(t.Shape.Name, digest) + return err == nil && ok + }, + // builder_versions is asked before anything is stored (ADR 0605 §4). + Check: func(t Transfer) error { + if t.Shape.Version == "" { + return nil + } + ok, why, err := h.Builder.Recorded(t.Shape.Name, t.Shape.Version, t.Shape.Digest) + if err != nil { + return errors.New("builder_versions did not answer, so nothing is stored: " + err.Error()) + } + if !ok { + return errors.New("the builder did not number it: " + why) + } + return nil + }, + Store: func(t Transfer, held map[string][]byte) error { + path, err := h.Store.Put(t.Shape.PublishRequest, held) + ref := SeatReference(t.Shape.Seat, path) + if err != nil { + reason := "the store did not take it: " + err.Error() + return errors.Join(errors.New("refused: "+reason), h.refused(t, reason)) + } + h.Finding("publish-refused", t.Shape.Name+"@"+t.Shape.Version, "", false) + // Room was found again: a refusal for want of it clears with the first publish the holder takes. + h.Finding("publish-refused", IncomingObjects, "", false) + return h.Say(t.Shape.Seat, EventPublished, h.eventOf(t.Shape.Seat, t.Shape.PublishRequest, ref, "")) + }, + Refuse: h.refused, + Filled: func(used, cap uint64, full bool) { + h.Finding("incoming-full", IncomingObjects, fmt.Sprintf("incoming holds %d of its %d bytes (%d %%); registry_incoming lists and drops transfers", used, cap, used*100/cap), full) + }} +} + +// refused writes the finding and says refused on the transfer's seat. A refusal for want of room is the holder's, +// not the file's: its finding stands on incoming and clears when the holder takes a publish again. +func (h *Holder) refused(t Transfer, reason string) error { + subject := t.Shape.Name + "@" + t.Shape.Version + if parts.RefusedAtCap(reason) { + subject = IncomingObjects + } + h.Finding("publish-refused", subject, reason, true) + return h.Say(t.Shape.Seat, EventRefused, h.eventOf(t.Shape.Seat, t.Shape.PublishRequest, "", reason)) +} + +func answerOf(r parts.Received) PublishAnswer { + return PublishAnswer{Transfer: r.ID, Missing: r.Lacks, Ended: r.Ended} +} + +// Publish is `publish` on seat for the proved caller. +func (h *Holder) Publish(seat, caller string, r PublishRequest) (PublishAnswer, error) { + if err := h.Callers.Allowed(VerbPublish, caller); err != nil { + return PublishAnswer{}, err + } + if err := checkPublish(seat, r); err != nil { + return PublishAnswer{}, err + } + // A rebuild of a held fingerprint answers the held version and stores nothing. + if r.Version != "" { + if ref, ok, err := h.Store.Held(r.Name, r.Version, r.Digest); err != nil { + return PublishAnswer{}, fmt.Errorf("publish: the store cannot say whether it holds %s@%s: %w", r.Name, r.Version, err) + } else if ok { + return PublishAnswer{Transfer: r.Transfer, Ended: true}, h.Say(seat, EventPublished, h.eventOf(seat, r, SeatReference(seat, ref), "")) + } + } + wants := make([]parts.Want, 0, len(r.Parts)) + for _, p := range r.Parts { + wants = append(wants, parts.Want{Digest: p.Digest, Size: p.Size}) + } + got, err := h.receiver().Open(r.Transfer, caller, wants, Opening{PublishRequest: r, Seat: seat}) + if err != nil { + return answerOf(got), fmt.Errorf("publish: %w", err) + } + return answerOf(got), nil +} + +// PublishPart is `publish-part`: one chunk, from the caller that opened the transfer alone. +func (h *Holder) PublishPart(seat, caller string, r PartRequest) (PublishAnswer, error) { + if err := h.Callers.Allowed(VerbPublishPart, caller); err != nil { + return PublishAnswer{}, err + } + got, err := h.receiver().Take(r.Transfer, caller, r.Digest, r.Sequence, r.Last, r.Data, func(t Transfer) error { + if t.Shape.Seat != seat { + return fmt.Errorf("transfer %s is %s's, not %s's", r.Transfer, t.Shape.Seat, seat) + } + return nil + }) + if err != nil { + return answerOf(got), fmt.Errorf("publish-part: %w", err) + } + return answerOf(got), nil +} + +func (h *Holder) eventOf(seat string, r PublishRequest, ref, reason string) Event { + return Event{Name: r.Name, Version: r.Version, Kind: r.Kind, Tag: r.Tag, Digest: r.Digest, Fingerprint: r.Fingerprint, + Commit: r.Commit, Holder: Self, Transfer: r.Transfer, Reference: ref, Reason: reason} +} + +// Drop drops a transfer by hand (registry_incoming) or because it was idle for TransferIdle: its chunks and its key. +func (h *Holder) Drop(id string) (Transfer, error) { return h.receiver().Drop(id) } + +// DropIdle drops every transfer with no part for TransferIdle, answering them; it runs on each publish's open and each +// registry_incoming call, never on a timer (the object store's own lifetime of 24 hours drops the bytes too). +func (h *Holder) DropIdle() ([]string, error) { return h.receiver().DropIdle() } + +func checkPublish(seat string, r PublishRequest) error { + switch { + case !slices.Contains(Seats, seat): + return fmt.Errorf("publish: distribution holds no seat %s", seat) + case strings.TrimSpace(r.Name) == "": + return errors.New("publish: no name") + case !strings.HasPrefix(r.Digest, "sha256:"): + return fmt.Errorf("publish: digest %q is no sha256 digest", r.Digest) + } + switch { + case seat == ImageSeat && r.Kind != KindImage && r.Kind != KindUpstream: + return fmt.Errorf("publish: %s holds images, not a %s", seat, r.Kind) + case seat == ArtifactSeat && r.Kind != KindBundle && r.Kind != KindArchive: + return fmt.Errorf("publish: %s holds bundles and archives, not a %s", seat, r.Kind) + } + return nil +} + +func sum(b []byte) string { + s := sha256.Sum256(b) + return hex.EncodeToString(s[:]) +} diff --git a/modules/distribution/internal/holder/verbs.go b/modules/distribution/internal/holder/verbs.go new file mode 100644 index 00000000..cc90a063 --- /dev/null +++ b/modules/distribution/internal/holder/verbs.go @@ -0,0 +1,327 @@ +package holder + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "slices" + "strings" + "time" + + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" +) + +// Catalog is what the store answers of what it holds. +type Catalog interface { + // Versions are the versions (tags) of name, sorted. + Versions(name string) ([]string, error) + // Digest is the digest name holds at version, empty when none. + Digest(name, version string) (string, error) + // Remove deletes name at version (its manifest); the store's collector frees its blobs. + Remove(name, version string) error + // RemoveDigest deletes an untagged manifest of name by its digest (retention). + RemoveDigest(name, digest string) error + // Platforms are the manifests an index of name names; none for a manifest that is not an index or is not held. + Platforms(name, digest string) ([]string, error) + // Mirror copies an outside image, pinned by digest, into upstream//, answering the reference. + Mirror(source, digest string) (string, error) + // Reference is the seat's answer for name at digest, known before anything is copied. + Reference(name, digest string) string +} + +// MirrorsState names the state of the copies `mirror` took on. +const MirrorsState = "mirrors" + +// Mirrors is the state mirrors: each copy `mirror` took on, kept until its outcome is said, so a copy cut by a +// restart is taken on again from the state's watch. +type Mirrors interface { + // Ask keeps r under its key; an ask already kept is no fault (the copy is taken on once). + Ask(key string, r MirrorRequest) error + // Claim writes r (its Claim set) under key at revision, compare-and-set: false, and no error, when another + // writer wrote the key first. + Claim(key string, r MirrorRequest, revision uint64) (bool, error) + Delete(key string) error +} + +// HoldState is distribution's own state holds (bus state, template state; ADR 0617 §2). +type HoldState interface { + Get(key string) (Hold, bool, error) + Put(h Hold) error + Delete(key string) error + List() ([]Hold, error) +} + +// Hold keeps one untagged bundle, archive or copied base from retention's removal. +type Hold struct { + Seat string `json:"seat"` + Name string `json:"name"` + Kind string `json:"kind"` + Digest string `json:"digest"` + By string `json:"by"` + Why string `json:"why,omitempty"` + Since time.Time `json:"since"` +} + +// Key is the hold's key in the state. +func (h Hold) Key() string { + return h.Seat + "." + strings.ReplaceAll(strings.ReplaceAll(h.Name, ".", "="), "@", "=") + "." + strings.TrimPrefix(h.Digest, "sha256:") +} + +// Verbs answers the verbs other than the publish: versions, remove, hold, release, tags and mirror. +type Verbs struct { + *Holder + Catalog Catalog + Holds HoldState + Mirrors Mirrors +} + +// VersionsAnswer is `versions`: the versions the seat holds of name. +type VersionsAnswer struct { + Seat string `json:"seat"` + Name string `json:"name"` + Versions []string `json:"versions"` +} + +// Versions is `versions`. +func (v *Verbs) Versions(seat, caller, name string) (VersionsAnswer, error) { + if err := v.Callers.Allowed(VerbVersions, caller); err != nil { + return VersionsAnswer{}, err + } + if strings.TrimSpace(name) == "" { + return VersionsAnswer{}, errors.New("versions: no name") + } + got, err := v.Catalog.Versions(name) + if err != nil { + return VersionsAnswer{}, fmt.Errorf("versions: the store cannot list %s: %w", name, err) + } + if got == nil { + got = []string{} + } + return VersionsAnswer{Seat: seat, Name: name, Versions: got}, nil +} + +// RemoveRequest is `remove`: a tagged version, or with the reason retention an untagged one by its digest. +type RemoveRequest struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Digest string `json:"digest,omitempty"` + Kind string `json:"kind,omitempty"` + Reason string `json:"reason"` +} + +// ReasonRetention is the reason of the sweep's removal of an untagged version (ADR 0610 §3). +// The seat definitions name it (the removal reasons' enum), so the sweep and the holder share one value. +const ReasonRetention = reg.RemoveInputReasonRetention + +// Remove is `remove`: the builder removes a tagged version; the artifact sweep's holder an untagged bundle, archive or +// copied base with the reason retention, which a hold keeps. Every removal says `removed` with its reason. +func (v *Verbs) Remove(seat, caller string, r RemoveRequest) error { + if err := v.Callers.Allowed(VerbRemove, caller); err != nil { + return err + } + if strings.TrimSpace(r.Name) == "" || strings.TrimSpace(r.Reason) == "" { + return errors.New("remove: a name and a reason are required") + } + ev := Event{Name: r.Name, Version: r.Version, Kind: r.Kind, Digest: r.Digest, Holder: Self, Reason: r.Reason, By: caller} + switch { + case r.Reason == ReasonRetention: + if CallerModule(caller) != SweepHolder { + return fmt.Errorf("remove: refused for %s: a removal for retention is the artifact sweep's alone", caller) + } + if r.Digest == "" { + return errors.New("remove: a removal for retention names the digest") + } + if _, held, err := v.Holds.Get(Hold{Seat: seat, Name: r.Name, Digest: r.Digest}.Key()); err != nil { + return fmt.Errorf("remove: the holds cannot be read, so nothing is removed: %w", err) + } else if held { + return fmt.Errorf("remove: refused: %s@%s is held; release it first", r.Name, r.Digest) + } + if err := v.removeForRetention(seat, r.Name, r.Digest); err != nil { + return err + } + default: + if r.Version == "" || !strings.Contains(r.Version, "-") { + return fmt.Errorf("remove: refused: %s@%s has no tag; only a tagged version is removed, but for retention", r.Name, r.Version) + } + d, err := v.Catalog.Digest(r.Name, r.Version) + if err != nil { + return fmt.Errorf("remove: the store cannot read %s@%s: %w", r.Name, r.Version, err) + } + if d != "" { + if err := v.Catalog.Remove(r.Name, r.Version); err != nil { + return fmt.Errorf("remove: the store did not remove %s@%s: %w", r.Name, r.Version, err) + } + } + ev.Digest = d // a version already gone is said removed all the same, so the builder's retry ends + } + return v.Say(seat, EventRemoved, ev) +} + +// HoldRequest is `hold` and `release`. +type HoldRequest struct { + Name string `json:"name"` + Kind string `json:"kind"` + Digest string `json:"digest"` + Why string `json:"why,omitempty"` +} + +// HoldOne is `hold`: the artifact sweep's holder alone; an image is never held (its own rules keep images). +func (v *Verbs) HoldOne(seat, caller string, r HoldRequest) error { + if err := v.Callers.Allowed(VerbHold, caller); err != nil { + return err + } + // The seat's contract names no kind: on the image seat a hold is of a copied base (upstream/…), never an image of + // its own; on the artifact seat of a bundle or an archive. + if r.Kind == "" && seat == ImageSeat && strings.HasPrefix(r.Name, "upstream/") { + r.Kind = KindUpstream + } + switch { + case r.Kind == "" && seat == ArtifactSeat: + case r.Kind == KindBundle, r.Kind == KindArchive, r.Kind == KindUpstream: + default: + return fmt.Errorf("hold: refused: a %s is never held; only an untagged bundle, archive or copied base", r.Kind) + } + if r.Name == "" || !strings.HasPrefix(r.Digest, "sha256:") { + return errors.New("hold: a name and a sha256 digest are required") + } + h := Hold{Seat: seat, Name: r.Name, Kind: r.Kind, Digest: r.Digest, By: caller, Why: r.Why, Since: v.now()} + if err := v.Holds.Put(h); err != nil { + return fmt.Errorf("hold: the holds did not take it: %w", err) + } + return v.Say(seat, EventHeld, Event{Name: r.Name, Kind: r.Kind, Digest: r.Digest, Holder: Self, By: caller}) +} + +// Release is `release`: the artifact sweep's holder, or the operator's session through registry_holds (a hand act, +// recorded by the tool); distribution's own account is never on its list. +func (v *Verbs) Release(seat, caller string, r HoldRequest) (bool, error) { + if CallerModule(caller) == Self && !strings.HasPrefix(caller, "person.") { + return false, fmt.Errorf("release: refused for %s: distribution cannot release in its own name", caller) + } + if err := v.Callers.Allowed(VerbRelease, caller); err != nil { + return false, err + } + key := Hold{Seat: seat, Name: r.Name, Digest: r.Digest}.Key() + h, held, err := v.Holds.Get(key) + if err != nil { + return false, err + } + if !held { + return false, nil // nothing held: the outcome asked holds already, and nothing is said (the seat's definition) + } + if err := v.Holds.Delete(key); err != nil { + return false, err + } + return true, v.Say(seat, EventReleased, Event{Name: h.Name, Kind: h.Kind, Digest: h.Digest, Holder: Self, By: caller, Reason: r.Why}) +} + +// Tags is `tags` on the image seat: the tags of an image. +func (v *Verbs) Tags(seat, caller, name string) (VersionsAnswer, error) { + if seat != ImageSeat { + return VersionsAnswer{}, fmt.Errorf("tags: %s serves no tags", seat) + } + if err := v.Callers.Allowed(VerbTags, caller); err != nil { + return VersionsAnswer{}, err + } + got, err := v.Catalog.Versions(name) + if err != nil { + return VersionsAnswer{}, err + } + return VersionsAnswer{Seat: seat, Name: name, Versions: got}, nil +} + +// MirrorRequest is `mirror`: an outside image by its pinned digest; By is the proved caller, kept with the ask. +type MirrorRequest struct { + Source string `json:"source"` + Digest string `json:"digest"` + By string `json:"by,omitempty"` + // ID is the ask's own id, made when it is asked: the id of the event that says its end. + ID string `json:"id,omitempty"` + // Claim is the follower that took the copy on: none until one claims it. + Claim string `json:"claim,omitempty"` +} + +// MirrorsKey is an ask's key in the state mirrors: the copy's name and digest, in a key's characters. +func MirrorsKey(name, digest string) string { + return strings.NewReplacer(":", "_", "@", "_").Replace(name + "/" + digest) +} + +// Mirror is `mirror` on the image seat: mesh-build-image alone; an image pinned by no digest is refused. It answers at +// once with the reference the copy will have and keeps the ask in the state mirrors; the copy runs on that state's +// watch (Mirrored), which says published or refused on the seat's events when it ends. The caller waits on that +// event, not on the call, so a large base never outlasts a call's bound. +func (v *Verbs) Mirror(seat, caller string, r MirrorRequest) (string, error) { + if seat != ImageSeat { + return "", fmt.Errorf("mirror: %s serves no mirror", seat) + } + if err := v.Callers.Allowed(VerbMirror, caller); err != nil { + return "", err + } + if r.Source == "" || !strings.HasPrefix(r.Digest, "sha256:") { + return "", errors.New("mirror: refused: an upstream image is copied only by its pinned digest") + } + src, err := ParseSource(r.Source) + if err != nil { + return "", err + } + r.By, r.Claim = caller, "" + if r.ID == "" { + id := make([]byte, 8) + _, _ = rand.Read(id) + r.ID = "mirror-" + hex.EncodeToString(id) + } + if err := v.Mirrors.Ask(MirrorsKey(src.Name(), r.Digest), r); err != nil { + return "", fmt.Errorf("mirror: the copy of %s@%s cannot be taken on: %w", r.Source, r.Digest, err) + } + return SeatReference(seat, v.Catalog.Reference(src.Name(), r.Digest)), nil +} + +// Mirrored runs one asked copy and says its outcome on the image seat's events: published with the reference, or +// refused with the reason; the ask leaves the state either way. An outcome that cannot be said keeps the ask, so the +// next watch takes it on again. +func (v *Verbs) Mirrored(r MirrorRequest) error { + src, err := ParseSource(r.Source) + if err != nil { + return err + } + e := Event{Name: src.Name(), Kind: KindUpstream, Digest: r.Digest, Holder: Self, By: r.By, ID: r.ID} + event := EventPublished + path, err := v.Catalog.Mirror(r.Source, r.Digest) + e.Reference = SeatReference(ImageSeat, path) + if err != nil { + event, e.Reference, e.Reason = EventRefused, "", fmt.Sprintf("%s@%s was not copied: %v", r.Source, r.Digest, err) + } + if err := v.Say(ImageSeat, event, e); err != nil { + return fmt.Errorf("mirror: the outcome of %s@%s cannot be said: %w", r.Source, r.Digest, err) + } + return v.Mirrors.Delete(MirrorsKey(src.Name(), r.Digest)) +} + +// Take decides one change of the state mirrors for the follower self: an ask no follower claimed, or one claimed +// before this follower's watch began (its claimant may have been cut by a restart), is claimed by compare-and-set at +// revision and copied by the follower that won; an ask claimed since, by self or by another, is passed. Two followers +// that both start on the same cut ask can both win in turn; the outcome then carries the ask's id, so it is said once. +func (v *Verbs) Take(key string, r MirrorRequest, revision uint64, current bool, self string) error { + if r.Claim != "" && (r.Claim == self || !current) { + return nil + } + r.Claim = self + won, err := v.Mirrors.Claim(key, r, revision) + if err != nil { + return fmt.Errorf("mirror: the ask %s cannot be claimed: %w", key, err) + } + if !won { + return nil + } + return v.Mirrored(r) +} + +// HeldNow lists the holds, for registry_holds. +func (v *Verbs) HeldNow() ([]Hold, error) { + all, err := v.Holds.List() + if err != nil { + return nil, err + } + slices.SortFunc(all, func(a, b Hold) int { return strings.Compare(a.Key(), b.Key()) }) + return all, nil +} diff --git a/modules/distribution/internal/holder/window.go b/modules/distribution/internal/holder/window.go new file mode 100644 index 00000000..3336d74c --- /dev/null +++ b/modules/distribution/internal/holder/window.go @@ -0,0 +1,152 @@ +package holder + +import ( + "fmt" + "slices" + "sort" + "time" +) + +// The two windows distribution keeps (design 51): the old seat name `mesh-artifact-store`, served until every node whose +// declaration names an artifact applied a build whose node-engine states it reads the new names (ADR 0610 §4, ADR 0614 +// §4), and the direct writes to the store, accepted from the mesh until every node's have of every build module runs a +// build that publishes through the seats. Each wakes at distribution's start, on each `delivered` and on the node-engine +// seat's `manifest-reads-changed` (ADR 0618 §4), never on a timer; each reads every node, away ones included, through +// node_haves and `manifest-reads`, never a watch of another module's state (ADR 0596). +const ( + OldNameWindow = "old-name" + DirectWritesWindow = "direct-writes" + // BootstrapWindow is the bootstrap's one-off credential (BootstrapFile): open while the file is there. The tools + // bundle closes it at its first answered publish; the program, which runs as distribution's account and alone may + // remove the file in distribution's state directory, removes it on that close. + BootstrapWindow = "bootstrap-push" + // WindowFindingAfter is how long a node holds a window open before it is distribution's finding naming the node. + WindowFindingAfter = 7 * 24 * time.Hour + // The events a window's close says on distribution's own family distribution.events (7 days, 16 MiB, drop + // oldest), never on a seat's `.events`. + OldNameClosedEvent = "old-name-closed" + DirectWritesClosedEvent = "direct-writes-closed" + // NewNamesCapability is the reader capability a node-engine states when it reads the new seat names (ADR 0614 §4). + NewNamesCapability = "seat:" + ImageSeat +) + +// Window is one window's record, kept in distribution's state windows: what registry_old_name and +// registry_direct_writes answer. +type Window struct { + Name string `json:"name"` + Open bool `json:"open"` + Holding []string `json:"holding,omitempty"` + Since map[string]time.Time `json:"since,omitempty"` + Dropped []DroppedNode `json:"dropped,omitempty"` + Closed time.Time `json:"closed,omitzero"` + // LastHolding are the nodes that held it last, said when it closes. + LastHolding []string `json:"last_holding,omitempty"` +} + +// DroppedNode is one retired node dropped from a window by hand: a hand act. +type DroppedNode struct { + Node string `json:"node"` + By string `json:"by"` + Why string `json:"why"` + At time.Time `json:"at"` +} + +// Closed is the body of a window's closing event. +type Closed struct { + Window string `json:"window"` + Seat string `json:"seat,omitempty"` + At time.Time `json:"at"` + LastHolding []string `json:"last_holding"` + Dropped []DroppedNode `json:"dropped,omitempty"` +} + +// NodeRead is what distribution read of one node: the reader capabilities its node-engine states through +// `manifest-reads` (Serves false while it does not serve the verb: it states nothing), whether its declaration names an +// artifact, and its build modules' haves: whether each runs a build that publishes through the seats. +type NodeRead struct { + Node string `json:"node"` + Serves bool `json:"serves"` + Stated []string `json:"stated,omitempty"` + NamesArtifact bool `json:"names_artifact"` + BuildModules map[string]bool `json:"build_modules,omitempty"` +} + +// HoldingOldName are the nodes holding the old name open: a node whose declaration names an artifact and whose +// node-engine does not state the new names (one not serving manifest-reads states nothing). +func HoldingOldName(nodes []NodeRead, dropped []DroppedNode) []string { + var out []string + for _, n := range nodes { + if isDropped(dropped, n.Node) || !n.NamesArtifact { + continue + } + if !n.Serves || !slices.Contains(n.Stated, NewNamesCapability) { + out = append(out, n.Node) + } + } + sort.Strings(out) + return out +} + +// HoldingDirectWrites are the nodes one of whose build modules' haves runs a build that does not yet publish through +// the seats. +func HoldingDirectWrites(nodes []NodeRead, dropped []DroppedNode) []string { + var out []string + for _, n := range nodes { + if isDropped(dropped, n.Node) { + continue + } + for _, publishes := range n.BuildModules { + if !publishes { + out = append(out, n.Node) + break + } + } + } + sort.Strings(out) + return out +} + +func isDropped(dropped []DroppedNode, node string) bool { + return slices.ContainsFunc(dropped, func(d DroppedNode) bool { return d.Node == node }) +} + +// Step moves the window to the nodes holding it at now. It answers the next record, the closing event when it closed in +// this step (said once), and the nodes holding it past WindowFindingAfter (the finding; cleared when it closes). A +// window once closed stays closed: a node that holds it later is no reason to serve the old name again. +func (w Window) Step(holding []string, now time.Time) (next Window, closed *Closed, overdue []string) { + next = Window{Name: w.Name, Dropped: w.Dropped, Closed: w.Closed, LastHolding: w.LastHolding} + if !w.Open && !w.Closed.IsZero() { + return next, nil, nil + } + if len(holding) == 0 { + next.Closed, next.LastHolding = now, w.Holding + if next.LastHolding == nil { + next.LastHolding = []string{} + } + return next, &Closed{Window: w.Name, At: now, LastHolding: next.LastHolding, Dropped: w.Dropped}, nil + } + next.Open, next.Holding, next.Since = true, holding, map[string]time.Time{} + for _, n := range holding { + since, had := w.Since[n] + if !had { + since = now + } + next.Since[n] = since + if now.Sub(since) >= WindowFindingAfter { + overdue = append(overdue, n) + } + } + return next, nil, overdue +} + +// Drop drops a retired node from the window by hand, a hand act with who and why. +func (w Window) Drop(node, by, why string, now time.Time) (Window, error) { + if node == "" || why == "" { + return w, fmt.Errorf("drop: a node and why are required (a hand act)") + } + if isDropped(w.Dropped, node) { + return w, fmt.Errorf("drop: %s is dropped from %s already", node, w.Name) + } + w.Dropped = append(w.Dropped, DroppedNode{Node: node, By: by, Why: why, At: now}) + return w, nil +} diff --git a/modules/distribution/module.json b/modules/distribution/module.json index 5037f429..f3e98583 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -1,6 +1,6 @@ { "module": "distribution", - "version": "1", + "version": "2", "provides": [ { "name": "artifact-store", @@ -11,17 +11,120 @@ } ], "claims": [ + { + "name": "container-image-registry", + "scope": "mesh", + "serves": [ + "hold", + "mirror", + "publish", + "publish-part", + "release", + "remove", + "tags", + "versions" + ] + }, + { + "name": "oci-artifact-registry", + "scope": "mesh", + "serves": [ + "hold", + "publish", + "publish-part", + "release", + "remove", + "versions" + ] + }, { "name": "mesh-artifact-store", "scope": "mesh" } ], + "tools": [ + "registry_callers", + "registry_direct_writes", + "registry_health", + "registry_holds", + "registry_incoming", + "registry_old_name" + ], + "invokes": [ + "distribution.registry_callers", + "mesh-delivery-builder.builder_versions", + "mesh-delivery-view.node_haves", + "seat:node-engine.manifest-reads" + ], + "consumes": [ + "mesh-delivery-view.delivered", + "node-engine.manifest-reads-changed" + ], + "emits": [ + "old-name-closed", + "direct-writes-closed" + ], + "config": { + "callers": { + "type": "string", + "trusted": true, + "description": "per verb, the modules that may call it (JSON object of verb to modules; operator for the operator's session); empty is design 51's list" + } + }, + "requires": [ + { + "name": "incoming", + "storage": "object-store", + "template": "objects", + "options": { + "max-mib": 8192 + } + }, + { + "name": "transfers", + "storage": "key-value", + "template": "state", + "options": { + "ttl-seconds": 86400 + } + }, + { + "name": "holds", + "storage": "key-value", + "template": "state" + }, + { + "name": "windows", + "storage": "key-value", + "template": "state" + }, + { + "name": "mirrors", + "storage": "key-value", + "template": "state" + }, + { + "name": "findings", + "storage": "key-value", + "template": "state" + }, + { + "name": "events", + "storage": "stream", + "template": "events", + "options": { + "max-age-days": 7, + "max-mib": 16 + } + } + ], "capabilities": [ "container-runtime" ], "own-secrets": { "broker": "/var/lib/mesh/registry/broker" }, + "secrets-owner": "distribution", "serves": { "artifact-store": { "port": 5000 @@ -33,7 +136,7 @@ "port": 5000, "protocol": "tcp", "from": "mesh", - "why": "every machine pulls images and artifacts from here" + "why": "every machine pulls images and artifacts from here; distribution's program serves it in front of the store" } ], "data": { @@ -49,19 +152,28 @@ ] }, "resources": [ + { + "id": "account", + "type": "user", + "name": "distribution", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh/registry" + }, { "id": "state", "type": "directory", "path": "/var/lib/mesh/registry", "mode": "0700", - "unknown": "program" + "unknown": "program", + "owner": "distribution" }, { "id": "registry-data", "type": "directory", "path": "/var/lib/mesh-registry", "mode": "0700", - "unknown": "program" + "unknown": "program", + "owner": "distribution" }, { "id": "store", @@ -69,7 +181,7 @@ "name": "mesh-registry", "image": "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373", "ports": [ - "5000:5000" + "127.0.0.1:5001:5000" ], "volumes": [ "/var/lib/mesh-registry:/var/lib/registry" @@ -78,6 +190,29 @@ "REGISTRY_STORAGE_DELETE_ENABLED": "true" } }, + { + "id": "program", + "type": "process", + "name": "distribution", + "artifact": "program", + "run": [ + "./distribution" + ], + "env": { + "MESH_BROKER_FILE": "/var/lib/mesh/registry/broker", + "MESH_REGISTRY_LOOPBACK": "http://127.0.0.1:5001", + "MESH_REGISTRY_LISTEN": ":5000" + }, + "user": "distribution", + "health": { + "kind": "tool", + "tool": "registry_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "90s" + } + }, { "id": "collect", "type": "container", @@ -95,5 +230,28 @@ "store" ] } - ] + ], + "build": { + "artifacts": [ + { + "name": "program", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/distribution", + "binary": "distribution" + }, + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/distribution-tools", + "binary": "distribution-tools", + "loads": [ + "distribution-tools" + ] + } + ] + } } diff --git a/modules/gitea/README.md b/modules/gitea/README.md index c6fe87c5..5b0f8934 100644 --- a/modules/gitea/README.md +++ b/modules/gitea/README.md @@ -1,8 +1,8 @@ # gitea The forge, and the mesh's npm package registry: it claims the `git` and `npm-package-registry` seats. -Its tools, its events and the provisioner of registry accounts are the TypeScript bundle (`index.ts`, -`tools/`, `provisioner/`). This README covers the two Go bundles beside it. +Its tools and its events are the TypeScript bundle (`index.ts`, +`tools/`). This README covers the two Go bundles beside it. **What a registry consumer is told** (`serves.npm-package-registry`): where the registry answers on the private network (`scheme`, `port`, `npm-path`), and its `public-root`, the address the forge writes every @@ -11,6 +11,11 @@ tarball at: `https://` and the forge's route's composed public name, under the n locked tarball only from the registry it is configured with, so a consumer that installs from lock files trusts that root. On a machine with no public domain the key is left out, and the plan says so. +**Both package provisions carry no credential** (`credential.form` none, design 51 *Publishing*): reads of the +organisation's packages are anonymous, and every write goes through the seats' verbs, so no consumer is given an +account of the forge. The registry accounts the TypeScript pair provisioner made before stay on the forge until +removed by hand; nothing hands their passwords out any more. + ## Announcing pull requests to the merge check (novox/hq to-be 45 §9, issue 347) Every new head of an open pull request is announced as `pull.updated`, once, and the controller has it checked. @@ -91,7 +96,7 @@ is read once and searched in memory; a binary file is passed over, a file over 4 `skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. A search that reaches the call's deadline stops there and answers what it found, with `stopped_at_deadline`. -It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), as the +It reaches the forge through the same HTTP client as the package-registry bundle (`internal/forgehttp`), as the tools' own account (novox/hq ADR 0365 §2, issue 469): the token the TypeScript bundle mints for `mesh-tools` and keeps (`MESH_GITEA_TOKEN_FILE`), sent only when its scopes line names that account, never the admin login. One tool call takes at most 50 seconds. @@ -181,16 +186,28 @@ No tool creates in the organisation: `gitea_create_repo` creates under the tools keeps `can_create_org_repo` false and its token has no `write:organization` (ADR 0428 §8). `go test ./cmd/repositories` runs it against a fake forge. +## The two package seats (novox/hq design 51, ADR 0605, ADR 0611) + +The forge module holds `npm-package-registry` and `go-package-registry`, and the Go bundle `cmd/package-registry` +serves both seats' verbs as tools `.`: `publish` and `publish-part` (a package's one file in chunks into +the module's own object store `incoming`, its transfers in the state `transfers`; mesh-sdk's seat-neutral receiving +side), `versions` and `remove` (a tagged version, by the builder). Before anything is uploaded the holder asks the +builder whether it numbered the version with that digest; a package goes up as npm's publish sends it, a Go module's +zip to the Go upload. Every outcome is said on the seat's events (`published`, `refused`, `removed`), a Go package's +with its module path. Who may call each verb is the trusted config value `callers` (empty: design 51's list). + ## The package registry's retention (novox/hq ADR 0251 §4, to-be 51) Every publish adds a version to the registry, and until this nothing removed one. The Go bundle -`cmd/npm-registry` says which versions each package holds, what keeps each one, and — asked with a why — +`cmd/package-registry` says which versions each package holds, what keeps each one, and — asked with a why — deletes the versions nothing keeps. | tool | | what | |---|---|---| | `npm_packages` | r | every package of the registry's owner, each version newest first with its publish time, its size and why it is kept | | `npm_retention` | a | the versions retention would delete, and the bytes. **A dry run unless `dry_run` is false**; a real run needs `why` | +| `go_packages` | r | every Go module of the registry's owner (the Go seat's), each version newest first with its publish time, its size and why it is kept: a `go.mod` on a default branch requires it, or it is among the newest `keep` | +| `go_retention` | a | the Go module versions retention would delete, and the bytes; as `npm_retention`, and a `go.mod` that could not be read stops a real run. Each deletion says `removed` on the Go seat | **A version is kept when:** @@ -223,11 +240,18 @@ version already gone counts as deleted. ### How it reaches the forge -Over the forge's HTTP interface on the machine (`MESH_GITEA_URL`), as the admin account the vault -delivered (`MESH_GITEA_ADMIN_USER`, the password in `MESH_GITEA_ADMIN_PASSWORD_FILE` — the same file -the TypeScript bundle mints its token with), by basic authentication: this bundle mints no token and -keeps nothing. The password is read per call and never logged, answered or put in an error. The -registry's owner is `MESH_NPM_OWNER`, the owner in the seat's `npm-path`. +Over the forge's HTTP interface on the machine (`MESH_GITEA_URL`), never as the admin account (design 51: each +holder keeps its write side as an own secret beside a read-only tool token). It **reads** as the tools' account, +with the token the TypeScript bundle keeps for it (`MESH_GITEA_TOKEN_FILE`, scopes including `read:package`), and +**writes** (a publish, a remove, retention's deletion) as the packages account `mesh-packages` +(`MESH_GITEA_PACKAGES_USER`), with its own token (`MESH_GITEA_PACKAGES_TOKEN_FILE`, the one scope `write:package`). +That account is plain, a member of the team `mesh-packages` that writes the organisation's packages and nothing else; +the watcher makes it with the administrator's login and mints and keeps its token at its start (`packages-account.ts`). +No timer looks at it after that: when the forge refuses a write (401) this bundle raises the finding +`packages-token-refused` (key `seat.gitea.packages-token-refused_write` in the module's `findings` state), the watcher, +watching that key, mints the token afresh and clears it; a write the forge takes clears it too. This bundle mints nothing; a token is +read per call and never logged, answered or put in an error. The registry's owner is `MESH_NPM_OWNER`, the owner in +the seat's `npm-path`. Calls to the forge run eight at a time, and one tool call reads for at most 50 seconds, inside a module's 60-second ask. @@ -347,5 +371,4 @@ Each directory or archive this module declares says how it treats a file it did - `${dir:mesh-state}/state`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `/var/lib/mesh/forge-patches`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). -- `${dir:grants}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `${dir:data}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index 5f80c6de..8a8079a8 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -207,7 +207,8 @@ export class GiteaClient { /** * Build from the module's resolved environment. The URL comes from MESH_GITEA_URL (the mesh's own * name), falling back to the bare GITEA_URL and to the forge's loopback port. The token, in order: - * one configured in the environment (MESH_GITEA_TOKEN / GITEA_TOKEN), which wins — never a setting, which is + * one configured in the environment for a hand-run instance (GITEA_TOKEN; nothing on the mesh sets it, so no + * MESH_* word names it: checks/environment), which wins — never a setting, which is * ignored and said (hq issue 342); else * one the module mints for itself with the admin account the vault delivered and keeps in its own * state (token.ts; hq issue 100). Throws only when neither is possible, naming what is missing, @@ -217,7 +218,7 @@ export class GiteaClient { const cfg = meshConfig(env.MESH_GITEA_CONFIG_FILE); const url = cfg.url ?? env.MESH_GITEA_URL ?? env.GITEA_URL ?? `http://127.0.0.1:${env.GITEA_PORT ?? "3000"}`; ignoreSecretSetting(cfg, "token"); - const configured = env.MESH_GITEA_TOKEN ?? env.GITEA_TOKEN; + const configured = env.GITEA_TOKEN; if (configured) return new GiteaClient(url, new ConfiguredToken(configured)); return new GiteaClient(url, MintedToken.fromEnv(url, env)); } diff --git a/modules/gitea/cmd/agent-token/main.go b/modules/gitea/cmd/agent-token/main.go index 0303b670..c755d5a2 100644 --- a/modules/gitea/cmd/agent-token/main.go +++ b/modules/gitea/cmd/agent-token/main.go @@ -17,7 +17,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" ) -// ToolNames are the tools this bundle serves. The module's manifest lists no `tools` (see the npm-registry bundle +// ToolNames are the tools this bundle serves. The module's manifest lists no `tools` (see the package-registry bundle // for why); the README names this one, and a test holds the three together. var ToolNames = []string{ToolName} diff --git a/modules/gitea/cmd/forge/forge.go b/modules/gitea/cmd/forge/forge.go index a28a915b..a9d27930 100644 --- a/modules/gitea/cmd/forge/forge.go +++ b/modules/gitea/cmd/forge/forge.go @@ -12,7 +12,7 @@ import ( ) // The forge as this bundle reaches it: its own HTTP interface on the machine, as the tools' own account (novox/hq -// ADR 0365 §2; internal/forgehttp, shared with the npm-registry bundle beside it). +// ADR 0365 §2; internal/forgehttp, shared with the package-registry bundle beside it). // Forge is the forge's HTTP interface. type Forge struct { diff --git a/modules/gitea/cmd/forge/forge_test.go b/modules/gitea/cmd/forge/forge_test.go index 01883b7c..42733087 100644 --- a/modules/gitea/cmd/forge/forge_test.go +++ b/modules/gitea/cmd/forge/forge_test.go @@ -648,7 +648,13 @@ func TestToolsAreTheOnesTheReadmeNamesAndTheManifestBuilds(t *testing.T) { t.Fatal(err) } var m struct { - Tools []string `json:"tools"` + Tools []string `json:"tools"` + Serves map[string]map[string]any `json:"serves"` + Provides []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Credential map[string]string `json:"credential"` + } `json:"provides"` Build struct { Artifacts []struct { From string `json:"from"` @@ -664,28 +670,52 @@ func TestToolsAreTheOnesTheReadmeNamesAndTheManifestBuilds(t *testing.T) { if len(m.Tools) != 0 { t.Error("the manifest lists tools: its claims would offer them as both seats' verbs") } - found := false + // CAT-G2b (ADR 0346 §6, the operator 2026-11-04): the forge keeps serving the npm registry's public root, so a lock + // file naming it installs as today. CAT-G2 (design 51, E2): the Go seat's read side is the provision + // go-package-registry, mesh scope, read with no credential, offered only by this build. + if root, _ := m.Serves["npm-package-registry"]["public-root"].(string); !strings.HasPrefix(root, "https://") || !strings.HasSuffix(root, "/api/packages/novox/npm/") { + t.Errorf("npm-package-registry serves public-root %q: the public root must stay", root) + } + if g := m.Serves["go-package-registry"]; g["go-path"] != "/api/packages/novox/go" { + t.Errorf("go-package-registry serves %v, want the forge's Go proxy path", g) + } + goProvided := false + for _, p := range m.Provides { + if p.Name == "go-package-registry" { + goProvided = p.Scope == "mesh" && p.Credential["form"] == "none" + } + } + if !goProvided { + t.Error("go-package-registry is not provided at mesh scope with no credential") + } + // The tools' own account, never the admin login (novox/hq ADR 0365 §2); the package registry's bundle also writes + // as the packages account (design 51: the write side an own secret beside a read-only tool token). + wants := map[string][]string{ + "cmd/forge": {"MESH_GITEA_URL", "MESH_GITEA_TOOLS_USER", "MESH_GITEA_TOKEN_FILE"}, + "cmd/package-registry": {"MESH_GITEA_URL", "MESH_GITEA_TOOLS_USER", "MESH_GITEA_TOKEN_FILE", "MESH_GITEA_PACKAGES_USER", "MESH_GITEA_PACKAGES_TOKEN_FILE"}, + } for _, a := range m.Build.Artifacts { - if a.From == "cmd/forge" { - found = true - if len(a.Loads) != 1 || a.Loads[0] != a.Binary { - t.Errorf("the bundle loads %v, not its binary %s", a.Loads, a.Binary) + want, ok := wants[a.From] + if !ok { + continue + } + delete(wants, a.From) + if len(a.Loads) != 1 || a.Loads[0] != a.Binary { + t.Errorf("%s loads %v, not its binary %s", a.From, a.Loads, a.Binary) + } + for _, k := range want { + if a.Env[k] == "" { + t.Errorf("%s is not given %s", a.From, k) } - // The tools' own account, never the admin login (novox/hq ADR 0365 §2). - for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_TOOLS_USER", "MESH_GITEA_TOKEN_FILE"} { - if a.Env[k] == "" { - t.Errorf("the bundle is not given %s", k) - } - } - for _, k := range []string{"MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} { - if a.Env[k] != "" { - t.Errorf("the bundle is given %s: it acts as the tools' account, not the administrator", k) - } + } + for _, k := range []string{"MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} { + if a.Env[k] != "" { + t.Errorf("%s is given %s: it acts as an account of its own, not the administrator", a.From, k) } } } - if !found { - t.Error("the manifest builds no cmd/forge bundle") + for from := range wants { + t.Errorf("the manifest builds no %s bundle", from) } } diff --git a/modules/gitea/cmd/forge/main.go b/modules/gitea/cmd/forge/main.go index 3be08aa8..89900a64 100644 --- a/modules/gitea/cmd/forge/main.go +++ b/modules/gitea/cmd/forge/main.go @@ -1,6 +1,6 @@ // forge: the forge module's Go bundle for an agent's commit and the reads around it (novox/hq issue 375). // A process the node's runtime launches and speaks MCP over stdio to, beside the module's TypeScript bundle -// and its npm-registry bundle. gitea_commit puts one commit on a branch of its own — never the default branch, +// and its package-registry bundle. gitea_commit puts one commit on a branch of its own — never the default branch, // never one a protection rule names (novox/hq ADR 0266) — and the reads answer a commit history, a compare, a // pull request's reviews with their line comments, a directory's entries and a code search, summarised. // stdout is the protocol; it says what it says on stderr, and never the forge's credential. @@ -16,7 +16,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" ) -// ToolNames are the tools this bundle serves. The module's manifest lists no `tools` (see the npm-registry +// ToolNames are the tools this bundle serves. The module's manifest lists no `tools` (see the package-registry // bundle for why); the README names these, and a test holds the three together. var ToolNames = []string{"gitea_commit", "gitea_commits", "gitea_compare", "gitea_pull_reviews", "gitea_list_dir", "gitea_search_code"} diff --git a/modules/gitea/cmd/npm-registry/forge.go b/modules/gitea/cmd/package-registry/forge.go similarity index 59% rename from modules/gitea/cmd/npm-registry/forge.go rename to modules/gitea/cmd/package-registry/forge.go index c90daf9e..f752a411 100644 --- a/modules/gitea/cmd/npm-registry/forge.go +++ b/modules/gitea/cmd/package-registry/forge.go @@ -13,20 +13,81 @@ import ( "gitea/internal/forgehttp" ) -// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin account -// (internal/forgehttp, shared with the forge bundle beside it). Basic authentication, so this bundle mints -// nothing and keeps nothing: the TypeScript bundle's token is its own. +// The forge as this bundle reaches it: its own HTTP interface on the machine (internal/forgehttp, shared with the +// forge bundle beside it), never as the admin account. It reads as the tools' account, whose token carries +// read:package, and writes (a publish, a remove, retention's deletion) as the packages account, whose token carries +// write:package (design 51: each holder keeps its write side as an own secret beside a read-only tool token). The +// TypeScript bundle mints and keeps both tokens (token.ts, packages-account.ts); this bundle mints nothing. // Forge is the forge's HTTP interface. type Forge struct { - forgehttp.Client - Owner string // the package registry's owner, as the seat serves it + forgehttp.Client // reads, as the tools' account + // Writer writes into the package registry, as the packages account; nil writes nothing. + Writer *forgehttp.Client + Owner string // the package registry's owner, as the seat serves it + // Type is the registry's package type retention reads: TypeNpm when empty, or TypeGo. + Type string + // Refused says whether the forge refused the write side's token (401), as the finding PackagesTokenRefused in the + // module's findings state: the TypeScript bundle mints a new token on that key's event and clears it (no timer + // looks at the token). Nil says nothing. + Refused func(raised bool) +} + +// PackagesTokenRefused is the finding this bundle raises when the forge refuses the packages account's token, and its +// key in the module's findings state (packages-account.ts PACKAGES_TOKEN_REFUSED watches that key). +const ( + PackagesTokenRefused = "packages-token-refused" + PackagesTokenRefusedKey = "seat.gitea.packages-token-refused_write" +) + +// wrote is a write's answer, said on Refused: a 401 raises the finding, a write the forge took clears it. +func (f *Forge) wrote(err error) error { + if f.Refused == nil { + return err + } + var status *forgehttp.StatusError + switch { + case errors.As(err, &status) && status.Status == http.StatusUnauthorized: + f.Refused(true) + case err == nil: + f.Refused(false) + } + return err +} + +// The package types the retention reads: npm packages (the npm seat) and Go modules (the Go seat). +const ( + TypeNpm = "npm" + TypeGo = "go" +) + +// typ is the package type this forge's retention reads. +func (f *Forge) typ() string { + if f.Type == "" { + return TypeNpm + } + return f.Type +} + +// As is the same forge, its retention reading the package type typ. +func (f *Forge) As(typ string) *Forge { + out := *f + out.Type = typ + return &out } // ForgeFromEnv is the forge as the manifest hands it to this bundle. func ForgeFromEnv() (*Forge, error) { c, missing := forgehttp.FromEnv() - f := &Forge{Client: c, Owner: os.Getenv("MESH_NPM_OWNER")} + w, writeMissing := forgehttp.WriterFromEnv() + for _, m := range writeMissing { + if m != "MESH_GITEA_URL" { + missing = append(missing, m) + } + } + f := &Forge{Client: c, Writer: &w, Owner: os.Getenv("MESH_NPM_OWNER"), Refused: func(raised bool) { + finding(PackagesTokenRefused, "write", "the forge refused the package seats' write token; the module mints it afresh", raised) + }} if f.Owner == "" { missing = append(missing, "MESH_NPM_OWNER") } @@ -56,12 +117,12 @@ type PackageVersion struct { const pageSize = 50 -// Packages is every npm package version of the owner, all pages. +// Packages is every package version of the owner of this forge's type, all pages. func (f *Forge) Packages(ctx context.Context) ([]PackageVersion, error) { var all []PackageVersion for page := 1; page <= 1000; page++ { var batch []PackageVersion - path := fmt.Sprintf("/api/v1/packages/%s?type=npm&page=%d&limit=%d", url.PathEscape(f.Owner), page, pageSize) + path := fmt.Sprintf("/api/v1/packages/%s?type=%s&page=%d&limit=%d", url.PathEscape(f.Owner), f.typ(), page, pageSize) if err := f.GetJSON(ctx, path, &batch); err != nil { return nil, err } @@ -74,7 +135,7 @@ func (f *Forge) Packages(ctx context.Context) ([]PackageVersion, error) { } func (f *Forge) versionPath(name, version string) string { - return fmt.Sprintf("/api/v1/packages/%s/npm/%s/%s", url.PathEscape(f.Owner), url.PathEscape(name), url.PathEscape(version)) + return f.packagePath(f.typ(), name, version) } // Size is the bytes of one version's files. @@ -104,10 +165,22 @@ func (f *Forge) DistTags(ctx context.Context, name string) (map[string]string, e return meta.DistTags, nil } -// Delete removes one version from the registry. +// write is the registry's write side, refused when the manifest handed none. +func (f *Forge) write() (*forgehttp.Client, error) { + if f.Writer == nil { + return nil, errors.New("the package registry's write side is not handed to this bundle (MESH_GITEA_PACKAGES_TOKEN_FILE); nothing was written") + } + return f.Writer, nil +} + +// Delete removes one version from the registry, as the packages account. func (f *Forge) Delete(ctx context.Context, name, version string) error { - _, err := f.do(ctx, http.MethodDelete, f.versionPath(name, version)) - return err + w, err := f.write() + if err != nil { + return err + } + _, err = w.Do(ctx, http.MethodDelete, f.versionPath(name, version), nil, maxBody) + return f.wrote(err) } // Repository is one repository on the forge, as far as reading its manifests needs. diff --git a/modules/gitea/cmd/npm-registry/main.go b/modules/gitea/cmd/package-registry/main.go similarity index 54% rename from modules/gitea/cmd/npm-registry/main.go rename to modules/gitea/cmd/package-registry/main.go index e964d0c6..9f1ca0fc 100644 --- a/modules/gitea/cmd/npm-registry/main.go +++ b/modules/gitea/cmd/package-registry/main.go @@ -1,4 +1,5 @@ -// npm-registry: the forge module's Go bundle for its package registry (novox/hq ADR 0251 §4, to-be 51). +// package-registry: the forge module's Go bundle for its package registry (novox/hq ADR 0251 §4, design 51): its +// retention tools, and the verbs of the two package seats it holds (seats.go). // A process the node's runtime launches and speaks MCP over stdio to, beside the module's TypeScript // bundle. It says which versions of each npm package the registry holds, what keeps each one, and — // asked with a why — deletes the versions nothing keeps. stdout is the protocol; it says what it says on @@ -13,15 +14,19 @@ import ( "strings" stdio "git.novox.be/novox/mesh-sdk/go" + + "gitea/internal/packageseats" ) // ToolNames are the tools this bundle serves. The module's manifest lists no `tools`: its TypeScript // bundle's tools are served without one, and a claim without `serves` would offer a `tools` list to both // of the module's seats as their verbs. The README names these two, and a test holds the two together. -var ToolNames = []string{"npm_packages", "npm_retention"} +var ToolNames = []string{"go_packages", "go_retention", "npm_packages", "npm_retention"} func main() { - if err := stdio.Serve("", Tools(func() (*Forge, error) { return ForgeFromEnv() })); err != nil { + forge := func() (*Forge, error) { return ForgeFromEnv() } + tools := append(Tools(forge), SeatTools(func() (*packageseats.Holder, error) { return busHolder(forge) })...) + if err := stdio.Serve("", tools); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } @@ -30,17 +35,45 @@ func main() { // Tools are the bundle's tools over the forge the function gives; asked per call, so a manifest that // does not set the environment is a refusal naming what is missing, not a bundle that will not start. func Tools(forge func() (*Forge, error)) []stdio.Tool { + return append(typeTools(TypeGo, forge), typeTools(TypeNpm, forge)...) +} + +// typeTools are one package type's two tools, _packages and _retention, over the forge reading that type. +func typeTools(typ string, forgeOf func() (*Forge, error)) []stdio.Tool { + forge := func() (*Forge, error) { + f, err := forgeOf() + if err != nil { + return nil, err + } + return f.As(typ), nil + } pkg := map[string]any{"type": "string", "description": "one package, by its name (default every package)"} keep := map[string]any{"type": "integer", "description": fmt.Sprintf("how many of each package's newest versions are kept whatever names them (default %d, at least 1)", DefaultKeep)} + listed := "Every npm package in the forge's package registry, each version newest first with when it was published, " + + "its size and what keeps it: a lockfile on a repository's default branch that names it, the highest version " + + "satisfying a range a package.json there names, a dist-tag, or being among the newest `keep` of its package " + + "(novox/hq ADR 0251). Repositories and files that could not be read are named. Changes nothing. Replaces " + + "npm view and npm dist-tag ls. (r)" + deletes := "What retention would delete from the forge's package registry: every version nothing keeps — no lockfile " + + "on a repository's default branch names it, it is not the highest version satisfying any range a package.json " + + "there names, no dist-tag names it, and it is not among the newest `keep` of its package — with the bytes. " + + "A dry run unless dry_run is false; a real run needs why, and deletes nothing at all when any repository or " + + "file could not be read (novox/hq ADR 0251). Each deletion says removed on the npm seat. Replaces npm unpublish. (a)" + if typ == TypeGo { + listed = "Every Go module in the forge's package registry (the Go seat's), each version newest first with when it " + + "was published, its size and what keeps it: a go.mod on a repository's default branch that requires it, or " + + "being among the newest `keep` of its module (design 51 *Retention*). Repositories and go.mod files that " + + "could not be read are named. Changes nothing. Replaces go list -m -versions. (r)" + deletes = "What retention would delete from the forge's Go registry: every module version no go.mod on a " + + "repository's default branch requires and not among the newest `keep` of its module, with the bytes. A dry " + + "run unless dry_run is false; a real run needs why, and deletes nothing at all when any repository or go.mod " + + "could not be read. Each deletion says removed on the Go seat. (a)" + } return []stdio.Tool{ { - Name: "npm_packages", - Description: "Every npm package in the forge's package registry, each version newest first with when it was published, " + - "its size and what keeps it: a lockfile on a repository's default branch that names it, the highest version " + - "satisfying a range a package.json there names, a dist-tag, or being among the newest `keep` of its package " + - "(novox/hq ADR 0251). Repositories and files that could not be read are named. Changes nothing. Replaces " + - "npm view and npm dist-tag ls. (r)", - Input: map[string]any{"package": pkg, "keep": keep}, + Name: typ + "_packages", + Description: listed, + Input: map[string]any{"package": pkg, "keep": keep}, Run: func(args map[string]any) (any, error) { f, err := forge() if err != nil { @@ -70,12 +103,8 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool { }, }, { - Name: "npm_retention", - Description: "What retention would delete from the forge's package registry: every version nothing keeps — no lockfile " + - "on a repository's default branch names it, it is not the highest version satisfying any range a package.json " + - "there names, no dist-tag names it, and it is not among the newest `keep` of its package — with the bytes. " + - "A dry run unless dry_run is false; a real run needs why, and deletes nothing at all when any repository or " + - "file could not be read (novox/hq ADR 0251). Replaces npm unpublish. (a)", + Name: typ + "_retention", + Description: deletes, Input: map[string]any{ "package": pkg, "keep": keep, diff --git a/modules/gitea/cmd/npm-registry/refs.go b/modules/gitea/cmd/package-registry/refs.go similarity index 82% rename from modules/gitea/cmd/npm-registry/refs.go rename to modules/gitea/cmd/package-registry/refs.go index 55fcfa45..744d9120 100644 --- a/modules/gitea/cmd/npm-registry/refs.go +++ b/modules/gitea/cmd/package-registry/refs.go @@ -7,6 +7,7 @@ import ( "fmt" "path" "regexp" + "slices" "sort" "strings" ) @@ -36,18 +37,77 @@ var ( "pnpm-lock.yaml": true} ) -// interesting is whether a path is a file this reads; nothing under node_modules, which is what a -// lockfile already says and is not the repository's own. -func interesting(p string) bool { +// goModName is the file a Go module's requirements are read from. +const goModName = "go.mod" + +// interesting is whether a path is a file this reads for packages of the type typ: an npm manifest or lockfile, or +// a go.mod; nothing under node_modules or vendor, which is what a lockfile or go.mod already says and is not the +// repository's own. +func interesting(p, typ string) bool { for _, part := range strings.Split(p, "/") { - if part == "node_modules" { + if part == "node_modules" || part == "vendor" { return false } } base := path.Base(p) + if typ == TypeGo { + return base == goModName + } return manifestNames[base] || lockNames[base] } +// ReadGoMod is every registry module version a go.mod requires, or names on the right of a replace (design 51 +// *Retention*: a Go version is referenced when a go.mod on a repository's default branch requires it). A line it +// cannot read, or a block left open, is an error beside what was read: the file is then not read whole, and a real +// run stops before anything is deleted. +func ReadGoMod(content []byte, where string, known map[string]bool) ([]Pinned, error) { + var out []Pinned + pin := func(module, version string) { + if known[module] { + out = append(out, Pinned{Package: module, Version: version, Where: where}) + } + } + block := "" + for n, line := range strings.Split(string(content), "\n") { + if i := strings.Index(line, "//"); i >= 0 { + line = line[:i] + } + f := strings.Fields(line) + if len(f) == 0 { + continue + } + if block != "" { + if f[0] == ")" { + block = "" + continue + } + f = append([]string{block}, f...) + } else if len(f) == 2 && f[1] == "(" { + block = f[0] + continue + } + switch f[0] { + case "require": + if len(f) != 3 { + return out, fmt.Errorf("line %d: a require is , not %q", n+1, strings.Join(f[1:], " ")) + } + pin(f[1], f[2]) + case "replace": + arrow := slices.Index(f, "=>") + if arrow < 0 { + return out, fmt.Errorf("line %d: a replace without =>", n+1) + } + if right := f[arrow+1:]; len(right) == 2 { + pin(right[0], right[1]) + } + } + } + if block != "" { + return out, fmt.Errorf("its %s block is never closed", block) + } + return out, nil +} + var dependencyFields = []string{"dependencies", "devDependencies", "peerDependencies", "optionalDependencies"} // ReadManifest is every range a package.json names for a registry package. An `npm:` alias names the diff --git a/modules/gitea/cmd/npm-registry/refs_test.go b/modules/gitea/cmd/package-registry/refs_test.go similarity index 99% rename from modules/gitea/cmd/npm-registry/refs_test.go rename to modules/gitea/cmd/package-registry/refs_test.go index 430e3094..d6d3c897 100644 --- a/modules/gitea/cmd/npm-registry/refs_test.go +++ b/modules/gitea/cmd/package-registry/refs_test.go @@ -85,7 +85,7 @@ func TestAPnpmLockNamesWhatItPinsInEachFormat(t *testing.T) { func TestOnlyTheRepositorysOwnManifestsAreRead(t *testing.T) { for p, want := range map[string]bool{"package.json": true, "modules/x/package-lock.json": true, "yarn.lock": true, "pnpm-lock.yaml": true, "node_modules/a/package.json": false, "README.md": false, "a/package.json.bak": false} { - if interesting(p) != want { + if interesting(p, TypeNpm) != want { t.Errorf("%s: %v", p, !want) } } diff --git a/modules/gitea/cmd/npm-registry/retention.go b/modules/gitea/cmd/package-registry/retention.go similarity index 90% rename from modules/gitea/cmd/npm-registry/retention.go rename to modules/gitea/cmd/package-registry/retention.go index 4f9915db..d0a16205 100644 --- a/modules/gitea/cmd/npm-registry/retention.go +++ b/modules/gitea/cmd/package-registry/retention.go @@ -9,6 +9,8 @@ import ( "strings" "sync" "time" + + "gitea/internal/packageseats" ) // Retention for the package registry (novox/hq ADR 0251 §4, to-be 51): a version is kept when a @@ -48,6 +50,7 @@ type Package struct { // Survey is the registry and what the forge's repositories say of it. type Survey struct { Owner string `json:"owner"` + Type string `json:"type"` Keep int `json:"keep"` Repositories int `json:"repositories_read"` Unread []string `json:"repositories_unread,omitempty"` @@ -71,7 +74,7 @@ func survey(ctx context.Context, f *Forge, only string, keep int, sizes func(Ite } if only != "" { if _, has := byName[only]; !has { - return nil, fmt.Errorf("the registry of %s holds no npm package %q", f.Owner, only) + return nil, fmt.Errorf("the registry of %s holds no %s package %q", f.Owner, f.typ(), only) } } known := map[string]bool{} @@ -79,7 +82,7 @@ func survey(ctx context.Context, f *Forge, only string, keep int, sizes func(Ite known[name] = true } - s := &Survey{Owner: f.Owner, Keep: keep, Said: []string{}} + s := &Survey{Owner: f.Owner, Type: f.typ(), Keep: keep, Said: []string{}} wanted, pinned, err := readRepositories(ctx, f, known, s) if err != nil { return nil, err @@ -92,7 +95,10 @@ func survey(ctx context.Context, f *Forge, only string, keep int, sizes func(Ite } } sort.Strings(names) - tags := distTags(ctx, f, names) + tags := map[string]tagAnswer{} // a Go module has no dist-tags + if f.typ() == TypeNpm { + tags = distTags(ctx, f, names) + } for _, name := range names { s.Packages = append(s.Packages, decide(name, byName[name], tags[name], wanted[name], pinned[name], keep)) } @@ -167,7 +173,7 @@ func readRepositories(ctx context.Context, f *Forge, known map[string]bool, s *S var ps []Pinned var notRead, unread []string for _, p := range files { - if !interesting(p) { + if !interesting(p, f.typ()) { continue } where := r.FullName + ":" + p @@ -177,6 +183,14 @@ func readRepositories(ctx context.Context, f *Forge, known map[string]bool, s *S continue } base := path.Base(p) + if base == goModName { + got, err := ReadGoMod(content, where, known) + ps = append(ps, got...) + if err != nil { + notRead = append(notRead, fmt.Sprintf("%s: not readable (%v)", where, err)) + } + continue + } if manifestNames[base] { got, err := ReadManifest(content, where, known) if err != nil { @@ -418,12 +432,29 @@ func retention(ctx context.Context, f *Forge, only string, keep int, dryRun bool } del, cancel := context.WithTimeout(context.Background(), budget) defer cancel() + store := seatStore{forge: func() (*Forge, error) { return f, nil }} + seat := packageseats.NpmSeat + if f.typ() == TypeGo { + seat = packageseats.GoSeat + } for _, c := range p.Candidates { + digest, _, _ := store.Held(seat, c.Package, c.Version) if err := f.Delete(del, c.Package, c.Version); err != nil && !errors.Is(err, errNotFound) { p.Refused = append(p.Refused, fmt.Sprintf("%s@%s: %v", c.Package, c.Version, err)) continue } p.Deleted = append(p.Deleted, c.Package+"@"+c.Version) + // The seat says every removal (design 51 *Retention*), so the builder's entry turns removed. + // The seat's version is the builder's, without Go's leading v (forgeVersion adds it for the forge). + o := packageseats.Outcome{Name: c.Package, Version: strings.TrimPrefix(c.Version, "v"), Kind: packageseats.KindPackage, Digest: digest, Reason: packageseats.ReasonRetention} + if seat == packageseats.NpmSeat { + o.Version = c.Version + } else { + o.Module = c.Package + } + if err := say(seat, packageseats.EventRemoved, o); err != nil { + s.Said = append(s.Said, fmt.Sprintf("%s@%s is deleted but its removed could not be said: %v", c.Package, c.Version, err)) + } } s.Said = append(s.Said, fmt.Sprintf("deleted %d, refused %d, because: %s", len(p.Deleted), len(p.Refused), p.Why)) return p, nil diff --git a/modules/gitea/cmd/npm-registry/retention_test.go b/modules/gitea/cmd/package-registry/retention_test.go similarity index 77% rename from modules/gitea/cmd/npm-registry/retention_test.go rename to modules/gitea/cmd/package-registry/retention_test.go index 03917f59..b70c643c 100644 --- a/modules/gitea/cmd/npm-registry/retention_test.go +++ b/modules/gitea/cmd/package-registry/retention_test.go @@ -17,12 +17,14 @@ import ( "time" "gitea/internal/forgehttp" + "gitea/internal/packageseats" ) // fakeForge answers the forge's interface from maps, and records every delete. type fakeForge struct { t *testing.T versions map[string][]string // package -> versions + modules map[string][]string // Go module -> versions, listed as type go tags map[string]map[string]string // package -> dist-tags repos map[string]map[string]string // repository -> path -> content broken map[string]bool // repositories whose tree answers 500 @@ -40,11 +42,12 @@ func (f *fakeForge) serve() *Forge { if err := os.WriteFile(file, []byte(f.password+"\n"), 0o600); err != nil { f.t.Fatal(err) } - return &Forge{Client: forgehttp.Client{URL: srv.URL, User: "admin", PasswordFile: file}, Owner: "acme"} + c := forgehttp.Client{URL: srv.URL, User: "admin", PasswordFile: file} + return &Forge{Client: c, Writer: &c, Owner: "acme"} } var ( - versionFiles = regexp.MustCompile(`^/api/v1/packages/acme/npm/([^/]+)/([^/]+)(/files)?$`) + versionFiles = regexp.MustCompile(`^/api/v1/packages/acme/(?:npm|go)/([^/]+)/([^/]+)(/files)?$`) treePath = regexp.MustCompile(`^/api/v1/repos/([^/]+)/([^/]+)/git/trees/`) rawPath = regexp.MustCompile(`^/api/v1/repos/([^/]+)/([^/]+)/raw/(.+)$`) ) @@ -62,10 +65,14 @@ func (f *fakeForge) handle(w http.ResponseWriter, r *http.Request) { switch { case p == "/api/v1/packages/acme" && r.Method == http.MethodGet: var all []map[string]any - names := keys(f.versions) + listed := f.versions + if q.Get("type") == TypeGo { + listed = f.modules + } + names := keys(listed) for _, n := range names { - for i, v := range f.versions[n] { - all = append(all, map[string]any{"name": n, "version": v, "type": "npm", + for i, v := range listed[n] { + all = append(all, map[string]any{"name": n, "version": v, "type": q.Get("type"), "created_at": time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC)}) } } @@ -276,10 +283,19 @@ func TestARealRunWithoutWhyIsRefusedBeforeAnythingIsRead(t *testing.T) { func TestARealRunDeletesOnlyWhatNothingKeeps(t *testing.T) { f := aRegistry(t) + var removed []string + say = func(seat, event string, o packageseats.Outcome) error { + removed = append(removed, seat+" "+event+" "+o.Name+"@"+o.Version+" "+o.Reason) + return nil + } + t.Cleanup(func() { say = sayOn }) p, err := retention(context.Background(), f.serve(), "", 3, false, "the registry keeps what is named") if err != nil { t.Fatal(err) } + if sort.Strings(removed); len(removed) != 4 || removed[0] != "npm-package-registry removed @acme/sdk@0.1.1 retention" { + t.Errorf("said %v", removed) + } sort.Strings(f.deleted) if strings.Join(f.deleted, " ") != "@acme/sdk@0.1.1 @acme/sdk@0.1.2 @acme/sdk@0.2.0 @acme/sdk@0.2.1" { t.Fatalf("deleted %v", f.deleted) @@ -289,6 +305,48 @@ func TestARealRunDeletesOnlyWhatNothingKeeps(t *testing.T) { } } +// TestAVersionAGoModRequiresSurvivesGoRetention: the Go seat keeps what the npm seat keeps (design 51 *Retention*): a +// version a go.mod on a default branch requires, or names on the right of a replace, and the newest keep; the others are +// deleted and said removed on the Go seat, without Go's leading v; a go.mod that cannot be read stops a real run. +func TestAVersionAGoModRequiresSurvivesGoRetention(t *testing.T) { + f := aRegistry(t) + f.modules = map[string][]string{"git.acme/sdk": {"v0.1.0", "v0.1.1", "v0.1.2", "v0.1.3", "v0.1.4"}} + f.repos["acme/svc"] = map[string]string{ + "go.mod": "module git.acme/svc\n\ngo 1.26\n\nrequire (\n\tgit.acme/sdk v0.1.0 // indirect\n\tgolang.org/x/mod v0.20.0\n)\n", + "tools/go.mod": "module git.acme/svc/tools\nrequire git.acme/other v1.0.0\nreplace git.acme/x => git.acme/sdk v0.1.2\n", + "vendor/go.mod": "module broken\nrequire (\n", + } + var removed []string + say = func(seat, event string, o packageseats.Outcome) error { + removed = append(removed, seat+" "+event+" "+o.Module+"@"+o.Version+" "+o.Reason) + return nil + } + t.Cleanup(func() { say = sayOn }) + p, err := retention(context.Background(), f.serve().As(TypeGo), "", 1, false, "tidy the Go registry") + if err != nil { + t.Fatal(err) + } + sort.Strings(f.deleted) + if strings.Join(f.deleted, " ") != "git.acme/sdk@v0.1.1 git.acme/sdk@v0.1.3" { + t.Errorf("deleted %v, want all but the required v0.1.0, the replaced-to v0.1.2 and the newest v0.1.4", f.deleted) + } + if sort.Strings(removed); strings.Join(removed, ",") != "go-package-registry removed git.acme/sdk@0.1.1 retention,go-package-registry removed git.acme/sdk@0.1.3 retention" { + t.Errorf("said %v", removed) + } + if p.Type != TypeGo || len(p.Packages) != 1 { + t.Errorf("surveyed %s %v, want the Go registry alone", p.Type, p.Packages) + } + // A go.mod outside vendor that cannot be read stops the real run before anything is deleted. + f.deleted = nil + f.repos["acme/svc"]["cmd/go.mod"] = "module git.acme/svc/cmd\nrequire (\n\tgit.acme/sdk v0.1.1\n" + if _, err := retention(context.Background(), f.serve().As(TypeGo), "", 1, false, "tidy"); err == nil || !strings.Contains(err.Error(), "not read whole") { + t.Errorf("an unreadable go.mod = %v, want the real run refused", err) + } + if len(f.deleted) != 0 { + t.Errorf("deleted %v while a go.mod was unread", f.deleted) + } +} + func TestAnUnreadRepositoryStopsARealRunBeforeAnythingIsDeleted(t *testing.T) { f := aRegistry(t) f.broken = map[string]bool{"acme/site": true} @@ -349,7 +407,7 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { for _, tl := range tools { served = append(served, tl.Name) suffix := "(r)" - if tl.Name == "npm_retention" { + if strings.HasSuffix(tl.Name, "_retention") { suffix = "(a)" } if !strings.HasSuffix(tl.Description, suffix) { @@ -373,8 +431,14 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { t.Fatal(err) } var m struct { - Tools []string `json:"tools"` - Build struct { + Tools []string `json:"tools"` + Claims []struct { + Name string `json:"name"` + Serves []string `json:"serves"` + } `json:"claims"` + Requires []any `json:"requires"` + Invokes []string `json:"invokes"` + Build struct { Artifacts []struct { From string `json:"from"` Loads []string `json:"loads"` @@ -385,14 +449,41 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } + // Both package seats are claimed serving the four verbs this bundle answers as ., over its own incoming + // and transfers, asking the builder's numbering (design 51 *Publishing*). + claimed := map[string]string{} + for _, c := range m.Claims { + claimed[c.Name] = strings.Join(c.Serves, ",") + } + for _, seat := range []string{packageseats.NpmSeat, packageseats.GoSeat} { + if claimed[seat] != strings.Join(packageseats.Verbs, ",") { + t.Errorf("%s is claimed serving %q, want %v", seat, claimed[seat], packageseats.Verbs) + } + } + names := map[string]bool{} + for _, tool := range SeatTools(nil) { + names[tool.Name] = true + } + if len(names) != 8 || !names[packageseats.GoSeat+".publish-part"] { + t.Errorf("the seat tools are %v", names) + } + req, _ := json.Marshal(m.Requires) + for _, want := range []string{`"name":"incoming"`, `"name":"transfers"`, `"name":"findings"`} { + if !strings.Contains(string(req), want) { + t.Errorf("the manifest does not require %s", want) + } + } + if strings.Join(m.Invokes, ",") != "mesh-delivery-builder.builder_versions" { + t.Errorf("the manifest invokes %v", m.Invokes) + } if len(m.Tools) != 0 { t.Error("the manifest lists tools: its claims would offer them as both seats' verbs") } found := false for _, a := range m.Build.Artifacts { - if a.From == "cmd/npm-registry" { + if a.From == "cmd/package-registry" { found = true - for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE", "MESH_NPM_OWNER"} { + for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_TOKEN_FILE", "MESH_GITEA_PACKAGES_TOKEN_FILE", "MESH_NPM_OWNER"} { if a.Env[k] == "" { t.Errorf("the bundle is not given %s", k) } @@ -400,7 +491,7 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { } } if !found { - t.Error("the manifest builds no cmd/npm-registry bundle") + t.Error("the manifest builds no cmd/package-registry bundle") } // Refused cleanly when the environment is missing, not a crash. if _, err := tools[0].Run(map[string]any{}); err == nil { diff --git a/modules/gitea/cmd/package-registry/seats.go b/modules/gitea/cmd/package-registry/seats.go new file mode 100644 index 00000000..759bdfad --- /dev/null +++ b/modules/gitea/cmd/package-registry/seats.go @@ -0,0 +1,372 @@ +package main + +// The two package seats' verbs (novox/hq design 51 *The four registry seats*, *One contract*): publish, publish-part, +// versions and remove on `npm-package-registry` and `go-package-registry`, served as tools named . and +// answered in each seat's own contract types (mesh-contracts npmpackageregistry, gopackageregistry; ADR 0620). The +// holding itself is internal/packageseats; this file binds it to the forge's package registry, the runtime's bus state +// and the seats' events. + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "crypto/sha1" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "strings" + "time" + + goreg "git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry" + npmreg "git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/parts" + + "gitea/internal/packageseats" +) + +// seatStore is the forge's package registry as both seats store into it. +type seatStore struct{ forge func() (*Forge, error) } + +func forgeType(seat string) string { return packageseats.Seats[seat] } + +// forgeVersion is the version as the forge keeps it: a Go module's carries Go's leading v. +func forgeVersion(seat, version string) string { + if seat == packageseats.GoSeat && !strings.HasPrefix(version, "v") { + return "v" + version + } + return version +} + +func (f *Forge) packagePath(typ, name, version string) string { + return fmt.Sprintf("/api/v1/packages/%s/%s/%s/%s", url.PathEscape(f.Owner), typ, url.PathEscape(name), url.PathEscape(version)) +} + +func (s seatStore) Held(seat, name, version string) (string, bool, error) { + f, err := s.forge() + if err != nil { + return "", false, err + } + var files []struct { + SHA256 string `json:"sha256"` + } + err = f.GetJSON(context.Background(), f.packagePath(forgeType(seat), name, forgeVersion(seat, version))+"/files", &files) + switch { + case errors.Is(err, errNotFound): + return "", false, nil + case err != nil: + return "", false, err + case len(files) == 0: + return "", false, fmt.Errorf("%s@%s is listed with no file", name, version) + } + return "sha256:" + files[0].SHA256, true, nil +} + +func (s seatStore) Put(seat string, p packageseats.Publish, file []byte) (string, error) { + f, err := s.forge() + if err != nil { + return "", err + } + w, err := f.write() + if err != nil { + return "", err + } + ctx := context.Background() + if seat == packageseats.GoSeat { + module, err := goModuleOf(file, forgeVersion(seat, p.Version)) + if err != nil { + return "", err + } + _, err = w.Send(ctx, http.MethodPut, fmt.Sprintf("/api/packages/%s/go/upload", url.PathEscape(f.Owner)), file, "application/zip", maxBody) + return module, f.wrote(err) + } + doc, err := npmDocument(f.URL, f.Owner, p, file) + if err != nil { + return "", err + } + _, err = w.Do(ctx, http.MethodPut, fmt.Sprintf("/api/packages/%s/npm/%s", url.PathEscape(f.Owner), url.PathEscape(p.Name)), doc, maxBody) + return "", f.wrote(err) +} + +func (s seatStore) Versions(seat, name string) ([]packageseats.Version, error) { + f, err := s.forge() + if err != nil { + return nil, err + } + var out []packageseats.Version + for page := 1; page <= 1000; page++ { + var batch []PackageVersion + p := fmt.Sprintf("/api/v1/packages/%s?type=%s&q=%s&page=%d&limit=%d", url.PathEscape(f.Owner), forgeType(seat), url.QueryEscape(name), page, pageSize) + if err := f.GetJSON(context.Background(), p, &batch); err != nil { + return nil, err + } + for _, v := range batch { + if v.Name != name { + continue + } + d, _, err := s.Held(seat, name, v.Version) + if err != nil { + return nil, err + } + out = append(out, packageseats.Version{Version: strings.TrimPrefix(v.Version, "v"), Digest: d, Published: v.CreatedAt}) + } + if len(batch) < pageSize { + break + } + } + // Newest first, by when each was published. + for i := 1; i < len(out); i++ { + for j := i; j > 0 && out[j].Published.After(out[j-1].Published); j-- { + out[j], out[j-1] = out[j-1], out[j] + } + } + return out, nil +} + +func (s seatStore) Remove(seat, name, version string) error { + f, err := s.forge() + if err != nil { + return err + } + return f.As(forgeType(seat)).Delete(context.Background(), name, forgeVersion(seat, version)) +} + +// goModuleOf is the Go module path of a module zip whose every file sits under @/, as Go's module +// proxy protocol has it; a zip of another version is refused before the forge sees it. +func goModuleOf(file []byte, version string) (string, error) { + z, err := zip.NewReader(bytes.NewReader(file), int64(len(file))) + if err != nil { + return "", fmt.Errorf("the Go package is not a module zip: %w", err) + } + if len(z.File) == 0 { + return "", errors.New("the Go package's zip holds no file") + } + module, rest, at := strings.Cut(z.File[0].Name, "@") + v, _, ok := strings.Cut(rest, "/") + if !ok || !at || module == "" { + return "", fmt.Errorf("the Go package's zip does not hold @/: %s", z.File[0].Name) + } + if v != version { + return "", fmt.Errorf("the Go package's zip is %s, not %s", v, version) + } + return module, nil +} + +// npmDocument is the body npm's publish sends: the package.json of the tarball's package/, its dist, the tarball as +// its one attachment, and the dist-tag: the version's tag for a tagged version, so that it never becomes latest. +func npmDocument(base, owner string, p packageseats.Publish, tgz []byte) (map[string]any, error) { + manifest, err := npmManifest(tgz) + if err != nil { + return nil, err + } + if manifest["name"] != p.Name || manifest["version"] != p.Version { + return nil, fmt.Errorf("the tarball is %v@%v, not %s@%s", manifest["name"], manifest["version"], p.Name, p.Version) + } + file := path.Base(p.Name) + "-" + p.Version + ".tgz" + s1, s512 := sha1.Sum(tgz), sha512.Sum512(tgz) + manifest["_id"] = p.Name + "@" + p.Version + manifest["dist"] = map[string]any{"shasum": hex.EncodeToString(s1[:]), "integrity": "sha512-" + base64.StdEncoding.EncodeToString(s512[:]), + "tarball": fmt.Sprintf("%s/api/packages/%s/npm/%s/-/%s/%s", base, owner, p.Name, p.Version, file)} + tag := "latest" + if p.Tag != "" { + tag = p.Tag + } + return map[string]any{"_id": p.Name, "name": p.Name, "dist-tags": map[string]string{tag: p.Version}, + "versions": map[string]any{p.Version: manifest}, + "_attachments": map[string]any{file: map[string]any{"content_type": "application/octet-stream", "data": base64.StdEncoding.EncodeToString(tgz), "length": len(tgz)}}}, nil +} + +func npmManifest(tgz []byte) (map[string]any, error) { + gz, err := gzip.NewReader(bytes.NewReader(tgz)) + if err != nil { + return nil, fmt.Errorf("the npm package is not a gzipped tarball: %w", err) + } + tr := tar.NewReader(gz) + for { + h, err := tr.Next() + if err == io.EOF { + return nil, errors.New("the npm package holds no package/package.json") + } + if err != nil { + return nil, fmt.Errorf("the npm package's tarball cannot be read: %w", err) + } + if h.Name != "package/package.json" { + continue + } + var m map[string]any + if err := json.NewDecoder(io.LimitReader(tr, 1<<20)).Decode(&m); err != nil { + return nil, fmt.Errorf("the npm package's package.json is not JSON: %w", err) + } + return m, nil + } +} + +// as is v in T's shape: what crosses the bus is the seat's contract type, read or written through its JSON. +func as[T any](v any) (T, error) { + var out T + raw, err := json.Marshal(v) + if err == nil { + err = json.Unmarshal(raw, &out) + } + return out, err +} + +// outcomeBody is an outcome as the seat's contract's Outcome names it. +func outcomeBody(o packageseats.Outcome) map[string]any { + return map[string]any{"name": o.Name, "version": o.Version, "kind": o.Kind, "tag": o.Tag, "digest": o.Digest, + "fingerprint": o.Fingerprint, "commit": o.Commit, "holder": packageseats.Self, "transfer": o.Transfer, + "reference": o.Reference, "reason": o.Reason, "module": o.Module} +} + +// say says a seat's event; a test hears it instead. +var say = sayOn + +// sayOn says an event on the seat's family, in the seat's contract's Outcome. +func sayOn(seat, event string, o packageseats.Outcome) error { + var body any + var err error + if seat == packageseats.GoSeat { + body, err = as[goreg.Outcome](outcomeBody(o)) + } else { + body, err = as[npmreg.Outcome](outcomeBody(o)) + } + if err != nil { + return err + } + return stdio.SeatPublish("mesh.seat."+seat+".event."+event, body, "") +} + +func finding(kind, subject, words string, raised bool) { + key := health.FindingKey(health.ScopeSeat, packageseats.Self, kind+"."+strings.NewReplacer(".", "_", "@", "_", "/", "_").Replace(subject)) + if !raised { + _ = stdio.State(health.FindingsState).Delete(key) + return + } + f := health.Finding{V: health.FindingVersion, Key: key, Kind: kind, Ref: key[strings.LastIndex(key, ".")+1:], + Subject: health.Subject{Scope: health.ScopeSeat, ID: packageseats.Self}, Severity: health.Warning, + Summary: words, Headline: words, Explanation: words, Resolved: "cleared once it no longer holds", + Source: kind, Observer: packageseats.Self, At: time.Now().UTC()} + if _, err := stdio.State(health.FindingsState).Put(key, f); err != nil { + fmt.Fprintf(os.Stderr, "the finding %s could not be written: %v\n", key, err) + } +} + +// busHolder is the holder over the runtime's bus state, the builder's numbering and the forge. +func busHolder(forge func() (*Forge, error)) (*packageseats.Holder, error) { + callers, err := packageseats.ConfiguredCallers() + if err != nil { + return nil, err + } + return &packageseats.Holder{Callers: callers, Objects: parts.BusObjects(packageseats.IncomingObjects), + Transfers: parts.BusTransfers[packageseats.Opening](packageseats.TransfersState), + Builder: delivery.BuilderVersions{Ask: stdio.Ask}, Store: seatStore{forge}, Say: say, Finding: finding}, nil +} + +// SeatTools are both seats' verbs as tools .. +func SeatTools(h func() (*packageseats.Holder, error)) []stdio.Tool { + return append(verbTools[npmreg.PublishInput, npmreg.PublishAnswer, npmreg.PublishPartInput, npmreg.PublishPartAnswer, + npmreg.VersionsInput, npmreg.VersionsAnswer, npmreg.RemoveInput, npmreg.RemoveAnswer](packageseats.NpmSeat, h), + verbTools[goreg.PublishInput, goreg.PublishAnswer, goreg.PublishPartInput, goreg.PublishPartAnswer, + goreg.VersionsInput, goreg.VersionsAnswer, goreg.RemoveInput, goreg.RemoveAnswer](packageseats.GoSeat, h)...) +} + +// chunk is publish-part's input as the holder reads it. +type chunk struct { + Data, Part, Transfer string + Last bool + Sequence int +} + +// verbTools are one seat's four verbs, each reading its input as the seat's contract type and answering in it. +func verbTools[PI, PA, PPI, PPA, VI, VA, RI, RA any](seat string, holder func() (*packageseats.Holder, error)) []stdio.Tool { + obj := map[string]any{"type": "object"} + run := func(do func(h *packageseats.Holder, args map[string]any) (any, error)) func(map[string]any) (any, error) { + return func(args map[string]any) (any, error) { + h, err := holder() + if err != nil { + return nil, err + } + return do(h, args) + } + } + return []stdio.Tool{ + {Name: seat + "." + packageseats.VerbPublish, Input: obj, + Description: "Open a transfer of a package's one file (ADR 0611): its name, the builder's version, kind package, its sha256 digest and the file as its one part; answers the part the forge lacks. A version held with that digest answers held and says published.", + Run: run(func(h *packageseats.Holder, args map[string]any) (any, error) { + in, err := as[PI](args) + if err != nil { + return nil, fmt.Errorf("publish: %w", err) + } + p, err := as[packageseats.Publish](in) + if err != nil { + return nil, err + } + a, err := h.Publish(seat, stdio.CallerOf(args), p) + out, _ := as[PA](map[string]any{"transfer": a.Transfer, "lacks": a.Lacks, "held": a.Ended && len(a.Lacks) == 0}) + return out, err + })}, + {Name: seat + "." + packageseats.VerbPublishPart, Input: obj, + Description: "Send one chunk (at most 760 KiB before base64) of the package's file, from the caller that opened the transfer; the last chunk ends it and the forge says published or refused.", + Run: run(func(h *packageseats.Holder, args map[string]any) (any, error) { + in, err := as[PPI](args) + if err != nil { + return nil, fmt.Errorf("publish-part: %w", err) + } + c, err := as[chunk](in) + if err != nil { + return nil, err + } + data, err := base64.StdEncoding.DecodeString(c.Data) + if err != nil { + return nil, fmt.Errorf("publish-part: data is not base64: %w", err) + } + a, err := h.PublishPart(seat, stdio.CallerOf(args), c.Transfer, c.Part, c.Sequence, c.Last, data) + out, _ := as[PPA](map[string]any{"transfer": c.Transfer, "part": c.Part, "received": c.Sequence + 1, "complete": a.Ended}) + return out, err + })}, + {Name: seat + "." + packageseats.VerbVersions, Input: obj, + Description: "The versions the seat holds of one package, newest first, each with its digest.", + Run: run(func(h *packageseats.Holder, args map[string]any) (any, error) { + in, err := as[VI](args) + if err != nil { + return nil, fmt.Errorf("versions: %w", err) + } + q, _ := as[struct{ Name string }](in) + got, err := h.Versions(seat, stdio.CallerOf(args), q.Name) + if err != nil { + return nil, err + } + list := []map[string]any{} + for _, v := range got { + list = append(list, map[string]any{"name": q.Name, "version": v.Version, "digest": v.Digest, "kind": packageseats.KindPackage}) + } + return as[VA](map[string]any{"name": q.Name, "versions": list}) + })}, + {Name: seat + "." + packageseats.VerbRemove, Input: obj, + Description: "Remove a tagged version (the builder, when its branch ended); says removed. An untagged version goes by the forge's retention alone.", + Run: run(func(h *packageseats.Holder, args map[string]any) (any, error) { + in, err := as[RI](args) + if err != nil { + return nil, fmt.Errorf("remove: %w", err) + } + r, _ := as[struct{ Name, Version, Reason string }](in) + o, err := h.Remove(seat, stdio.CallerOf(args), r.Name, r.Version, r.Reason) + if err != nil { + return nil, err + } + return as[RA](map[string]any{"removed": true, "artifact": map[string]any{"name": o.Name, "version": o.Version, "digest": o.Digest, "kind": o.Kind}}) + })}, + } +} diff --git a/modules/gitea/cmd/package-registry/seats_test.go b/modules/gitea/cmd/package-registry/seats_test.go new file mode 100644 index 00000000..92ac71e4 --- /dev/null +++ b/modules/gitea/cmd/package-registry/seats_test.go @@ -0,0 +1,197 @@ +package main + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" + "gitea/internal/forgehttp" + "gitea/internal/packageseats" +) + +func tarball(t *testing.T, manifest string) []byte { + var b bytes.Buffer + gz := gzip.NewWriter(&b) + tw := tar.NewWriter(gz) + _ = tw.WriteHeader(&tar.Header{Name: "package/package.json", Mode: 0o644, Size: int64(len(manifest))}) + _, _ = tw.Write([]byte(manifest)) + _ = tw.Close() + _ = gz.Close() + return b.Bytes() +} + +func moduleZip(t *testing.T, prefix string) []byte { + var b bytes.Buffer + z := zip.NewWriter(&b) + w, _ := z.Create(prefix + "/go.mod") + _, _ = w.Write([]byte("module x\n")) + _ = z.Close() + return b.Bytes() +} + +func sha(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } + +// registry is the forge's package registry: what was uploaded, by path, and what the files API answers. +type registry struct { + mu sync.Mutex + uploaded map[string][]byte // "npm " or "go" + held map[string][]byte // "//" -> file + deleted []string +} + +func (g *registry) handle(w http.ResponseWriter, r *http.Request) { + g.mu.Lock() + defer g.mu.Unlock() + body, _ := io.ReadAll(r.Body) + p := r.URL.EscapedPath() + // Reads come as the tools' account (read:package), writes as the packages account (write:package), never the admin. + want := "token reader" + if r.Method != http.MethodGet { + want = "token writer" + } + if got := r.Header.Get("Authorization"); got == "token stale" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } else if got != want { + http.Error(w, r.Method+" "+p+" came as "+got+", not "+want, http.StatusForbidden) + return + } + switch { + case r.Method == http.MethodPut && p == "/api/packages/acme/go/upload": + if r.Header.Get("Content-Type") != "application/zip" { + http.Error(w, "not a zip", 400) + return + } + g.uploaded["go"] = body + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodPut && strings.HasPrefix(p, "/api/packages/acme/npm/"): + g.uploaded["npm "+strings.TrimPrefix(p, "/api/packages/acme/npm/")] = body + w.WriteHeader(http.StatusCreated) + case strings.HasPrefix(p, "/api/v1/packages/acme/") && strings.HasSuffix(p, "/files"): + f, ok := g.held[strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/packages/acme/"), "/files")] + if !ok { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode([]map[string]any{{"sha256": sha(f), "size": len(f)}}) + case r.Method == http.MethodGet && p == "/api/v1/packages/acme": + var out []map[string]any + for k := range g.held { + parts := strings.Split(k, "/") + if parts[0] == r.URL.Query().Get("type") { + out = append(out, map[string]any{"name": strings.Join(parts[1:len(parts)-1], "/"), "version": parts[len(parts)-1], "created_at": time.Unix(int64(len(parts[len(parts)-1])), 0)}) + } + } + _ = json.NewEncoder(w).Encode(out) + case r.Method == http.MethodDelete: + g.deleted = append(g.deleted, strings.TrimPrefix(p, "/api/v1/packages/acme/")) + w.WriteHeader(http.StatusNoContent) + default: + http.Error(w, "unexpected "+r.Method+" "+p, 500) + } +} + +// **The package seats store into the forge's package registry** (design 51 *Publishing*): an npm package goes up as +// npm's publish sends it, its package.json from the tarball and its tag as its dist-tag; a Go module's zip goes up to +// the Go upload, its module path read from the zip; held, versions and remove read and delete through the forge's +// package API, a Go version with Go's leading v. +func TestThePackageSeatsStoreIntoTheForge(t *testing.T) { + g := ®istry{uploaded: map[string][]byte{}, held: map[string][]byte{}} + srv := httptest.NewServer(http.HandlerFunc(g.handle)) + defer srv.Close() + dir := t.TempDir() + kept := func(name, token, line string) string { + file := filepath.Join(dir, name) + _ = os.WriteFile(file, []byte(token+"\n"), 0o600) + _ = os.WriteFile(file+".scopes", []byte(line+"\n"), 0o600) + return file + } + reader := forgehttp.Client{URL: srv.URL, TokenFile: kept("token", "reader", "read:package,read:user for mesh-tools"), Account: "mesh-tools"} + writer := forgehttp.Client{URL: srv.URL, TokenFile: kept("packages-token", "writer", "write:package for mesh-packages"), Account: "mesh-packages"} + forge := &Forge{Client: reader, Writer: &writer, Owner: "acme"} + if _, err := (seatStore{forge: func() (*Forge, error) { return &Forge{Client: reader, Owner: "acme"}, nil }}).Put(packageseats.GoSeat, packageseats.Publish{}, nil); err == nil || + !strings.Contains(err.Error(), "write side") { + t.Errorf("a forge handed no write side wrote: %v", err) + } + s := seatStore{forge: func() (*Forge, error) { return forge, nil }} + // The forge refuses the write side's token: the finding the TypeScript bundle renews on is raised under its key, + // and the next write the forge takes clears it (no timer looks at the token). + var refused []bool + forge.Refused = func(raised bool) { refused = append(refused, raised) } + if k := health.FindingKey(health.ScopeSeat, packageseats.Self, PackagesTokenRefused+".write"); k != PackagesTokenRefusedKey { + t.Errorf("the refusal's key is %s, the watcher watches %s", k, PackagesTokenRefusedKey) + } + _ = os.WriteFile(writer.TokenFile, []byte("stale\n"), 0o600) + if _, err := s.Put(packageseats.GoSeat, packageseats.Publish{Version: "0.1.15"}, moduleZip(t, "git.example/acme/sdk@v0.1.15")); err == nil || + len(refused) != 1 || !refused[0] { + t.Errorf("a write the forge refused with 401 raised no finding: %v %v", err, refused) + } + _ = os.WriteFile(writer.TokenFile, []byte("writer\n"), 0o600) + + tgz := tarball(t, `{"name":"@acme/contracts","version":"0.1.15-feat-x.2","main":"index.js"}`) + p := packageseats.Publish{Name: "@acme/contracts", Version: "0.1.15-feat-x.2", Tag: "feat-x", Kind: packageseats.KindPackage, Digest: "sha256:" + sha(tgz)} + if module, err := s.Put(packageseats.NpmSeat, p, tgz); err != nil || module != "" { + t.Fatalf("npm put: %q, %v", module, err) + } + var doc struct { + DistTags map[string]string `json:"dist-tags"` + Versions map[string]map[string]any + Attachments map[string]struct{ Data string } `json:"_attachments"` + } + if err := json.Unmarshal(g.uploaded["npm @acme%2Fcontracts"], &doc); err != nil { + t.Fatalf("npm's publish body %v: %s", err, g.uploaded) + } + att := doc.Attachments["contracts-0.1.15-feat-x.2.tgz"].Data + if doc.DistTags["feat-x"] != "0.1.15-feat-x.2" || doc.DistTags["latest"] != "" || doc.Versions["0.1.15-feat-x.2"]["main"] != "index.js" || + att != base64.StdEncoding.EncodeToString(tgz) { + t.Errorf("npm's publish body: %+v", doc) + } + p.Name = "@acme/other" + if _, err := s.Put(packageseats.NpmSeat, p, tgz); err == nil || !strings.Contains(err.Error(), "not @acme/other") { + t.Errorf("a tarball of another package was uploaded: %v", err) + } + + zipped := moduleZip(t, "git.example/acme/sdk@v0.1.15") + gp := packageseats.Publish{Name: "git.example/acme/sdk", Version: "0.1.15", Kind: packageseats.KindPackage, Digest: "sha256:" + sha(zipped)} + if module, err := s.Put(packageseats.GoSeat, gp, zipped); err != nil || module != "git.example/acme/sdk" || !bytes.Equal(g.uploaded["go"], zipped) { + t.Fatalf("go put: %q, %v", module, err) + } + gp.Version = "0.1.16" + if _, err := s.Put(packageseats.GoSeat, gp, zipped); err == nil { + t.Error("a zip of another version was uploaded") + } + + g.held["go/git.example/acme/sdk/v0.1.15"] = zipped + if d, ok, err := s.Held(packageseats.GoSeat, "git.example/acme/sdk", "0.1.15"); err != nil || !ok || d != "sha256:"+sha(zipped) { + t.Errorf("held: %s %v %v", d, ok, err) + } + if _, ok, err := s.Held(packageseats.NpmSeat, "x", "0.1.0"); ok || err != nil { + t.Errorf("a version the forge does not hold: %v %v", ok, err) + } + g.held["go/git.example/acme/sdk/v0.1.9"] = zipped + if v, err := s.Versions(packageseats.GoSeat, "git.example/acme/sdk"); err != nil || len(v) != 2 || v[0].Version != "0.1.15" { + t.Errorf("versions: %+v %v", v, err) + } + if err := s.Remove(packageseats.GoSeat, "git.example/acme/sdk", "0.1.15-feat.1"); err != nil || len(g.deleted) != 1 || + g.deleted[0] != "go/git.example%2Facme%2Fsdk/v0.1.15-feat.1" { + t.Errorf("remove: %v %v", err, g.deleted) + } + if len(refused) < 2 || refused[len(refused)-1] { + t.Errorf("a write the forge took after the refusal did not clear it: %v", refused) + } +} diff --git a/modules/gitea/cmd/npm-registry/semver.go b/modules/gitea/cmd/package-registry/semver.go similarity index 100% rename from modules/gitea/cmd/npm-registry/semver.go rename to modules/gitea/cmd/package-registry/semver.go diff --git a/modules/gitea/cmd/npm-registry/semver_test.go b/modules/gitea/cmd/package-registry/semver_test.go similarity index 100% rename from modules/gitea/cmd/npm-registry/semver_test.go rename to modules/gitea/cmd/package-registry/semver_test.go diff --git a/modules/gitea/cmd/package-registry/whole_integration_test.go b/modules/gitea/cmd/package-registry/whole_integration_test.go new file mode 100644 index 00000000..9085bd12 --- /dev/null +++ b/modules/gitea/cmd/package-registry/whole_integration_test.go @@ -0,0 +1,408 @@ +//go:build integration + +package main + +// The forge module's package seats whole against a real bus (CAT-G3; design 51 *Acceptance tests*, the holder's cases): +// the built package-registry bundle, played over its real stdin and stdout as the node's tool runtime plays it, its +// `incoming` an object store and its `transfers` and `findings` key-value buckets of a nats-server of this test alone +// (so the publish in parts runs through mesh-sdk's parts.BusObjects and BusTransfers, ruling 4 of the lead's 14:0x), the +// builder's builder_versions answered on mesh/ask, the config value `callers` on mesh/provision.get, and the forge an +// httptest server that takes reads only with the tools' token and writes only with the packages account's. +// +// A Go package the builder numbered goes up in two chunks, is uploaded whole as the packages account and says +// published on the Go seat with its module path; one the builder did not number is refused before the forge sees it, +// says refused and writes the seat's finding; versions answers what the forge holds. + +import ( + "archive/zip" + "bufio" + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/testbus" + + "gitea/internal/packageseats" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` +} + +// player is the node's tool runtime, as far as the package seats need it, over a real bus. +type player struct { + ctx context.Context + t *testing.T + nc *nats.Conn + js jetstream.JetStream + numbered map[string]string // name@version -> the digest the builder numbered + mu sync.Mutex + in io.Writer + next int + answers map[string]chan wire +} + +func (p *player) send(m map[string]any) { + m["jsonrpc"] = "2.0" + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<26) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var id string + _ = json.Unmarshal(m.ID, &id) + p.mu.Lock() + ch := p.answers[id] + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// handle answers one call of the bundle on the runtime. +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Objects string `json:"objects"` + Name string `json:"name"` + Data []byte `json:"data"` + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + Kind string `json:"kind"` + } + _ = json.Unmarshal(m.Params, &a) + switch { + case m.Method == "mesh/ask" && a.Key == delivery.BuilderVersionsAddress: + var q struct{ Name, Version, Digest string } + _ = json.Unmarshal(a.Body, &q) + d, ok := p.numbered[q.Name+"@"+q.Version] + p.answer(m.ID, map[string]any{"recorded": ok && d == q.Digest, "why": "not numbered with that digest"}, nil) + case m.Method == "mesh/provision.get" && a.Kind == "config": + p.answer(m.ID, nil, nil) // `callers` unset: design 51's list + case m.Method == "mesh/seat.publish": + p.answer(m.ID, map[string]any{}, p.nc.Publish(a.Subject, a.Body)) + case m.Method == "mesh/publish": + p.answer(m.ID, map[string]any{}, nil) + case strings.HasPrefix(m.Method, "mesh/objects."): + p.objects(m, a.Objects, a.Name, a.Data) + case strings.HasPrefix(m.Method, "mesh/state."): + p.state(m, a.State, a.Key, a.Value) + default: + p.answer(m.ID, nil, errors.New("not played here: "+m.Method)) + } +} + +func (p *player) objects(m wire, local, name string, data []byte) { + os, err := p.js.ObjectStore(p.ctx, "gitea_"+local) + if err != nil { + p.answer(m.ID, nil, err) + return + } + switch m.Method { + case "mesh/objects.put": + info, err := os.PutBytes(p.ctx, name, data) + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"name": info.Name, "size": info.Size, "digest": info.Digest}, nil) + case "mesh/objects.get": + got, err := os.GetBytes(p.ctx, name) + p.answer(m.ID, map[string]any{"info": map[string]any{"name": name, "size": len(got)}, "data": got}, err) + case "mesh/objects.delete": + err := os.Delete(p.ctx, name) + if errors.Is(err, jetstream.ErrObjectNotFound) { + err = nil + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/objects.info": + st, err := os.Status(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"bytes": st.Size(), "max_bytes": 2048 << 20}, nil) + default: + p.answer(m.ID, nil, errors.New("not played here: "+m.Method)) + } +} + +func (p *player) state(m wire, state, key string, value json.RawMessage) { + kv, err := p.js.KeyValue(p.ctx, "gitea_"+state) + if err != nil { + p.answer(m.ID, nil, err) + return + } + switch m.Method { + case "mesh/state.get": + e, err := kv.Get(p.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + rev, err := kv.Put(p.ctx, key, value) + p.answer(m.ID, map[string]any{"revision": rev}, err) + case "mesh/state.delete": + p.answer(m.ID, map[string]any{}, kv.Delete(p.ctx, key)) + case "mesh/state.keys": + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + default: + p.answer(m.ID, nil, errors.New("not played here: "+m.Method)) + } +} + +// call is one tools/call as the runtime makes it, the caller named in _meta as the runtime proves it. +func (p *player) call(caller, tool string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("call-%d", p.next) + ch := make(chan wire, 1) + p.answers[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": tool, "arguments": args, + "_meta": map[string]any{"mesh/caller": caller}}}) + select { + case m := <-ch: + var r struct { + Content []struct{ Text string } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) == 0 { + return nil, fmt.Errorf("%s answered %s", tool, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(30 * time.Second): + return nil, fmt.Errorf("%s did not answer", tool) + } +} + +// bigZip is a Go module zip of about 1.2 MiB that does not compress, so its file goes in two chunks. +func bigZip(t *testing.T, prefix string) []byte { + noise := make([]byte, 1200<<10) + _, _ = rand.Read(noise) + var b bytes.Buffer + z := zip.NewWriter(&b) + w, _ := z.Create(prefix + "/go.mod") + _, _ = w.Write([]byte("module " + strings.Split(prefix, "@")[0] + "\n")) + w, _ = z.CreateHeader(&zip.FileHeader{Name: prefix + "/noise.bin", Method: zip.Store}) + _, _ = w.Write(noise) + _ = z.Close() + return b.Bytes() +} + +func TestThePackageSeatsWhole(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + bin := filepath.Join(t.TempDir(), "package-registry") + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("package-registry does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, _ := jetstream.New(nc) + for _, b := range []string{"gitea_" + packageseats.TransfersState, "gitea_findings"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: "gitea_" + packageseats.IncomingObjects}); err != nil { + t.Fatal(err) + } + heard := make(chan *nats.Msg, 8) + if _, err := nc.ChanSubscribe("mesh.seat.go-package-registry.event.>", heard); err != nil { + t.Fatal(err) + } + + // The forge: reads only as the tools' account, writes only as the packages account. + g := ®istry{uploaded: map[string][]byte{}, held: map[string][]byte{}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + g.handle(w, r) + if r.Method == http.MethodPut && r.URL.Path == "/api/packages/acme/go/upload" { + g.mu.Lock() + g.held["go/git.example/acme/sdk/v0.1.15"] = g.uploaded["go"] + g.mu.Unlock() + } + })) + t.Cleanup(srv.Close) + dir := t.TempDir() + kept := func(name, token, line string) string { + file := filepath.Join(dir, name) + _ = os.WriteFile(file, []byte(token+"\n"), 0o600) + _ = os.WriteFile(file+".scopes", []byte(line+"\n"), 0o600) + return file + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_GITEA_URL="+srv.URL, "MESH_NPM_OWNER=acme", "MESH_GITEA_TOOLS_USER=mesh-tools", + "MESH_GITEA_TOKEN_FILE="+kept("token", "reader", "read:package,read:user,write:issue,write:repository for mesh-tools"), + "MESH_GITEA_PACKAGES_TOKEN_FILE="+kept("packages-token", "writer", "write:package for mesh-packages")) + cmd.Stderr = os.Stderr + in, _ := cmd.StdinPipe() + out, _ := cmd.StdoutPipe() + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = in.Close(); _ = cmd.Wait() }) + + zipped := bigZip(t, "git.example/acme/sdk@v0.1.15") + digest := "sha256:" + sha(zipped) + p := &player{ctx: ctx, t: t, nc: nc, js: js, in: in, answers: map[string]chan wire{}, + numbered: map[string]string{"git.example/acme/sdk@0.1.15": digest}} + go p.serve(out) + p.send(map[string]any{"id": "init", "method": "initialize", "params": map[string]any{}}) + + builder := "ace.mesh-build-go" + publish := func(version, digest string, size int) (string, []string) { + raw, err := p.call(builder, "go-package-registry.publish", map[string]any{"name": "git.example/acme/sdk", "version": version, + "kind": "package", "digest": digest, "transfer": "sdk-" + strings.ReplaceAll(version, ".", "-"), "parts": []map[string]any{{"digest": digest, "size": size}}}) + if err != nil { + t.Fatalf("publish %s: %v", version, err) + } + var a struct { + Transfer string `json:"transfer"` + Lacks []string `json:"lacks"` + } + _ = json.Unmarshal(raw, &a) + return a.Transfer, a.Lacks + } + send := func(transfer, digest string, file []byte) error { + const chunk = 700 << 10 + for seq, at := 0, 0; at < len(file); seq, at = seq+1, at+chunk { + end := min(at+chunk, len(file)) + if _, err := p.call(builder, "go-package-registry.publish-part", map[string]any{"transfer": transfer, "part": digest, + "sequence": seq, "last": end == len(file), "data": base64.StdEncoding.EncodeToString(file[at:end])}); err != nil { + return err + } + } + return nil + } + event := func(want string) map[string]any { + select { + case m := <-heard: + if !strings.HasSuffix(m.Subject, "."+want) { + t.Fatalf("the Go seat said %s, want %s: %s", m.Subject, want, m.Data) + } + var o map[string]any + _ = json.Unmarshal(m.Data, &o) + return o + case <-time.After(30 * time.Second): + t.Fatalf("the Go seat said no %s", want) + } + return nil + } + + // A publish naming no transfer is refused: the transfer's name is the caller's own. + if _, err := p.call(builder, "go-package-registry.publish", map[string]any{"name": "git.example/acme/sdk", "version": "0.1.15", + "kind": "package", "digest": digest, "parts": []map[string]any{{"digest": digest, "size": len(zipped)}}}); err == nil || + !strings.Contains(err.Error(), "no transfer named") { + t.Errorf("a publish naming no transfer: %v", err) + } + // A version the builder numbered: two chunks through incoming and transfers, uploaded whole as the packages account. + transfer, lacks := publish("0.1.15", digest, len(zipped)) + if transfer == "" || len(lacks) != 1 || lacks[0] != digest { + t.Fatalf("publish answered transfer %q lacks %v", transfer, lacks) + } + if err := send(transfer, digest, zipped); err != nil { + t.Fatal(err) + } + if o := event("published"); o["module"] != "git.example/acme/sdk" || o["reference"] != "git.example/acme/sdk@0.1.15" || o["holder"] != "gitea" { + t.Errorf("published says %v", o) + } + g.mu.Lock() + up := g.uploaded["go"] + g.mu.Unlock() + if !bytes.Equal(up, zipped) { + t.Errorf("the forge took %d bytes, not the %d published", len(up), len(zipped)) + } + transfers, _ := js.KeyValue(ctx, "gitea_"+packageseats.TransfersState) + if keys, _ := transfers.Keys(ctx); len(keys) != 0 { + t.Errorf("an ended transfer is still kept: %v", keys) + } + + // One the builder did not number: refused before the forge sees it, said, and the seat's finding written. + other := bigZip(t, "git.example/acme/sdk@v0.1.16") + od := "sha256:" + sha(other) + transfer, _ = publish("0.1.16", od, len(other)) + _ = send(transfer, od, other) + if o := event("refused"); !strings.Contains(fmt.Sprint(o["reason"]), "did not number") { + t.Errorf("refused says %v", o) + } + findings, _ := js.KeyValue(ctx, "gitea_findings") + if keys, _ := findings.Keys(ctx); len(keys) == 0 { + t.Error("a refused publish wrote no finding") + } + g.mu.Lock() + _, uploadedOther := g.held["go/git.example/acme/sdk/v0.1.16"] + g.mu.Unlock() + if uploadedOther { + t.Error("a version the builder did not number reached the forge") + } + + // Anyone reads versions; the forge was read as the tools' account. + raw, err := p.call("g14.someone", "go-package-registry.versions", map[string]any{"name": "git.example/acme/sdk"}) + if err != nil || !strings.Contains(string(raw), `"version":"0.1.15"`) || !strings.Contains(string(raw), digest) { + t.Errorf("versions answered %s, %v", raw, err) + } +} diff --git a/modules/gitea/go.mod b/modules/gitea/go.mod index 56c3ae98..782aa34c 100644 --- a/modules/gitea/go.mod +++ b/modules/gitea/go.mod @@ -3,6 +3,8 @@ module gitea go 1.26.0 require ( + git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index b2af463b..bb15ebc6 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -62,6 +62,7 @@ import { announcementOf, weighTrunks } from "./trunk.js"; import { ReadAll, readAllTools } from "./readall.js"; import { DeliveredWrites, INDEXER_MODULES_EVENTS, INDEXER_MODULES_TOOL, INDEXER_OWNER, writeTools, type ModulesChange } from "./writes.js"; import { TOOLS_ACCOUNT, TOOLS_ORG, meshAccounts } from "./tools-account.js"; +import { PACKAGES_TOKEN_REFUSED, keepPackagesToken, packagesToken, renewOnRefusal } from "./packages-account.js"; // Without a way to a token — configured, or mintable with the admin account (token.ts) — there is // nothing to watch; log and stay quiet rather than crash the runtime. With one, the first poll mints @@ -538,6 +539,18 @@ if (gitea) { // Main takes no push and only the tools' account merges, in every repository, kept and not only when a token is // minted: at the start and every ten minutes after, so a repository made later gets it too (issue 948, ADR 0398 §2). // Only while the setting keep-main-rules is true (mainrules.ts); false, the default, leaves every repository's rule as set. + // The package seats' write token (packages-account.ts): minted at the start; after that renewed only when the Go + // bundle that writes with it says the forge refused it (the finding PACKAGES_TOKEN_REFUSED), never on a timer. + try { + const url = process.env.MESH_GITEA_URL ?? "http://127.0.0.1:3000"; + const source = packagesToken(url); + const say = (line: string) => console.log(`[gitea] ${line}`); + void keepPackagesToken(source, url).then(say, (err) => console.error(`[gitea] the package seats' write token could not be minted at the start — ${err instanceof Error ? err.message : err}`)); + const findings = state("findings"); + void bindUntilHeld("findings", () => findings.watch(renewOnRefusal(source, url, findings, say), { key: PACKAGES_TOKEN_REFUSED })); + } catch (err) { + console.error(`[gitea] the package seats cannot write — ${err instanceof Error ? err.message : err}`); + } tick(mainRulesStep(keepMainRules(), () => GiteaAdmin.fromEnv().ensureMainMerges("novox"), (line) => console.log(line)), 600_000); tick(() => pollRepos(client), 60_000); // **No timer asks the forge for pull requests or merges** (issue 1422): the forge's events announce them. One pass over diff --git a/modules/gitea/internal/forgehttp/forgehttp.go b/modules/gitea/internal/forgehttp/forgehttp.go index 3b3e8c89..3951ce34 100644 --- a/modules/gitea/internal/forgehttp/forgehttp.go +++ b/modules/gitea/internal/forgehttp/forgehttp.go @@ -1,5 +1,5 @@ // Package forgehttp is the forge's HTTP interface as this module's Go bundles reach it (cmd/forge, -// cmd/npm-registry), on the forge's own machine. One copy for both bundles (novox/hq issue 539). +// cmd/package-registry), on the forge's own machine. One copy for both bundles (novox/hq issue 539). // // Two ways in, by what the manifest hands the bundle: // @@ -46,6 +46,29 @@ type Client struct { // ToolsAccount is the account the forge's tools act as when the manifest names none (token.ts, tools-account.ts). const ToolsAccount = "mesh-tools" +// PackagesAccount is the account the package seats write as when the manifest names none (packages-account.ts): +// its token carries write:package and nothing else (design 51: each holder keeps its write side as an own secret +// beside a read-only tool token). +const PackagesAccount = "mesh-packages" + +// WriterFromEnv is the package registry's write side as the manifest hands it to a bundle: the packages account's +// token (MESH_GITEA_PACKAGES_TOKEN_FILE, kept by the module beside the tools' token) and the names of the variables +// it left unset. Never the admin login: a bundle handed no writer's token writes nothing. +func WriterFromEnv() (Client, []string) { + c := Client{URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), TokenFile: os.Getenv("MESH_GITEA_PACKAGES_TOKEN_FILE"), + Account: os.Getenv("MESH_GITEA_PACKAGES_USER")} + if c.Account == "" { + c.Account = PackagesAccount + } + var missing []string + for _, kv := range [][2]string{{"MESH_GITEA_URL", c.URL}, {"MESH_GITEA_PACKAGES_TOKEN_FILE", c.TokenFile}} { + if kv[1] == "" { + missing = append(missing, kv[0]) + } + } + return c, missing +} + // FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset. A bundle // handed MESH_GITEA_TOKEN_FILE acts as the tools' account and needs no admin login. func FromEnv() (Client, []string) { @@ -113,7 +136,7 @@ func (c *Client) password() (string, error) { func (c *Client) token() (string, error) { raw, err := os.ReadFile(c.TokenFile) if err != nil { - return "", fmt.Errorf("the tools' token is not kept yet (the module mints it for %s at its first call): %w", c.Account, err) + return "", fmt.Errorf("the token for %s is not kept yet (the module mints it): %w", c.Account, err) } line, err := os.ReadFile(c.TokenFile + ".scopes") if err != nil || !strings.HasSuffix(strings.TrimSpace(string(line)), " for "+c.Account) { @@ -148,12 +171,21 @@ func (c *Client) authorize(req *http.Request) error { // 404 is ErrNotFound. One byte past the limit is read, so a body the limit cut is told from one that fits it // exactly: a cut body is refused, never read as if it were whole. func (c *Client) Do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) { + if payload == nil { + return c.Send(ctx, method, path, nil, "", limit) + } + raw, err := json.Marshal(payload) + if err != nil { + return nil, err + } + return c.Send(ctx, method, path, raw, "application/json", limit) +} + +// Send is Do with a body of its own content type, sent as it is (a package's file uploaded into the registry); +// a nil body sends none. +func (c *Client) Send(ctx context.Context, method, path string, raw []byte, contentType string, limit int64) ([]byte, error) { var body io.Reader - if payload != nil { - raw, err := json.Marshal(payload) - if err != nil { - return nil, err - } + if raw != nil { body = bytes.NewReader(raw) } req, err := http.NewRequestWithContext(ctx, method, c.URL+path, body) @@ -164,8 +196,8 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit return nil, err } req.Header.Set("Accept", "application/json") - if payload != nil { - req.Header.Set("Content-Type", "application/json") + if raw != nil { + req.Header.Set("Content-Type", contentType) } res, err := c.client().Do(req) if err != nil { @@ -181,8 +213,8 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit } switch { case res.StatusCode == http.StatusUnauthorized && c.TokenFile != "": - return nil, fmt.Errorf("the forge rejected the tools' token for %s (401): the module mints it afresh at its next call; nothing was done: %w", - method+" "+path, &StatusError{method, path, res.StatusCode, "unauthorized"}) + return nil, fmt.Errorf("the forge rejected %s's token for %s (401): the module mints it afresh; nothing was done: %w", + c.Account, method+" "+path, &StatusError{method, path, res.StatusCode, "unauthorized"}) case res.StatusCode == http.StatusNotFound: return nil, fmt.Errorf("%s %s: %w", method, path, ErrNotFound) case res.StatusCode >= 300: diff --git a/modules/gitea/internal/packageseats/holder.go b/modules/gitea/internal/packageseats/holder.go new file mode 100644 index 00000000..7a3706a3 --- /dev/null +++ b/modules/gitea/internal/packageseats/holder.go @@ -0,0 +1,348 @@ +// Package packageseats is the forge module's holding of the two package seats, `npm-package-registry` and +// `go-package-registry` (novox/hq design 51 *The four registry seats*, ADR 0605, ADR 0611). A build module publishes +// a package through `publish` and `publish-part`: the package's one file lands in the forge module's own object store +// `incoming`, is hashed, checked against the builder's numbering and then uploaded into the forge's package registry +// with the module's own write side. The receiving side is mesh-sdk's seat-neutral parts.Receiver and the caller list +// mesh-sdk's VerbCallers (ADR 0627 §2): this package binds both to the two seats and to the forge. +package packageseats + +import ( + "errors" + "fmt" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/parts" +) + +// The seats the forge module holds. +const ( + NpmSeat = "npm-package-registry" + GoSeat = "go-package-registry" +) + +// Seats are the package seats, each with the forge's own package type behind it. +var Seats = map[string]string{NpmSeat: "npm", GoSeat: "go"} + +// The verbs both seats serve (design 51 *One contract*). +const ( + VerbPublish = "publish" + VerbPublishPart = "publish-part" + VerbVersions = "versions" + VerbRemove = "remove" +) + +// Verbs are the verbs of both seats, in order. +var Verbs = []string{VerbPublish, VerbPublishPart, VerbRemove, VerbVersions} + +// The events of both seats. +const ( + EventPublished = "published" + EventRefused = "refused" + EventRemoved = "removed" +) + +// The bounds of incoming (design 51 *Publishing*, ADR 0611): a chunk at most 1 MiB, a transfer with no chunk for 24 +// hours dropped, a finding past 80 % of the cap. A package is one file, far smaller than an image, so the cap is the +// manifest's 2 GiB. +const ( + MaxChunk = 1 << 20 + IncomingCap = 2 << 30 + TransferIdle = 24 * time.Hour + IncomingFullPct = 80 + IncomingObjects = "incoming" + TransfersState = "transfers" + // KindPackage is the one kind a package seat holds. + KindPackage = "package" + // Self is the forge module, the holder every outcome names. + Self = "gitea" + // ReasonRetention is the reason of the forge's own retention removing a version nothing keeps. + ReasonRetention = "retention" +) + +// Callers is the trusted config value `callers`, read as mesh-sdk's list per verb. +type Callers = stdio.VerbCallers + +// DefaultCallers is design 51's list for the package seats: publish and publish-part the two build modules that +// publish packages; remove the builder (a tagged version whose branch ended); versions anyone on the mesh, as the +// registry's reads are anonymous. +func DefaultCallers() Callers { + builds := []string{"mesh-build-go", "mesh-build-node"} + return Callers{VerbPublish: builds, VerbPublishPart: builds, VerbRemove: {"mesh-delivery-builder"}, + VerbVersions: {stdio.AnyCaller}} +} + +// ConfiguredCallers is the config value `callers` as mesh-config resolved it; unset is DefaultCallers. +func ConfiguredCallers() (Callers, error) { + return stdio.ConfiguredVerbCallers("callers", Verbs, DefaultCallers()) +} + +// Publish is a publish as the seats' contract names it (mesh-contracts npmpackageregistry and gopackageregistry share +// one shape): what the tools bundle reads from either seat's PublishInput. +type Publish struct { + Transfer string `json:"transfer"` + Name string `json:"name"` + Version string `json:"version,omitempty"` + Tag string `json:"tag,omitempty"` + Kind string `json:"kind"` + Fingerprint string `json:"fingerprint,omitempty"` + Commit string `json:"commit,omitempty"` + Digest string `json:"digest"` + Parts []parts.Want `json:"parts,omitempty"` +} + +// Opening is what the holder keeps of a publish beside the receiving side's own fields. +type Opening struct { + Publish + Seat string `json:"seat"` +} + +// Transfer is one open publish in the state transfers. +type Transfer = parts.Transfer[Opening] + +// Outcome is the body of published, refused and removed, as the contract's Outcome names it. +type Outcome struct { + Name, Version, Kind, Tag, Digest, Fingerprint, Commit, Transfer, Reference, Reason string + // Module is the Go module path a Go package is fetched under; empty on the npm seat. + Module string +} + +// Answer is what publish and publish-part answer: the transfer, the parts lacked, and whether it ended. +type Answer struct { + Transfer string + Lacks []string + Ended bool +} + +// Version is one version a seat holds of a name. +type Version struct { + Version string + Digest string + Published time.Time +} + +// Builder answers whether the builder numbered a version with a digest (mesh-sdk delivery.BuilderVersions). +type Builder interface { + Recorded(name, version, digest string) (ok bool, why string, err error) +} + +// Store is the forge's package registry, written with the module's own write side alone. +type Store interface { + // Held is the sha256 digest of name@version's file when the registry holds it. + Held(seat, name, version string) (digest string, ok bool, err error) + // Put uploads the package's file and answers the Go module path (empty for npm). + Put(seat string, p Publish, file []byte) (module string, err error) + // Versions is every version of name, newest first. + Versions(seat, name string) ([]Version, error) + // Remove deletes name@version. + Remove(seat, name, version string) error +} + +// Holder answers the verbs of both seats. +type Holder struct { + Callers Callers + Objects parts.Objects + Transfers parts.Transfers[Opening] + Builder Builder + Store Store + // Say says an event on a seat's family. + Say func(seat, event string, o Outcome) error + // Finding writes (raised) or clears the module's finding of a kind for a subject. + Finding func(kind, subject, words string, raised bool) + Now func() time.Time + Cap uint64 +} + +func (h *Holder) now() time.Time { + if h.Now != nil { + return h.Now().UTC() + } + return time.Now().UTC() +} + +// Reference is the seat's answer for a package: @ (ADR 0605 §9). +func Reference(name, version string) string { return name + "@" + version } + +// key is a transfer's key: the caller's own name under its seat, so the two seats' transfers never meet. +func key(seat, transfer string) string { return seat + "_" + transfer } + +func (h *Holder) receiver() *parts.Receiver[Opening] { + cap := h.Cap + if cap == 0 { + cap = IncomingCap + } + return &parts.Receiver[Opening]{Objects: h.Objects, Transfers: h.Transfers, Cap: cap, MaxChunk: MaxChunk, + Idle: TransferIdle, Full: IncomingFullPct, Now: h.now, + // The builder's numbering is asked before anything is uploaded (ADR 0605 §4). + Check: func(t Transfer) error { + ok, why, err := h.Builder.Recorded(t.Shape.Name, t.Shape.Version, t.Shape.Digest) + if err != nil { + return errors.New("builder_versions did not answer, so nothing is stored: " + err.Error()) + } + if !ok { + return errors.New("the builder did not number it: " + why) + } + return nil + }, + Store: func(t Transfer, held map[string][]byte) error { + file := held[t.Shape.Digest] + module, err := h.Store.Put(t.Shape.Seat, t.Shape.Publish, file) + if err != nil { + reason := "the forge did not take it: " + err.Error() + return errors.Join(errors.New("refused: "+reason), h.refused(t, reason)) + } + h.Finding("publish-refused", t.Shape.Name+"@"+t.Shape.Version, "", false) + h.Finding("publish-refused", IncomingObjects, "", false) + o := outcomeOf(t.Shape.Publish, "") + o.Reference, o.Module = Reference(t.Shape.Name, t.Shape.Version), module + return h.Say(t.Shape.Seat, EventPublished, o) + }, + Refuse: h.refused, + Filled: func(used, cap uint64, full bool) { + h.Finding("incoming-full", IncomingObjects, fmt.Sprintf("incoming holds %d of its %d bytes (%d %%)", used, cap, used*100/cap), full) + }} +} + +func (h *Holder) refused(t Transfer, reason string) error { + subject := t.Shape.Name + "@" + t.Shape.Version + if parts.RefusedAtCap(reason) { + subject = IncomingObjects + } + h.Finding("publish-refused", subject, reason, true) + return h.Say(t.Shape.Seat, EventRefused, outcomeOf(t.Shape.Publish, reason)) +} + +func outcomeOf(p Publish, reason string) Outcome { + return Outcome{Name: p.Name, Version: p.Version, Kind: p.Kind, Tag: p.Tag, Digest: p.Digest, Fingerprint: p.Fingerprint, + Commit: p.Commit, Transfer: p.Transfer, Reason: reason} +} + +func (h *Holder) answer(transfer string, r parts.Received) Answer { + return Answer{Transfer: transfer, Lacks: r.Lacks, Ended: r.Ended} +} + +// Publish is `publish` on seat for the proved caller: a package is one file, its digest the file's sha256. +func (h *Holder) Publish(seat, caller string, p Publish) (Answer, error) { + if err := h.Callers.Allowed(VerbPublish, caller); err != nil { + return Answer{}, err + } + if err := checkPublish(seat, p); err != nil { + return Answer{}, err + } + _, _ = h.receiver().DropIdle() + // A rebuild of a published version answers it held and uploads nothing; the same version with other content is + // refused, as the registry would refuse it. + if d, ok, err := h.Store.Held(seat, p.Name, p.Version); err != nil { + return Answer{}, fmt.Errorf("publish: the forge cannot say whether it holds %s@%s: %w", p.Name, p.Version, err) + } else if ok { + if d != p.Digest { + reason := fmt.Sprintf("%s@%s is published with %s, not %s", p.Name, p.Version, d, p.Digest) + return Answer{Transfer: p.Transfer, Ended: true}, h.refused(Transfer{Shape: Opening{Publish: p, Seat: seat}}, reason) + } + o := outcomeOf(p, "") + o.Reference = Reference(p.Name, p.Version) + return Answer{Transfer: p.Transfer, Ended: true}, h.Say(seat, EventPublished, o) + } + // The transfer's name is the caller's own (the seat's contract): none would open the seat's one nameless transfer, + // which key() still makes a valid id. + if strings.TrimSpace(p.Transfer) == "" { + return Answer{}, errors.New("publish: no transfer named; the caller names its own") + } + got, err := h.receiver().Open(key(seat, p.Transfer), caller, p.Parts, Opening{Publish: p, Seat: seat}) + if err != nil { + return h.answer(p.Transfer, got), fmt.Errorf("publish: %w", err) + } + return h.answer(p.Transfer, got), nil +} + +// PublishPart is `publish-part`: one chunk, from the caller that opened the transfer alone. +func (h *Holder) PublishPart(seat, caller, transfer, part string, sequence int, last bool, data []byte) (Answer, error) { + if err := h.Callers.Allowed(VerbPublishPart, caller); err != nil { + return Answer{}, err + } + got, err := h.receiver().Take(key(seat, transfer), caller, part, sequence, last, data, func(t Transfer) error { + if t.Shape.Seat != seat { + return fmt.Errorf("transfer %s is %s's, not %s's", transfer, t.Shape.Seat, seat) + } + return nil + }) + if err != nil { + return h.answer(transfer, got), fmt.Errorf("publish-part: %w", err) + } + return h.answer(transfer, got), nil +} + +// Versions is `versions`: every version the seat holds of name, newest first. +func (h *Holder) Versions(seat, caller, name string) ([]Version, error) { + if err := h.Callers.Allowed(VerbVersions, caller); err != nil { + return nil, err + } + if _, ok := Seats[seat]; !ok { + return nil, fmt.Errorf("versions: the forge holds no seat %s", seat) + } + if strings.TrimSpace(name) == "" { + return nil, errors.New("versions: no name") + } + v, err := h.Store.Versions(seat, name) + if err != nil { + return nil, fmt.Errorf("versions: the forge cannot list %s: %w", name, err) + } + return v, nil +} + +// Remove is `remove`: the builder removes a tagged version (its branch ended); an untagged version goes only by the +// forge's own retention, never by a call. A version already gone is said removed all the same, so a retry ends. +func (h *Holder) Remove(seat, caller, name, version, reason string) (Outcome, error) { + if err := h.Callers.Allowed(VerbRemove, caller); err != nil { + return Outcome{}, err + } + if _, ok := Seats[seat]; !ok { + return Outcome{}, fmt.Errorf("remove: the forge holds no seat %s", seat) + } + if strings.TrimSpace(name) == "" || strings.TrimSpace(reason) == "" { + return Outcome{}, errors.New("remove: a name and a reason are required") + } + if !strings.Contains(version, "-") { + return Outcome{}, fmt.Errorf("remove: refused: %s@%s has no tag; only a tagged version is removed, an untagged one by retention alone", name, version) + } + d, held, err := h.Store.Held(seat, name, version) + if err != nil { + return Outcome{}, fmt.Errorf("remove: the forge cannot read %s@%s: %w", name, version, err) + } + if held { + if err := h.Store.Remove(seat, name, version); err != nil { + return Outcome{}, fmt.Errorf("remove: the forge did not remove %s@%s: %w", name, version, err) + } + } + o := Outcome{Name: name, Version: version, Kind: KindPackage, Digest: d, Reason: reason} + return o, h.Say(seat, EventRemoved, o) +} + +// Retained says `removed` for a version the forge's own retention deleted (design 51 *Retention*). +func (h *Holder) Retained(seat, name, version, digest string) error { + return h.Say(seat, EventRemoved, Outcome{Name: name, Version: version, Kind: KindPackage, Digest: digest, Reason: ReasonRetention}) +} + +// Drop drops a transfer by hand; DropIdle every transfer idle for TransferIdle. +func (h *Holder) Drop(seat, transfer string) (Transfer, error) { + return h.receiver().Drop(key(seat, transfer)) +} + +func checkPublish(seat string, p Publish) error { + switch { + case Seats[seat] == "": + return fmt.Errorf("publish: the forge holds no seat %s", seat) + case strings.TrimSpace(p.Name) == "": + return errors.New("publish: no name") + case strings.TrimSpace(p.Version) == "": + return errors.New("publish: a package is published under the version the builder gave it") + case p.Kind != KindPackage: + return fmt.Errorf("publish: %s holds packages, not a %s", seat, p.Kind) + case !strings.HasPrefix(p.Digest, "sha256:"): + return fmt.Errorf("publish: digest %q is no sha256 digest", p.Digest) + case len(p.Parts) != 1 || p.Parts[0].Digest != p.Digest: + return errors.New("publish: a package is one file, its one part the package's digest") + } + return nil +} diff --git a/modules/gitea/internal/packageseats/holder_test.go b/modules/gitea/internal/packageseats/holder_test.go new file mode 100644 index 00000000..41aeaae9 --- /dev/null +++ b/modules/gitea/internal/packageseats/holder_test.go @@ -0,0 +1,249 @@ +package packageseats + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/parts" +) + +type memObjects map[string][]byte + +func (m memObjects) Put(n string, d []byte) error { m[n] = append([]byte(nil), d...); return nil } +func (m memObjects) Get(n string) ([]byte, error) { + if d, ok := m[n]; ok { + return d, nil + } + return nil, errors.New("no object " + n) +} +func (m memObjects) Delete(n string) error { delete(m, n); return nil } +func (m memObjects) Used() (uint64, error) { + var u uint64 + for _, d := range m { + u += uint64(len(d)) + } + return u, nil +} + +type memTransfers map[string]Transfer + +func (m memTransfers) Get(id string) (Transfer, bool, error) { t, ok := m[id]; return t, ok, nil } +func (m memTransfers) Put(t Transfer) error { m[t.ID] = t; return nil } +func (m memTransfers) Delete(id string) error { delete(m, id); return nil } +func (m memTransfers) List() ([]Transfer, error) { + var out []Transfer + for _, t := range m { + out = append(out, t) + } + return out, nil +} + +type builder struct{ ok bool } + +func (b builder) Recorded(_, _, _ string) (bool, string, error) { return b.ok, "not numbered", nil } + +// forge is the forge's package registry: seat -> name@version -> file. +type forge struct { + files map[string]map[string][]byte + removed []string +} + +func (f *forge) Held(seat, name, version string) (string, bool, error) { + b, ok := f.files[seat][name+"@"+version] + if !ok { + return "", false, nil + } + return digest(b), true, nil +} +func (f *forge) Put(seat string, p Publish, file []byte) (string, error) { + if f.files[seat] == nil { + f.files[seat] = map[string][]byte{} + } + f.files[seat][p.Name+"@"+p.Version] = file + if seat == GoSeat { + return "git.example/" + p.Name, nil + } + return "", nil +} +func (f *forge) Versions(seat, name string) ([]Version, error) { + var out []Version + for k, b := range f.files[seat] { + if n, v, _ := strings.Cut(k, "@"); n == name { + out = append(out, Version{Version: v, Digest: digest(b)}) + } + } + return out, nil +} +func (f *forge) Remove(seat, name, version string) error { + delete(f.files[seat], name+"@"+version) + f.removed = append(f.removed, seat+" "+name+"@"+version) + return nil +} + +func digest(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) } + +type said struct { + seat, event string + o Outcome +} + +type bed struct { + h *Holder + forge *forge + said []said + findings map[string]bool +} + +func newBed(numbered bool) *bed { + b := &bed{forge: &forge{files: map[string]map[string][]byte{}}, findings: map[string]bool{}} + b.h = &Holder{Callers: DefaultCallers(), Objects: memObjects{}, Transfers: memTransfers{}, Builder: builder{ok: numbered}, + Store: b.forge, Now: func() time.Time { return time.Unix(1e9, 0) }, + Say: func(seat, event string, o Outcome) error { b.said = append(b.said, said{seat, event, o}); return nil }, + Finding: func(kind, subject, _ string, raised bool) { b.findings[kind+" "+subject] = raised }} + return b +} + +func pub(name, version string, file []byte) Publish { + return Publish{Transfer: "t-" + name, Name: name, Version: version, Kind: KindPackage, Digest: digest(file), + Parts: []parts.Want{{Digest: digest(file), Size: int64(len(file))}}} +} + +const goBuild, nodeBuild = "ace.mesh-build-go", "ace.mesh-build-node" + +// **The forge module holds both package seats** (design 51 *Publishing*, *Testing* for the holder): a package's one +// file comes in parts, is checked against the builder's numbering and its digest, and is uploaded with the forge's own +// write side; the outcome is said on the seat's events. +func TestTheForgeHoldsBothPackageSeats(t *testing.T) { + file := []byte("a package's tarball") + t.Run("a publish in parts is uploaded and said published", func(t *testing.T) { + for _, seat := range []string{NpmSeat, GoSeat} { + b := newBed(true) + a, err := b.h.Publish(seat, goBuild, pub("mesh-sdk", "0.1.15", file)) + if err != nil || len(a.Lacks) != 1 || a.Transfer != "t-mesh-sdk" { + t.Fatalf("%s: publish answered %+v, %v", seat, a, err) + } + a, err = b.h.PublishPart(seat, goBuild, "t-mesh-sdk", digest(file), 0, true, file) + if err != nil || !a.Ended { + t.Fatalf("%s: the last chunk answered %+v, %v", seat, a, err) + } + if len(b.said) != 1 || b.said[0].event != EventPublished || b.said[0].o.Reference != "mesh-sdk@0.1.15" { + t.Fatalf("%s: said %+v", seat, b.said) + } + if (seat == GoSeat) != (b.said[0].o.Module != "") { + t.Errorf("%s: the Go module path said is %q", seat, b.said[0].o.Module) + } + if len(b.h.Objects.(memObjects)) != 0 || len(b.h.Transfers.(memTransfers)) != 0 { + t.Errorf("%s: the transfer's chunks or key stayed after its end", seat) + } + if v, _ := b.h.Versions(seat, "g14.anyone", "mesh-sdk"); len(v) != 1 || v[0].Digest != digest(file) { + t.Errorf("%s: versions answered %+v", seat, v) + } + // A rebuild of the published version is held: nothing uploaded, published said again. + if a, err := b.h.Publish(seat, goBuild, pub("mesh-sdk", "0.1.15", file)); err != nil || !a.Ended || len(a.Lacks) != 0 { + t.Errorf("%s: a held version answered %+v, %v", seat, a, err) + } + } + }) + t.Run("refusals", func(t *testing.T) { + for _, c := range []struct { + name string + do func(b *bed) error + want string + }{ + {"a caller not on publish's list", func(b *bed) error { + _, err := b.h.Publish(NpmSeat, "ace.claude-code", pub("x", "0.1.0", file)) + return err + }, "not on the holder's caller list"}, + {"an unproven caller", func(b *bed) error { + _, err := b.h.Publish(NpmSeat, stdio.Unproven, pub("x", "0.1.0", file)) + return err + }, "no caller the tool runner proved"}, + {"a seat the forge does not hold", func(b *bed) error { + _, err := b.h.Publish("cargo-package-registry", goBuild, pub("x", "0.1.0", file)) + return err + }, "holds no seat"}, + {"a package in two files", func(b *bed) error { + p := pub("x", "0.1.0", file) + p.Parts = append(p.Parts, parts.Want{Digest: digest([]byte("b")), Size: 1}) + _, err := b.h.Publish(NpmSeat, goBuild, p) + return err + }, "one file"}, + {"a part from another module than its opener", func(b *bed) error { + if _, err := b.h.Publish(NpmSeat, nodeBuild, pub("x", "0.1.0", file)); err != nil { + return err + } + _, err := b.h.PublishPart(NpmSeat, goBuild, "t-x", digest(file), 0, true, file) + return err + }, "opened"}, + {"a transfer of the other seat", func(b *bed) error { + if _, err := b.h.Publish(NpmSeat, nodeBuild, pub("x", "0.1.0", file)); err != nil { + return err + } + _, err := b.h.PublishPart(GoSeat, nodeBuild, "t-x", digest(file), 0, true, file) + return err + }, "not open"}, + {"remove by a build module", func(b *bed) error { _, err := b.h.Remove(GoSeat, goBuild, "x", "0.1.1-feat.1", "tag-gone"); return err }, "not on the holder's caller list"}, + {"remove of a version with no tag", func(b *bed) error { + _, err := b.h.Remove(GoSeat, "novox.mesh-delivery-builder", "x", "0.1.1", "tag-gone") + return err + }, "has no tag"}, + } { + b := newBed(true) + if err := c.do(b); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: %v, want %q", c.name, err, c.want) + } + } + }) + t.Run("a refused publish is said refused and raises the finding; nothing is uploaded", func(t *testing.T) { + for _, c := range []struct { + name string + numbered bool + data []byte + want string + }{{"the builder did not number it", false, file, "did not number"}} { + b := newBed(c.numbered) + if _, err := b.h.Publish(NpmSeat, nodeBuild, pub("x", "0.1.0", file)); err != nil { + t.Fatal(err) + } + _, _ = b.h.PublishPart(NpmSeat, nodeBuild, "t-x", digest(file), 0, true, c.data) + if len(b.said) != 1 || b.said[0].event != EventRefused || !strings.Contains(b.said[0].o.Reason, c.want) { + t.Errorf("%s: said %+v", c.name, b.said) + } + if !b.findings["publish-refused x@0.1.0"] || len(b.forge.files[NpmSeat]) != 0 { + t.Errorf("%s: finding %v, uploaded %v", c.name, b.findings, b.forge.files) + } + } + // A part whose bytes do not match its digest is refused to its sender, to send again; nothing is said yet. + b := newBed(true) + if _, err := b.h.Publish(NpmSeat, nodeBuild, pub("x", "0.1.0", file)); err != nil { + t.Fatal(err) + } + if _, err := b.h.PublishPart(NpmSeat, nodeBuild, "t-x", digest(file), 0, true, []byte("other bytes")); err == nil || + !strings.Contains(err.Error(), "send the part again") || len(b.said) != 0 || len(b.forge.files[NpmSeat]) != 0 { + t.Errorf("a part with other bytes: %v, said %+v", err, b.said) + } + // The same version published with other content is refused, never uploaded over. + b = newBed(true) + b.forge.files[NpmSeat] = map[string][]byte{"x@0.1.0": []byte("first")} + if _, err := b.h.Publish(NpmSeat, nodeBuild, pub("x", "0.1.0", file)); err != nil || len(b.said) != 1 || b.said[0].event != EventRefused { + t.Errorf("a published version with other content: %v, said %+v", err, b.said) + } + }) + t.Run("the builder removes a tagged version; one already gone is said removed all the same", func(t *testing.T) { + b := newBed(true) + b.forge.files[GoSeat] = map[string][]byte{"x@0.1.1-feat.1": file} + for range 2 { + if _, err := b.h.Remove(GoSeat, "novox.mesh-delivery-builder", "x", "0.1.1-feat.1", "tag-gone"); err != nil { + t.Fatal(err) + } + } + if len(b.forge.removed) != 1 || len(b.said) != 2 || b.said[0].event != EventRemoved || b.said[0].o.Digest != digest(file) { + t.Errorf("removed %v, said %+v", b.forge.removed, b.said) + } + }) +} diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 9348fe3b..0caba4c2 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -5,7 +5,28 @@ "requires": [ "postgres-database", "route", - "secret" + "secret", + { + "name": "incoming", + "storage": "object-store", + "template": "objects", + "options": { + "max-mib": 2048 + } + }, + { + "name": "transfers", + "storage": "key-value", + "template": "state", + "options": { + "ttl-seconds": 86400 + } + }, + { + "name": "findings", + "storage": "key-value", + "template": "state" + } ], "contributes": { "postgres-database": { @@ -99,7 +120,7 @@ "port": 22, "protocol": "tcp", "from": "mesh", - "why": "git over ssh, gitea's own unmodified sshd. Published on the machine's own side at 222, the mesh's fixed public convention — not 22, which the machine's own daemon holds and a module does not take" + "why": "git over ssh, gitea's own unmodified sshd. Published on the machine's own side at 222, the mesh's fixed public convention \u2014 not 22, which the machine's own daemon holds and a module does not take" }, { "name": "forge-events", @@ -119,18 +140,33 @@ "git": { "scheme": "http", "port": 3000 + }, + "go-package-registry": { + "scheme": "http", + "port": 3000, + "go-path": "/api/packages/novox/go" } }, - "receives": { - "npm-package-registry": "${dir:grants}/npm.json" - }, - "grants": { - "npm-package-registry": "${dir:grants}" - }, "claims": [ { "name": "npm-package-registry", - "scope": "mesh" + "scope": "mesh", + "serves": [ + "publish", + "publish-part", + "remove", + "versions" + ] + }, + { + "name": "go-package-registry", + "scope": "mesh", + "serves": [ + "publish", + "publish-part", + "remove", + "versions" + ] }, { "name": "git", @@ -145,14 +181,7 @@ "class": "valuable", "why": "every repository, its issues and attachments, and the package registry" } - ], - "consumers": { - "npm-package-registry": { - "class": "rebuildable", - "in": "data", - "why": "a consumer's packages are published again from its source" - } - } + ] }, "config": { "url": { @@ -169,7 +198,7 @@ }, "keep-main-rules": { "type": "boolean", - "description": "Whether the module keeps main's branch protection as the mesh wants it. True: every ten minutes, and at the start, main's rule in every repository of the organisation novox is put to no push and the tools' account alone merging, and each apply of the `repositories` value narrows the main rule of the repositories it lists, requiring no status. False (the default): the module changes no existing repository's branch protection, whoever set it at the forge, but takes away a required status it finds on a rule covering main, once (ADR 0534 §1); a repository listed in `repositories` that does not exist is still made, with its first protection.", + "description": "Whether the module keeps main's branch protection as the mesh wants it. True: every ten minutes, and at the start, main's rule in every repository of the organisation novox is put to no push and the tools' account alone merging, and each apply of the `repositories` value narrows the main rule of the repositories it lists, requiring no status. False (the default): the module changes no existing repository's branch protection, whoever set it at the forge, but takes away a required status it finds on a rule covering main, once (ADR 0534 \u00a71); a repository listed in `repositories` that does not exist is still made, with its first protection.", "trusted": true, "default": false }, @@ -181,15 +210,23 @@ }, "check-gates-merge": { "type": "boolean", - "description": "Whether gitea_merge_pull_request judges mesh/check before it merges. True (the default): a head merges only on a passing mesh/check judged against the trunk's head (ADR 0447, ADR 0534 §4). False (the operator, 2026-10-25 22:56: \"run the check locally only, but not as a gate, it loses us too much time\"): a head the operator's Approve names in approved_heads merges whatever its mesh/check says (pending, failed, missing or judged on an older trunk); a head that moved, a branch no rule guards and a merge git refuses are still refused. Set together with mesh-delivery-gate's check-gates-merge.", + "description": "Whether gitea_merge_pull_request judges mesh/check before it merges. True (the default): a head merges only on a passing mesh/check judged against the trunk's head (ADR 0447, ADR 0534 \u00a74). False (the operator, 2026-10-25 22:56: \"run the check locally only, but not as a gate, it loses us too much time\"): a head the operator's Approve names in approved_heads merges whatever its mesh/check says (pending, failed, missing or judged on an older trunk); a head that moved, a branch no rule guards and a merge git refuses are still refused. Set together with mesh-delivery-gate's check-gates-merge.", "trusted": true, "default": true }, "main-mergers": { "type": "list", - "description": "The forge logins that may merge into main beside the tools' account, on every repository of the organisation (ADR 0534 §2): the operator's own, so the operator can always merge by hand at the forge. The tools' account is always on main's merge whitelist; no agent's account belongs here.", + "description": "The forge logins that may merge into main beside the tools' account, on every repository of the organisation (ADR 0534 \u00a72): the operator's own, so the operator can always merge by hand at the forge. The tools' account is always on main's merge whitelist; no agent's account belongs here.", "trusted": true, - "default": ["jschoubben"] + "default": [ + "jschoubben" + ] + }, + "callers": { + "type": "string", + "trusted": true, + "default": "", + "description": "per verb of the two package seats, the callers that may call it (JSON object of verb to callers; * every proved caller, operator the operator's session); empty is design 51's list: publish and publish-part mesh-build-go and mesh-build-node, remove mesh-delivery-builder, versions anyone" } }, "facts": { @@ -228,12 +265,6 @@ "place": ".", "unknown": "program" }, - { - "id": "grants", - "type": "directory", - "mode": "0700", - "unknown": "program" - }, { "id": "server-env", "type": "file", @@ -397,16 +428,23 @@ "provides": [ { "name": "npm-package-registry", - "credential": {"form": "pair"}, - "scope": "mesh", - "identity": { - "max": 40, - "in": "a Gitea user name" - } + "credential": { + "form": "none" + }, + "scope": "mesh" + }, + { + "name": "go-package-registry", + "credential": { + "form": "none" + }, + "scope": "mesh" }, { "name": "git", - "credential": {"form": "none"}, + "credential": { + "form": "none" + }, "scope": "mesh" }, { @@ -419,7 +457,9 @@ }, { "name": "forge-account", - "credential": {"form": "none"}, + "credential": { + "form": "none" + }, "scope": "mesh", "answered-by": "helper" } @@ -432,13 +472,11 @@ "language": "typescript", "entrypoints": [ "index.js", - "tools/index.js", - "provisioner/index.js" + "tools/index.js" ], "loads": [ "index.js", - "tools/index.js", - "provisioner/index.js" + "tools/index.js" ], "env": { "MESH_GITEA_URL": "http://127.0.0.1:${port:3000}", @@ -447,26 +485,27 @@ "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret", "MESH_GITEA_TOOLS_USER": "mesh-tools", "MESH_GITEA_STATE_DIR": "${dir:runtime-state}", - "MESH_RECEIVES": "${dir:grants}/npm.json", "MESH_GITEA_HOOK_URL": "http://172.17.0.1:3047", "MESH_GITEA_HOOK_PORT": "3047", "MESH_GITEA_HOOK_HOST": "172.17.0.1" } }, { - "name": "npm-registry", + "name": "package-registry", "kind": "bundle", "language": "go", "system": "arch", - "from": "cmd/npm-registry", - "binary": "npm-registry", + "from": "cmd/package-registry", + "binary": "package-registry", "loads": [ - "npm-registry" + "package-registry" ], "env": { "MESH_GITEA_URL": "http://127.0.0.1:${port:3000}", - "MESH_GITEA_ADMIN_USER": "mesh-admin", - "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret", + "MESH_GITEA_TOOLS_USER": "mesh-tools", + "MESH_GITEA_TOKEN_FILE": "${dir:runtime-state}/token", + "MESH_GITEA_PACKAGES_USER": "mesh-packages", + "MESH_GITEA_PACKAGES_TOKEN_FILE": "${dir:runtime-state}/packages-token", "MESH_NPM_OWNER": "novox" } }, @@ -577,7 +616,8 @@ ], "invokes": [ "mesh-delivery-indexer.indexer_read_modules", - "seat:mesh-hand-acts.hand-act" + "seat:mesh-hand-acts.hand-act", + "mesh-delivery-builder.builder_versions" ], "consumes": [ "mesh-delivery-indexer.views.modules" diff --git a/modules/gitea/package.json b/modules/gitea/package.json index 8c6171fe..8e33075b 100644 --- a/modules/gitea/package.json +++ b/modules/gitea/package.json @@ -5,7 +5,7 @@ "type": "module", "private": true, "scripts": { - "build": "tsc client.ts token.ts index.ts provisioner/index.ts provisioner/teams.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist", + "build": "tsc client.ts token.ts index.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist", "test": "npm run build && tsc --module NodeNext --moduleResolution NodeNext --target ES2022 --types node --rootDir . --outDir . $(find . -name '*.ts' ! -name '*.d.ts' ! -path './test/*' ! -path './node_modules/*' ! -path './dist/*') && node --test --experimental-strip-types 'test/*.test.ts'" }, "dependencies": { diff --git a/modules/gitea/packages-account.ts b/modules/gitea/packages-account.ts new file mode 100644 index 00000000..ffbfab7b --- /dev/null +++ b/modules/gitea/packages-account.ts @@ -0,0 +1,113 @@ +// The package seats write as an account of their own, never the administrator (design 51: each holder keeps its write +// side as an own secret beside a read-only tool token; the forge module's is `write:package`, its tool token +// `read:package`). +// +// The Go bundle (cmd/package-registry) serves both package seats' verbs: it reads the registry with the tools' token +// (token.ts, which carries `read:package`) and writes (a publish, a remove, retention's deletion) with this one: +// +// - **the account**: `mesh-packages`, plain, made by the module with the administrator's login as tools-account.ts +// makes `mesh-tools`; its password random, sent once and kept nowhere. +// - **the team**: `mesh-packages` in the organisation that owns the registry, writing packages and nothing else: +// no code, no issues, no pull requests, no administration. +// - **the token**: minted for the account by the administrator's login with the one scope `write:package`, kept at +// 0600 in the module's runtime state as `packages-token`, its scopes line beside it saying whose it is +// (MintedToken: one mint at a time across the bundle's processes, re-minted when the kept one is gone or other). +// +// The watcher mints it at its start (keepPackagesToken). No timer looks at it after that: the Go bundle mints nothing, +// and when the forge refuses its write it raises the finding PACKAGES_TOKEN_REFUSED in the module's findings state; the +// watcher, watching that key, looks again (a refused token is minted afresh) and clears the finding (renewOnRefusal). + +import { randomBytes } from "node:crypto"; +import { join } from "node:path"; +import { MintedToken } from "./token.js"; +import { TOOLS_ORG, type ToolsForge } from "./tools-account.js"; + +/** The account the package seats write as (internal/forgehttp PackagesAccount). */ +export const PACKAGES_ACCOUNT = "mesh-packages"; +/** Its team in the registry's organisation, and the token's name in the account's list. */ +export const PACKAGES_TEAM = "mesh-packages"; +export const PACKAGES_TOKEN_NAME = "mesh-packages"; +/** The one scope the write side needs; reads go with the tools' token. */ +export const PACKAGES_TOKEN_SCOPES: readonly string[] = ["write:package"]; +/** Where the token is kept in the module's runtime state (the manifest's MESH_GITEA_PACKAGES_TOKEN_FILE). */ +export const PACKAGES_TOKEN_FILE = "packages-token"; + +/** What the team grants: write on the organisation's packages, and no more. */ +export const PACKAGES_TEAM_UNITS = { + units_map: { "repo.packages": "write" }, + includes_all_repositories: false, + can_create_org_repo: false, +} as const; + +/** Make the packages account, plain, a member of its team. Its password is set afresh each time and kept nowhere. */ +export async function setUpPackagesAccount(forge: ToolsForge, account = PACKAGES_ACCOUNT, password: () => string): Promise { + await forge.ensureUser(account, password(), `${account}@noreply.localhost`, true); + const team = await forge.ensureTeamWith(TOOLS_ORG, PACKAGES_TEAM, { name: PACKAGES_TEAM, ...PACKAGES_TEAM_UNITS }); + await forge.addUserToTeam(team, account); +} + +/** The packages account's token, minted by the administrator's login from what the manifest hands the watcher. */ +export function packagesToken(url: string, env: NodeJS.ProcessEnv = process.env, log?: (line: string) => void): MintedToken { + const admin = env.MESH_GITEA_ADMIN_USER; + const passwordFile = env.MESH_GITEA_ADMIN_PASSWORD_FILE; + const stateDir = env.MESH_GITEA_STATE_DIR; + const missing = [ + admin ? null : "MESH_GITEA_ADMIN_USER", + passwordFile ? null : "MESH_GITEA_ADMIN_PASSWORD_FILE", + stateDir ? null : "MESH_GITEA_STATE_DIR", + ].filter((v): v is string => v !== null); + if (missing.length) throw new Error(`the package seats' write token cannot be minted: ${missing.join(", ")} not set by the manifest`); + const account = env.MESH_GITEA_PACKAGES_USER || PACKAGES_ACCOUNT; + if (account === admin) throw new Error(`MESH_GITEA_PACKAGES_USER names the admin account "${admin}"; the package seats write as an account of their own`); + return new MintedToken({ + url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, PACKAGES_TOKEN_FILE), account, + name: PACKAGES_TOKEN_NAME, scopes: PACKAGES_TOKEN_SCOPES, log, + setUp: async (password) => { + const { GiteaAdmin } = await import("./client.js"); + await setUpPackagesAccount(new GiteaAdmin(url, admin!, password), account, () => randomBytes(24).toString("base64url")); + }, + }); +} + +/** + * One look at the packages token: mint it when none is kept, then ask the forge one read the scope covers (the + * organisation's package list); a 401 renews it. Answers what was done, for the watcher's line. + */ +export async function keepPackagesToken(source: MintedToken, url: string, owner = TOOLS_ORG, fetchImpl: typeof fetch = fetch): Promise { + const token = await source.current(); + const look = (t: string) => fetchImpl(`${url.replace(/\/+$/, "")}/api/v1/packages/${encodeURIComponent(owner)}?limit=1`, + { headers: { Authorization: `token ${t}`, Accept: "application/json" } }); + const res = await look(token); + if (res.status === 401 || MintedToken.lacksScope(res.status, await res.text())) { + const fresh = await source.renew(token); + const again = await look(fresh); + if (!again.ok) throw new Error(`the forge answered ${again.status} to the renewed package token's read`); + return "the forge rejected the package seats' write token; minted afresh"; + } + if (!res.ok) throw new Error(`the forge answered ${res.status} to the package token's read`); + return "the package seats' write token holds"; +} + +/** The finding the Go bundle raises in the module's findings state when the forge refuses the write token (cmd/package-registry PackagesTokenRefusedKey). */ +export const PACKAGES_TOKEN_REFUSED = "seat.gitea.packages-token-refused_write"; + +/** One change of the findings state, as its watch hands it. */ +export interface FindingChange { readonly key: string; readonly op: "put" | "delete" } + +/** + * The watcher's answer to the Go bundle's refusal: on the finding's put, look at the token (a refused one is minted + * afresh) and clear the finding; a look that fails leaves the finding standing, said. Other keys are not its own. + */ +export function renewOnRefusal(source: MintedToken, url: string, findings: { delete(key: string): Promise }, + log: (line: string) => void, fetchImpl: typeof fetch = fetch): (change: FindingChange) => Promise { + return async (change) => { + if (change.op !== "put" || change.key !== PACKAGES_TOKEN_REFUSED) return; + try { + log(await keepPackagesToken(source, url, TOOLS_ORG, fetchImpl)); + } catch (err) { + log(`the package seats' write token was refused and could not be minted afresh; the finding stands — ${err instanceof Error ? err.message : err}`); + return; + } + await findings.delete(change.key); + }; +} diff --git a/modules/gitea/provisioner/index.ts b/modules/gitea/provisioner/index.ts deleted file mode 100644 index 9efc083b..00000000 --- a/modules/gitea/provisioner/index.ts +++ /dev/null @@ -1,54 +0,0 @@ -// gitea's provisioner — the adapter that makes gitea a provider of the mesh -// `npm-package-registry` interface. The reconcile loop, the contributions file, and reading the -// mesh's minted password are the sdk harness's; this writes only the per-service half: how gitea -// creates and removes a consumer's npm credential (novox/hq ADR 0048/0076). -// -// **A package registry seat is one per ecosystem (novox/hq ADR 0109).** gitea holds the npm seat -// (ADR 0110). Adding cargo or PyPI is adding a provision — another `provides` entry, another -// `receives` path and another registration below — not widening this one. `git`, which gitea also -// provides, mints nothing and so registers nothing here: the mesh's own repositories are public, -// and a clone credential is not yet decided (ADR 0111). -// -// The `npm-package-registry` interface: a consumer authenticates to the npm registry at -// `/api/packages/novox/npm/` with basic auth, as `as` with the password the mesh minted, and can -// read packages under the `@novox` scope, and write them unless it asked for read only (teams.ts). The -// registry's npm owner is the gitea org `novox`; a consumer is a gitea *user* on one of that org's teams. -// -// **The user name and password are the mesh's, not the provisioner's (ADR 0048).** The mesh derives -// the login and hands it to both ends, and mints the password. gitea creates a user under exactly -// that login and sets exactly that password every run — so a rotation takes — and seals nothing: the -// consumer already has its copy through the mesh's own channel. -// -// The admin calls run through GiteaAdmin (basic auth as the mesh's gitea admin), which is the -// module's one boundary to the forge's admin API (see client.ts). - -import { runProvisioner, type Provision } from "@novox/mesh-sdk/provisioner"; -import { GiteaAdmin } from "../client.js"; -import { grant, holds } from "./teams.js"; - -// Which team a consumer joins — `packages` to publish, `readers` to read only — is teams.ts's. - -const gitea = GiteaAdmin.fromEnv(); - -// Where this registration's contributions land comes from $MESH_RECEIVES, never a path written -// here: the mesh writes the file where the manifest's `receives` says, and a second copy of that -// path in code would drift from it. One variable carries one path, so a second registration in this -// module needs the mesh to say where each provision's file is — not yet possible, and not faked. -runProvisioner("npm-package-registry", { - async create(p: Provision): Promise { - // The org and its teams are ensured per create: idempotent, and no separate bootstrap step. The - // user carries the consumer's login and the mesh's minted password, set every run so a rotation - // takes; membership of its team is what grants it what it may do. - await grant(gitea, p); - }, - - async remove(p: { as: string }): Promise { - // Login prohibited, never deleted (novox/hq issue 241: a withdrawal never destroys a consumer's data — on 2026-10-04 a misread grants file withdrew every consumer at once): deleting purges every repository the user owns. - await gitea.prohibitLogin(p.as); - }, - // Asked every minute by the harness: whether the backend still holds this consumer exactly as - // the mesh gave it, so a login lost behind the provisioner's back is made again (novox/hq issue 120). - async holds(p: Provision): Promise { - return holds(gitea, p); - }, -}); diff --git a/modules/gitea/provisioner/teams.ts b/modules/gitea/provisioner/teams.ts deleted file mode 100644 index 61100be8..00000000 --- a/modules/gitea/provisioner/teams.ts +++ /dev/null @@ -1,64 +0,0 @@ -// Which team of the forge's npm owner a consumer of `npm-package-registry` joins, and the per-consumer -// half of the provisioner around it — pure of the forge's address, so it is tested with a stand-in. -// -// **Two teams, by what the consumer said it needs** (novox/hq issue 624). A consumer that publishes — -// the build modules, whose builds publish the SDK and every TypeScript module's packages — joins `packages`: -// code read on the org's repositories and write on its packages. A consumer that contributes -// `{"access": "read"}` joins `readers`: code read and package read, and no write. The lab is the first: it -// clones branches that are not yet reviewed and runs their code beside its credential, and code that is -// not reviewed must not be able to publish the mesh's packages with it. - -/** The npm registry's owner: the gitea org every consumer's packages live under (ADR 0076). */ -export const ORG = "novox"; -/** The team whose members read and write the org's packages. */ -export const PACKAGE_TEAM = "packages"; -/** The team whose members read the org's code and packages, and write nothing. */ -export const READ_TEAM = "readers"; - -/** The team a consumer joins, from what it contributed; an access it may not ask for is refused. */ -export function teamFor(values: Readonly> | undefined): { team: string; packageWrite: boolean } { - const access = values?.access; - if (access === undefined || access === "write") return { team: PACKAGE_TEAM, packageWrite: true }; - if (access === "read") return { team: READ_TEAM, packageWrite: false }; - throw new Error(`npm-package-registry: access ${JSON.stringify(access)} is not "read" or "write"`); -} - -/** What the adapter asks of the forge's admin API (client.ts GiteaAdmin). */ -export interface Forge { - ensureOrg(org: string): Promise; - ensureTeam(org: string, team: string, packageWrite: boolean): Promise; - ensureUser(username: string, password: string, email: string): Promise; - addUserToTeam(teamId: number, username: string): Promise; - removeUserFromTeam(teamId: number, username: string): Promise; - holdsTeamMember(org: string, team: string, username: string, password: string): Promise; -} - -export interface Consumer { - readonly as: string; - readonly password: string; - readonly values: Readonly>; -} - -/** - * grant makes the consumer's user, with exactly the mesh's password, a member of its one team. A consumer - * that asks for read only is also taken out of the package team, so one that asked for write before and - * read now loses the write at once. - */ -export async function grant(forge: Forge, p: Consumer): Promise { - const { team, packageWrite } = teamFor(p.values); - await forge.ensureOrg(ORG); - const teamId = await forge.ensureTeam(ORG, team, packageWrite); - // The address is gitea's own convention for one that is not real: its email validation requires a - // dotted domain, so `@localhost` was refused at create, while `@noreply.localhost` is the shape gitea - // itself gives hidden addresses. - await forge.ensureUser(p.as, p.password, `${p.as}@noreply.localhost`); - await forge.addUserToTeam(teamId, p.as); - if (!packageWrite) { - await forge.removeUserFromTeam(await forge.ensureTeam(ORG, PACKAGE_TEAM, true), p.as); - } -} - -/** holds is whether the forge still holds the consumer as the mesh gave it: its login, in its team. */ -export async function holds(forge: Forge, p: Consumer): Promise { - return forge.holdsTeamMember(ORG, teamFor(p.values).team, p.as, p.password); -} diff --git a/modules/gitea/test/bundle-import.test.ts b/modules/gitea/test/bundle-import.test.ts index 5fb550b9..14f7786c 100644 --- a/modules/gitea/test/bundle-import.test.ts +++ b/modules/gitea/test/bundle-import.test.ts @@ -23,7 +23,7 @@ test("importing the bundle resolves while the broker keeps refusing the binds", await import(${JSON.stringify(index)}); console.log("imported"); process.exit(0);`; - const env = { ...process.env, MESH_GITEA_TOKEN: "x", MESH_GITEA_URL: "http://127.0.0.1:1" }; + const env = { ...process.env, GITEA_TOKEN: "x", MESH_GITEA_URL: "http://127.0.0.1:1" }; for (const k of ["MESH_GITEA_STATE_DIR", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD", "MESH_SERVED_MODULE"]) delete env[k]; const child = spawn(process.execPath, ["--input-type=module", "-e", script], { cwd: join(here, ".."), env, stdio: ["ignore", "pipe", "pipe"] }); let out = ""; @@ -86,7 +86,7 @@ test("a bundle whose read of the checker's owed is granted late writes the lates await new Promise((r) => setTimeout(r, 200)); console.log(JSON.stringify({ tools, bound, watched, statuses })); process.exit(0);`; - const env = { ...process.env, MESH_GITEA_TOKEN: "x" }; + const env = { ...process.env, GITEA_TOKEN: "x" }; for (const k of ["MESH_GITEA_STATE_DIR", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD", "MESH_SERVED_MODULE", "MESH_GITEA_URL"]) delete env[k]; const child = spawn(process.execPath, ["--input-type=module", "-e", script], { cwd: join(here, ".."), env, stdio: ["ignore", "pipe", "pipe"] }); let out = ""; @@ -141,7 +141,7 @@ test("importing the bundle resolves, its tools registered, while the forge never const { collectTools } = await import("@novox/mesh-sdk/tools"); console.log(JSON.stringify({ imported: true, tools: collectTools().flatMap((r) => r.tools.map((t) => t.name)) })); process.exit(0);`; - const env = { ...process.env, MESH_GITEA_TOKEN: "x", MESH_GITEA_ADMIN_USER: "admin", MESH_GITEA_HOOK_URL: "http://127.0.0.1:1" }; + const env = { ...process.env, GITEA_TOKEN: "x", MESH_GITEA_ADMIN_USER: "admin", MESH_GITEA_HOOK_URL: "http://127.0.0.1:1" }; for (const k of ["MESH_SERVED_MODULE", "MESH_GITEA_ADMIN_PASSWORD", "MESH_GITEA_CONFIG_FILE"]) delete env[k]; const child = spawn(process.execPath, ["--input-type=module", "-e", script], { cwd: join(here, ".."), env, stdio: ["ignore", "pipe", "pipe"] }); let out = ""; diff --git a/modules/gitea/test/teams.test.ts b/modules/gitea/test/teams.test.ts deleted file mode 100644 index 11def05e..00000000 --- a/modules/gitea/test/teams.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { test } from "node:test"; -import assert from "node:assert/strict"; - -import { grant, holds, PACKAGE_TEAM, READ_TEAM, teamFor, type Forge } from "../provisioner/teams.ts"; - -// Defends novox/hq issue 624: the lab reads the forge with a credential of its own that can write no -// package. A consumer that contributes {"access": "read"} joins a team with code and package read only. - -function aForge(): { forge: Forge; did: string[] } { - const did: string[] = []; - const teams: Record = {}; - const forge: Forge = { - async ensureOrg(org) { did.push(`org ${org}`); }, - async ensureTeam(org, team, packageWrite) { - did.push(`team ${org}/${team} packages:${packageWrite ? "write" : "read"}`); - return (teams[team] ??= Object.keys(teams).length + 1); - }, - async ensureUser(username) { did.push(`user ${username}`); }, - async addUserToTeam(id, username) { did.push(`add ${username} to ${id}`); }, - async removeUserFromTeam(id, username) { did.push(`remove ${username} from ${id}`); }, - async holdsTeamMember(org, team, username) { did.push(`holds ${username} in ${org}/${team}`); return true; }, - }; - return { forge, did }; -} - -test("a consumer that asks nothing, or write, publishes: the package team", () => { - assert.deepEqual(teamFor({}), { team: PACKAGE_TEAM, packageWrite: true }); - assert.deepEqual(teamFor({ access: "write" }), { team: PACKAGE_TEAM, packageWrite: true }); -}); - -test("a consumer that asks to read joins the readers, who write no package", () => { - assert.deepEqual(teamFor({ access: "read" }), { team: READ_TEAM, packageWrite: false }); - assert.throws(() => teamFor({ access: "admin" }), /not "read" or "write"/); -}); - -test("a reader is made, put on the readers team, and taken off the package team", async () => { - const { forge, did } = aForge(); - await grant(forge, { as: "lab-shanks", password: "pw", values: { access: "read" } }); - assert.deepEqual(did, [ - "org novox", - "team novox/readers packages:read", - "user lab-shanks", - "add lab-shanks to 1", - "team novox/packages packages:write", - "remove lab-shanks from 2", - ]); - did.length = 0; - await holds(forge, { as: "lab-shanks", password: "pw", values: { access: "read" } }); - assert.deepEqual(did, ["holds lab-shanks in novox/readers"]); -}); - -test("a publisher is made and put on the package team, as before", async () => { - const { forge, did } = aForge(); - await grant(forge, { as: "build-agent-ace", password: "pw", values: {} }); - assert.deepEqual(did, ["org novox", "team novox/packages packages:write", "user build-agent-ace", "add build-agent-ace to 1"]); -}); diff --git a/modules/gitea/test/token.test.ts b/modules/gitea/test/token.test.ts index c6399fce..54fc995f 100644 --- a/modules/gitea/test/token.test.ts +++ b/modules/gitea/test/token.test.ts @@ -293,11 +293,11 @@ test("first start: mints with the admin account, keeps the token at 0600, asks f assert.equal(repos[0]?.full_name, "novox/hq"); assert.equal(forge.mints, 1); // Never write:admin (novox/hq ADR 0266): the forge's administration is the operator's, not a tool's. - assert.deepEqual(forge.lastScopes, ["write:repository", "write:issue", "read:user"]); + assert.deepEqual(forge.lastScopes, ["write:repository", "write:issue", "read:user", "read:package"]); assert.deepEqual(forge.lastScopes, [...TOKEN_SCOPES]); const token = forge.tokens.get("mesh-tools")!; assert.equal(await readFile(file, "utf8"), token + "\n"); - assert.equal(await readFile(file + ".scopes", "utf8"), "read:user,write:issue,write:repository\n"); + assert.equal(await readFile(file + ".scopes", "utf8"), "read:package,read:user,write:issue,write:repository\n"); assert.equal((await stat(file)).mode & 0o777, 0o600); // Said that it minted, and where it keeps it — never what it is. assert.ok(logs.some((l) => l.startsWith("minted a token")), logs.join("\n")); @@ -500,16 +500,18 @@ test("a configured token wins, and is reported rather than minted over when the const { env } = await delivered(forge); const before = forge.mints; - const client = GiteaClient.fromEnv({ ...env, MESH_GITEA_TOKEN: "one-somebody-pasted-in" }); + const client = GiteaClient.fromEnv({ ...env, GITEA_TOKEN: "one-somebody-pasted-in" }); await assert.rejects(client.listRepos(), /rejected the configured Gitea token \(401\)/); assert.equal(forge.mints, before); + // MESH_GITEA_TOKEN is no word of the mesh's (nothing sets it, checks/environment): it is not read, the module mints. + await assert.doesNotReject(GiteaClient.fromEnv({ ...env, MESH_GITEA_TOKEN: "one-somebody-pasted-in" }).listRepos()); }); test("nothing to mint with and no token: the client says what is missing", async () => { assert.throws( () => GiteaClient.fromEnv({ MESH_GITEA_URL: forge.url, MESH_GITEA_ADMIN_USER: ADMIN }), - /set MESH_GITEA_TOKEN, or MESH_GITEA_ADMIN_PASSWORD_FILE, MESH_GITEA_STATE_DIR/, + /set GITEA_TOKEN, or MESH_GITEA_ADMIN_PASSWORD_FILE, MESH_GITEA_STATE_DIR/, ); }); diff --git a/modules/gitea/test/tools-account.test.ts b/modules/gitea/test/tools-account.test.ts index 6bf55535..d05ec24b 100644 --- a/modules/gitea/test/tools-account.test.ts +++ b/modules/gitea/test/tools-account.test.ts @@ -14,9 +14,10 @@ import { mkdtemp, readFile, writeFile, mkdir } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { MintedToken } from "../dist/token.js"; +import { MintedToken, TOKEN_SCOPES } from "../dist/token.js"; import { GiteaClient } from "../dist/client.js"; import { TOOLS_ACCOUNT, TOOLS_TEAM_UNITS, BUILDER_ACCOUNT, AGENT_ACCOUNT, meshAccounts } from "../dist/tools-account.js"; +import { PACKAGES_ACCOUNT, PACKAGES_TEAM_UNITS, PACKAGES_TOKEN_REFUSED, keepPackagesToken, packagesToken, renewOnRefusal } from "../dist/packages-account.js"; const ADMIN = "mesh-admin"; const PASSWORD = "the-vault-minted-this"; @@ -116,6 +117,13 @@ function fakeForge() { } } if (req.method === "GET" && p === "/user") return json(res, 200, { login: me, is_admin: isAdmin(me) }); + // The organisation's packages: a token whose scope reaches packages, of a member of a team writing them. + if (req.method === "GET" && p === "/packages/novox") { + const scopes = tokens.find((t) => `token ${t.sha1}` === req.headers.authorization)?.scopes ?? []; + if (!scopes.some((s) => /:package$/.test(s))) return json(res, 403, { message: "token does not have at least one of required scope(s): [read:package]" }); + if (!teams.some((t) => t.members.has(me) && t.units.units_map?.["repo.packages"] === "write")) return json(res, 403, { message: "no access to the packages" }); + return json(res, 200, []); + } m = p.match(/^\/users\/([^/]+)$/); if (m && req.method === "GET") { const u = users.get(decodeURIComponent(m[1])); @@ -359,6 +367,44 @@ test("the manifest hands the tools their account, and the Go forge bundle holds const forgeBundle = manifest.build.artifacts.find((a: any) => a.name === "forge"); assert.ok(forgeBundle.env.MESH_GITEA_TOKEN_FILE); assert.equal(forgeBundle.env.MESH_GITEA_ADMIN_PASSWORD_FILE, undefined); + // The package registry's bundle reads with the tools' token and writes with the packages account's (design 51). + const packages = manifest.build.artifacts.find((a: any) => a.name === "package-registry"); + assert.equal(packages.env.MESH_GITEA_TOKEN_FILE, forgeBundle.env.MESH_GITEA_TOKEN_FILE); + assert.equal(packages.env.MESH_GITEA_PACKAGES_USER, PACKAGES_ACCOUNT); + assert.match(packages.env.MESH_GITEA_PACKAGES_TOKEN_FILE, /^\$\{dir:runtime-state\}\/packages-token$/); + assert.equal(packages.env.MESH_GITEA_ADMIN_PASSWORD_FILE, undefined); + assert.equal(code.env.MESH_GITEA_STATE_DIR, "${dir:runtime-state}", "the watcher mints the packages token where the bundle reads it"); +}); + +test("the package seats write as an account of their own: plain, in a team writing only packages, its token write:package alone, renewed when the forge rejects it", async () => { + const env = await environment(); + const source = packagesToken(forge.url, env, () => {}); + assert.equal(await keepPackagesToken(source, forge.url), "the package seats' write token holds"); + const account = forge.users.get(PACKAGES_ACCOUNT); + assert.ok(account, "the module made the packages account"); + assert.equal(account!.admin, false, "the packages account is no administrator"); + const held = forge.tokens.filter((t) => t.name === "mesh-packages"); + assert.deepEqual(held.map((t) => [t.user, t.scopes]), [[PACKAGES_ACCOUNT, ["write:package"]]]); + assert.equal(await readFile(join(env.MESH_GITEA_STATE_DIR, "packages-token"), "utf8"), held[0].sha1 + "\n"); + assert.equal(await readFile(join(env.MESH_GITEA_STATE_DIR, "packages-token.scopes"), "utf8"), "write:package for mesh-packages\n"); + const team = forge.teams.find((t) => t.name === "mesh-packages"); + assert.ok(team?.members.has(PACKAGES_ACCOUNT)); + assert.deepEqual(team!.units.units_map, PACKAGES_TEAM_UNITS.units_map, "the team writes packages and nothing else"); + // The forge drops the token behind the module's back: the Go bundle's refusal (the finding's put) mints it afresh and + // clears the finding; no timer looks at it, and another key of the findings state is not its own. + forge.tokens.splice(forge.tokens.indexOf(held[0]), 1); + const cleared: string[] = []; + const said: string[] = []; + const renew = renewOnRefusal(source, forge.url, { delete: async (k) => { cleared.push(k); } }, (l) => said.push(l)); + await renew({ key: "seat.gitea.other", op: "put" }); + await renew({ key: PACKAGES_TOKEN_REFUSED, op: "delete" }); + assert.equal(forge.tokens.filter((t) => t.name === "mesh-packages").length, 0, "nothing renews but the refusal"); + await renew({ key: PACKAGES_TOKEN_REFUSED, op: "put" }); + assert.deepEqual(said, ["the forge rejected the package seats' write token; minted afresh"]); + assert.deepEqual(cleared, [PACKAGES_TOKEN_REFUSED]); + assert.equal(forge.tokens.filter((t) => t.name === "mesh-packages").length, 1); + // The tools' token reads packages too: read:package (design 51's read-only tool token). + assert.ok(TOKEN_SCOPES.includes("read:package")); }); // Review of issue 469: what reads the rules' list is moved to the branch, and nothing leaves the tools without a token. diff --git a/modules/gitea/token.ts b/modules/gitea/token.ts index 48592f33..2c02a65d 100644 --- a/modules/gitea/token.ts +++ b/modules/gitea/token.ts @@ -47,6 +47,9 @@ export const TOKEN_NAME = "mesh-tools"; * The least the fifteen tools and the watcher need (gitea's route groups, 1.20+ scoped tokens): * write:repository — create/delete repositories, pull requests (list/get/open/merge); * write:issue — issues, comments, labels; + * read:package — the package registry's reads by the Go bundle (cmd/package-registry): the package seats' + * held, versions and the retention's survey (design 51: a read-only tool token; the + * writes go with the packages account's own token, packages-account.ts); * read:user — GET /user/repos, which the watcher's poll and gitea_list_repos both call. * It sits under the `user` category despite listing repositories, not `repository` * — confirmed against the running forge (1.27.3), which answered @@ -62,7 +65,7 @@ export const TOKEN_NAME = "mesh-tools"; * `403 token does not have at least one of required scope(s)`, and the client treats that like a * 401 — the source re-mints by name, with the whole list, and the call is retried once. */ -export const TOKEN_SCOPES: readonly string[] = ["write:repository", "write:issue", "read:user"]; +export const TOKEN_SCOPES: readonly string[] = ["write:repository", "write:issue", "read:user", "read:package"]; /** Where a client's token comes from, and what to do when the forge says it is wrong. */ export interface TokenSource { @@ -82,7 +85,7 @@ export class ConfiguredToken implements TokenSource { async renew(): Promise { throw new Error( - "the forge rejected the configured Gitea token (401). It was set explicitly (settings or MESH_GITEA_TOKEN), " + + "the forge rejected the configured Gitea token (401). It was set explicitly (settings or GITEA_TOKEN), " + "so the module does not mint over it — fix it, or unset it and the module mints its own", ); } @@ -173,7 +176,7 @@ export class MintedToken implements TokenSource { stateDir ? null : "MESH_GITEA_STATE_DIR", ].filter((v): v is string => v !== null); if (missing.length) { - throw new Error(`no Gitea token, and nothing to mint one with — set MESH_GITEA_TOKEN, or ${missing.join(", ")}`); + throw new Error(`no Gitea token, and nothing to mint one with — set GITEA_TOKEN, or ${missing.join(", ")}`); } // The tools' own account, never the administrator (novox/hq ADR 0365 §2): MESH_GITEA_TOOLS_USER names it. const account = env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT; diff --git a/modules/gitea/tools/index.ts b/modules/gitea/tools/index.ts index 9b183a55..94736c81 100644 --- a/modules/gitea/tools/index.ts +++ b/modules/gitea/tools/index.ts @@ -24,8 +24,9 @@ import { /** A head named for a merge: at least 12 hex characters of its sha (novox/hq issue 381), up to a sha-256 commit's 64. */ const REVIEWED_HEAD = /^[0-9a-f]{12,64}$/; -/** The forge's container, where its repositories and git are: the note is written there (novox/hq ADR 0239). */ -const forgeContainer = process.env.MESH_GITEA_CONTAINER || "gitea"; +/** The forge's container, where its repositories and git are: the note is written there (novox/hq ADR 0239). Its name + * is the manifest's container's; no variable names it (a MESH_* word nothing sets is refused, checks/environment). */ +const forgeContainer = "gitea"; /** A protection rule as a person reads it: what it guards, not every field the forge keeps. */ export function summarised(p: BranchProtection) { diff --git a/modules/gitea/tsconfig.json b/modules/gitea/tsconfig.json index 7b5440b7..5e851876 100644 --- a/modules/gitea/tsconfig.json +++ b/modules/gitea/tsconfig.json @@ -8,5 +8,5 @@ "skipLibCheck": true, "noEmit": true }, - "include": ["client.ts", "token.ts", "pulls.ts", "protection.ts", "index.ts", "provisioner/index.ts", "provisioner/teams.ts", "tools/index.ts"] + "include": ["client.ts", "token.ts", "pulls.ts", "protection.ts", "index.ts", "tools/index.ts"] } diff --git a/modules/lab/README.md b/modules/lab/README.md index 6ce0f8d5..87bb7bd9 100644 --- a/modules/lab/README.md +++ b/modules/lab/README.md @@ -1,6 +1,16 @@ # lab -The `lab` module of the mesh catalogue: installs the package `git`; installs the package `make`; installs the package `python`; installs the package `file`; installs the package `iproute2`; installs the package `incus`; requires `go-toolchain` and `node-toolchain`, so the `go` module and the `nodejs` module are assigned to a machine running it (novox/hq ADR 0477 clause 4). +The `lab` module of the mesh catalogue: installs the package `git`; installs the package `openssh`; installs the package `make`; installs the package `python`; installs the package `file`; installs the package `iproute2`; installs the package `incus`; requires `go-toolchain` and `node-toolchain`, so the `go` module and the `nodejs` module are assigned to a machine running it (novox/hq ADR 0477 clause 4). + +## How its runs clone + +A run clones the mesh's repositories, private ones too (mesh-tools-go), as the build modules do: the lab requires +`forge-account`, the forge's provider registers the account's key (`~/.ssh/mesh_ed25519`) for it, and the mesh writes +`forge.gitconfig`, `forge.ssh` and `forge.known_hosts` into the lab's state from that binding. Every clone reads +`forge.gitconfig` (`git -c include.path=...`), which sends the forge's address, as the lab's `forge` setting or the +forge itself names it, over ssh to the forge with that key and only the forge's host key. No password is bound, +sealed or written for a run. npm reads from `npm-package-registry` are anonymous (novox/hq ADR 0346): the lab +contributes nothing to it and holds no secret of it; its binding gives the registry's public root only. ## Files it does not know diff --git a/modules/lab/files/forge.gitconfig b/modules/lab/files/forge.gitconfig new file mode 100644 index 00000000..d61dc537 --- /dev/null +++ b/modules/lab/files/forge.gitconfig @@ -0,0 +1,5 @@ +[url "ssh://${bound:forge-account:ssh_alias}/"] + insteadOf = ${bound:forge-account:https_base} + insteadOf = ${config:forge}/ +[core] + sshCommand = ssh -F ${dir:state}/forge.ssh diff --git a/modules/lab/files/forge.known_hosts b/modules/lab/files/forge.known_hosts new file mode 100644 index 00000000..e0df6d81 --- /dev/null +++ b/modules/lab/files/forge.known_hosts @@ -0,0 +1 @@ +${bound:forge-account:ssh_alias} ${bound:forge-account:host_key} diff --git a/modules/lab/files/forge.ssh b/modules/lab/files/forge.ssh new file mode 100644 index 00000000..8723cbf2 --- /dev/null +++ b/modules/lab/files/forge.ssh @@ -0,0 +1,10 @@ +Host ${bound:forge-account:ssh_alias} + HostName ${bound:forge-account:ssh_host} + Port ${bound:forge-account:ssh_port} + HostKeyAlias ${bound:forge-account:ssh_alias} + User git + IdentityFile ~/.ssh/mesh_ed25519 + IdentitiesOnly yes + UserKnownHostsFile ${dir:state}/forge.known_hosts + StrictHostKeyChecking yes + BatchMode yes diff --git a/modules/lab/files/lab.env b/modules/lab/files/lab.env index 35aa1a29..6b0a8e77 100644 --- a/modules/lab/files/lab.env +++ b/modules/lab/files/lab.env @@ -1,3 +1,3 @@ MESH_LAB_FORGE=${config:forge} MESH_LAB_FORGE_BINDING=${dir:state}/package-registry.json -MESH_LAB_FORGE_SECRET=${dir:state}/package-registry.secret +MESH_LAB_FORGE_GIT=${dir:state}/forge.gitconfig diff --git a/modules/lab/module.json b/modules/lab/module.json index ef82b87c..ccd89740 100644 --- a/modules/lab/module.json +++ b/modules/lab/module.json @@ -7,20 +7,13 @@ ], "requires": [ "npm-package-registry", + "forge-account", "go-toolchain", "node-toolchain" ], - "contributes": { - "npm-package-registry": { - "access": "read" - } - }, "binds": { "npm-package-registry": "${dir:state}/package-registry.json" }, - "secrets": { - "npm-package-registry": "${dir:state}/package-registry.secret" - }, "data": { "own": [ { @@ -61,11 +54,37 @@ "mode": "0600", "template": "files/lab.env" }, + { + "id": "forge-git", + "type": "file", + "path": "${dir:state}/forge.gitconfig", + "mode": "0600", + "template": "files/forge.gitconfig" + }, + { + "id": "forge-ssh", + "type": "file", + "path": "${dir:state}/forge.ssh", + "mode": "0600", + "template": "files/forge.ssh" + }, + { + "id": "forge-known-hosts", + "type": "file", + "path": "${dir:state}/forge.known_hosts", + "mode": "0644", + "template": "files/forge.known_hosts" + }, { "id": "git", "type": "package", "package": "git" }, + { + "id": "openssh", + "type": "package", + "package": "openssh" + }, { "id": "make", "type": "package", @@ -108,7 +127,7 @@ "MESH_LAB_WORK": "${dir:work}", "MESH_LAB_ENV_FILE": "${dir:state}/lab.env", "MESH_LAB_FORGE_BINDING": "${dir:state}/package-registry.json", - "MESH_LAB_FORGE_SECRET": "${dir:state}/package-registry.secret" + "MESH_LAB_FORGE_GIT": "${dir:state}/forge.gitconfig" } }, { diff --git a/modules/lab/test/base.test.ts b/modules/lab/test/base.test.ts index 93a6f8ec..36d53331 100644 --- a/modules/lab/test/base.test.ts +++ b/modules/lab/test/base.test.ts @@ -60,10 +60,10 @@ test("no run starts while the base image is rebuilt", () => { }); test("a rebuild is mesh-lab's base build from the forge's main, with no secret in its script", () => { - const s = baseScript("/work/base-x", "https://forge.invalid", "/work/.base-x.git-credentials"); - assert.match(s, /clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/); + const s = baseScript("/work/base-x", "https://forge.invalid", "/state/forge.gitconfig"); + assert.match(s, /git -c include\.path='\/state\/forge\.gitconfig' clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/); assert.match(s, /src\/cli.ts base build/); - assert.ok(s.indexOf("rm -f '/work/.base-x.git-credentials'") < s.indexOf("npm ci"), "the credential outlives the clone"); + assert.doesNotMatch(s, /credential|^trap |rm -f '\/state/m, "the forge account's configuration is the mesh's, never removed by a rebuild"); }); test("a rebuild is known by its status, and is not a run", () => { diff --git a/modules/lab/test/runs.test.ts b/modules/lab/test/runs.test.ts index b2071208..5383a3ea 100644 --- a/modules/lab/test/runs.test.ts +++ b/modules/lab/test/runs.test.ts @@ -5,7 +5,7 @@ import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { installPlan, installStep, NPM_REPOSITORIES, outcomeOf, readStatus, REPOSITORIES, script } from "../tools/runs.ts"; +import { baseScript, installPlan, installStep, NPM_REPOSITORIES, outcomeOf, readStatus, REPOSITORIES, script } from "../tools/runs.ts"; // Defends novox/hq issue 624: lab_run ran `npm ci` at the root of every checkout it installs, and // mesh-tools keeps its package in node-tools/ with no lock file at its root, so every run failed at its @@ -100,101 +100,109 @@ test("a run checks out mesh-tools-go and says where its checkouts are", () => { }); // Defends novox/hq issue 624, its second half: mesh-tools-go is private, and the lab cloned over HTTP with -// no credential ("could not read Username"). A run clones with the forge credential the mesh binds and -// seals to the lab, through git's credential store, and the secret is in no script, URL or argument. -import { credentialArgs, credentialPath, forgeCredential } from "../tools/runs.ts"; +// no credential ("could not read Username"). The lab clones as the build modules do (issue 1966): it requires +// the forge account, the mesh writes the account's git and ssh configuration into the lab's state, and a run's +// clones go through it over ssh with the account's key. No password is bound, sealed or written for a run. +import { cleanCredentials, forgeArgs, forgeConfig } from "../tools/runs.ts"; -function aBinding(as: string, secret: string): { binding: string; secret: string } { - const dir = mkdtempSync(join(tmpdir(), "lab-binding-")); - writeFileSync(join(dir, "package-registry.json"), JSON.stringify({ provision: "npm-package-registry", at: "10.42.0.1", as, - serves: { scheme: "http", port: 3000 } })); - writeFileSync(join(dir, "package-registry.secret"), `${secret}\n`); - return { binding: join(dir, "package-registry.json"), secret: join(dir, "package-registry.secret") }; +const labDir = join(import.meta.dirname, ".."); + +/** The lab's forge files as the mesh writes them for a bound forge account, into a state directory of its own. */ +function renderedForgeFiles(forge: string): string { + const state = mkdtempSync(join(tmpdir(), "lab-forge-")); + const values: Record = { + ssh_alias: "mesh-forge", ssh_host: "forge.invalid", ssh_port: "222", host_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOn", + https_base: "https://git.example.org/", + }; + for (const f of ["forge.gitconfig", "forge.ssh", "forge.known_hosts"]) { + const text = readFileSync(join(labDir, "files", f), "utf8") + .replace(/\$\{bound:forge-account:([a-z_]+)\}/g, (_, k: string) => values[k] ?? ``) + .replace(/\$\{config:forge\}/g, forge).replace(/\$\{dir:state\}/g, state); + writeFileSync(join(state, f), text); + } + return state; } -test("the forge credential is the bound account and its sealed secret, for the forge's own origin", () => { - const b = aBinding("lab-shanks", "s3cret-of-forty-characters"); - assert.equal(forgeCredential("http://novox.internal:20000", b.binding, b.secret), - "http://lab-shanks:s3cret-of-forty-characters@novox.internal:20000"); - assert.equal(forgeCredential("http://novox.internal:20000/", b.binding, ""), undefined); - assert.equal(forgeCredential("", b.binding, b.secret), undefined); - assert.equal(forgeCredential("http://novox.internal:20000", "/nowhere", b.secret), undefined); -}); - -test("a run clones through git's credential store, and its script carries no secret", () => { - const b = aBinding("lab-shanks", "s3cret-of-forty-characters"); - const credential = forgeCredential("http://novox.internal:20000", b.binding, b.secret); - const path = credentialPath("/work", "run-x"); - const s = script("/work", "run-x", "http://novox.internal:20000", ["t.test.ts"], {}, path); - assert.ok(credential); - assert.doesNotMatch(s, /s3cret/); - assert.match(s, /git -c credential\.helper= -c 'credential\.helper=store --file=\/work\/\.run-x\.git-credentials' clone --quiet --depth 50 --branch 'main' 'http:\/\/novox\.internal:20000\/novox\/mesh-tools-go\.git'/); - assert.match(s, /export GIT_TERMINAL_PROMPT=0/); - assert.equal(path.startsWith("/work/run-x/"), false, "the credential is kept out of the run's own directory"); - assert.equal(credentialArgs(undefined), ""); -}); - -// Held against git itself: the credential in the store is offered to the forge's origin when it asks, and -// a clone without it fails at once rather than prompting. -test("git presents the stored credential to the forge, and only when one is given", () => { - const dir = mkdtempSync(join(tmpdir(), "lab-git-")); - const store = join(dir, "git-credentials"); - writeFileSync(store, "http://lab-shanks:pw@forge.invalid:20000\n", { mode: 0o600 }); - const fill = spawnSync("sh", ["-c", `printf 'protocol=http\\nhost=forge.invalid:20000\\n\\n' | git ${credentialArgs(store)}credential fill`], - { encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } }); - assert.match(fill.stdout, /username=lab-shanks/); - assert.match(fill.stdout, /password=pw/); - const other = spawnSync("sh", ["-c", `printf 'protocol=http\\nhost=elsewhere.invalid\\n\\n' | git ${credentialArgs(store)}credential fill`], - { encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } }); - assert.doesNotMatch(other.stdout, /password=pw/); - rmSync(dir, { recursive: true, force: true }); -}); - -// Review 196 on mesh-catalog #272: the credential file outlived a stop, because SIGTERM to the run's -// process group killed the wrapper before its cleanup. It is now removed right after the clones, on -// every way the script ends, and at the next run's start and in lab_check for what a crash left. -import { cleanCredentials, credentialGuard } from "../tools/runs.ts"; - -test("a stopped script removes the credential, whatever ends it", () => { - for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"] as const) { - const dir = mkdtempSync(join(tmpdir(), "lab-guard-")); - const file = join(dir, ".run-x.git-credentials"); - writeFileSync(file, "http://u:p@h\n", { mode: 0o600 }); - const ran = spawnSync("bash", ["-c", `${credentialGuard(file)}\nkill -${signal.slice(3)} $$\nsleep 5`], { encoding: "utf8", timeout: 10_000 }); - assert.notEqual(ran.status, 0, `${signal}: the script was not stopped`); - assert.equal(existsSync(file), false, `${signal} left the credential on disk`); - rmSync(dir, { recursive: true, force: true }); +test("the lab requires the forge account for its clones, and holds no registry credential", () => { + const m = JSON.parse(readFileSync(join(labDir, "module.json"), "utf8")) as { + requires: string[]; contributes?: unknown; secrets?: unknown; resources: { id: string; path?: string; template?: string; package?: string }[]; + build: { artifacts: { name: string; env?: Record }[] }; + }; + assert.ok(m.requires.includes("forge-account") && m.requires.includes("npm-package-registry")); + assert.equal(m.contributes, undefined, "npm-package-registry reads are anonymous: nothing is contributed"); + assert.equal(m.secrets, undefined, "no registry secret is sealed to the lab"); + const env = m.build.artifacts.find((a) => a.name === "code")?.env ?? {}; + assert.equal(env.MESH_LAB_FORGE_SECRET, undefined); + assert.equal(env.MESH_LAB_FORGE_GIT, m.resources.find((r) => r.id === "forge-git")?.path); + assert.match(readFileSync(join(labDir, "files", "lab.env"), "utf8"), /^MESH_LAB_FORGE_GIT=\$\{dir:state\}\/forge\.gitconfig$/m); + assert.doesNotMatch(readFileSync(join(labDir, "files", "lab.env"), "utf8"), /SECRET/); + assert.ok(m.resources.some((r) => r.package === "openssh"), "the clones need ssh"); + for (const id of ["forge-git", "forge-ssh", "forge-known-hosts"]) { + const r = m.resources.find((x) => x.id === id); + assert.match(readFileSync(join(labDir, String(r?.template)), "utf8"), /\$\{bound:forge-account:/, id); } }); -test("the script removes the credential as soon as its clones are done", () => { - const s = script("/work", "run-x", "http://forge", ["t.test.ts"], {}, "/work/.run-x.git-credentials"); - const lastClone = s.lastIndexOf("clone --quiet"); - const removed = s.indexOf("rm -f '/work/.run-x.git-credentials'", lastClone); - const building = s.indexOf("npm ci"); - assert.ok(lastClone > 0 && removed > lastClone && removed < building, "not removed between the clones and the build"); - assert.match(s, /trap 'rm -f '\\''\/work\/\.run-x\.git-credentials'\\''' EXIT/); +test("a run clones through the forge account's git configuration, and its script holds no credential", () => { + const s = script("/work", "run-x", "http://novox.internal:20000", ["t.test.ts"], {}, "/state/forge.gitconfig"); + assert.match(s, /git -c include\.path='\/state\/forge\.gitconfig' clone --quiet --depth 50 --branch 'main' 'http:\/\/novox\.internal:20000\/novox\/mesh-tools-go\.git'/); + assert.match(s, /export GIT_TERMINAL_PROMPT=0/); + assert.doesNotMatch(s, /credential|^trap /m); + assert.match(baseScript("/work/base-x", "http://forge", "/state/forge.gitconfig"), /git -c include\.path='\/state\/forge\.gitconfig' clone/); + assert.equal(forgeArgs(undefined), ""); }); -test("credentials a crashed run left are removed, but not the running one's", () => { +test("the forge account's configuration is taken only when the mesh wrote it", () => { + const state = renderedForgeFiles("http://novox.internal:20000"); + assert.equal(forgeConfig(join(state, "forge.gitconfig")), join(state, "forge.gitconfig")); + assert.equal(forgeConfig(join(state, "nowhere")), undefined); + assert.equal(forgeConfig(undefined), undefined); + writeFileSync(join(state, "empty"), ""); + assert.equal(forgeConfig(join(state, "empty")), undefined); + rmSync(state, { recursive: true, force: true }); +}); + +function withoutSsh(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const { GIT_SSH_COMMAND: _c, GIT_SSH: _s, ...rest } = env; + return rest; +} + +// Held against git and ssh themselves: a clone of the forge's address, as the lab's setting names it or as the +// forge names itself, goes over ssh to the forge account's host with the account's key and the forge's host key. +test("git sends a clone of the forge's address over ssh, as the forge account's configuration says", () => { + const state = renderedForgeFiles("http://novox.internal:20000"); + const bin = mkdtempSync(join(tmpdir(), "lab-ssh-")); + writeFileSync(join(bin, "ssh"), `#!/bin/sh\nprintf '%s\\n' "$@" >> ${JSON.stringify(join(bin, "asked"))}\nexit 1\n`, { mode: 0o755 }); + for (const url of ["http://novox.internal:20000/novox/mesh-tools-go.git", "https://git.example.org/novox/mesh-tools-go.git"]) { + rmSync(join(bin, "asked"), { force: true }); + spawnSync("sh", ["-c", `git ${forgeArgs(join(state, "forge.gitconfig"))}ls-remote ${url}`], + { encoding: "utf8", env: { ...withoutSsh(process.env), PATH: `${bin}:${process.env.PATH}`, GIT_TERMINAL_PROMPT: "0" } }); + const asked = readFileSync(join(bin, "asked"), "utf8").split("\n"); + assert.ok(asked.includes("-F") && asked.includes(join(state, "forge.ssh")), `${url}: ${asked.join(" ")}`); + assert.ok(asked.includes("mesh-forge"), url); + assert.ok(asked.some((a) => a.includes("novox/mesh-tools-go.git")), url); + } + const g = spawnSync("ssh", ["-G", "-F", join(state, "forge.ssh"), "mesh-forge"], { encoding: "utf8" }); + for (const line of ["hostname forge.invalid", "port 222", "user git", "identitiesonly yes", "batchmode yes", + "stricthostkeychecking true", `userknownhostsfile ${join(state, "forge.known_hosts")}`, "hostkeyalias mesh-forge"]) { + assert.ok(g.stdout.split("\n").includes(line), `ssh -G lacks ${line}`); + } + assert.match(g.stdout, /^identityfile ~\/\.ssh\/mesh_ed25519$/m); + assert.equal(readFileSync(join(state, "forge.known_hosts"), "utf8"), "mesh-forge ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOn\n"); + rmSync(state, { recursive: true, force: true }); + rmSync(bin, { recursive: true, force: true }); +}); + +// What a run of before left on disk, its credential line, is removed at the next run's start and in lab_check. +test("credentials an earlier run left are removed", () => { const work = mkdtempSync(join(tmpdir(), "lab-work-")); - for (const n of [".run-a.git-credentials", ".run-b.git-credentials", "run-a", "other"]) writeFileSync(join(work, n), "x"); - const removed = cleanCredentials(work, join(work, ".run-b.git-credentials")); - assert.deepEqual(removed, [join(work, ".run-a.git-credentials")]); - assert.ok(existsSync(join(work, ".run-b.git-credentials")) && existsSync(join(work, "run-a")) && existsSync(join(work, "other"))); + for (const n of [".run-a.git-credentials", ".base-b.git-credentials", "run-a", "other"]) writeFileSync(join(work, n), "x"); + const removed = cleanCredentials(work).sort(); + assert.deepEqual(removed, [join(work, ".base-b.git-credentials"), join(work, ".run-a.git-credentials")]); + assert.ok(existsSync(join(work, "run-a")) && existsSync(join(work, "other"))); rmSync(work, { recursive: true, force: true }); }); -// The lab asks the forge for read only (review 196): its credential is for clones of branches nobody has -// reviewed yet, and must not be able to publish the mesh's packages. -test("the lab asks the package registry for read only", () => { - const m = JSON.parse(readFileSync(join(import.meta.dirname, "..", "module.json"), "utf8")) as { - requires: unknown[]; contributes?: Record>; - }; - assert.ok(m.requires.includes("npm-package-registry")); - assert.equal(m.contributes?.["npm-package-registry"]?.access, "read"); -}); - // lab_check failed for ever on "destination path '/check' already exists": it cloned into one fixed // path and ignored whether removing the last one worked (novox/hq issue 624). Each check now works in a // fresh directory, and what earlier checks left is cleared or named. @@ -239,19 +247,6 @@ test("leftover checks are cleared, whoever runs the check", () => { rmSync(work, { recursive: true, force: true }); }); -// The lab's tools read the bound credential, so the runtime is given that file by the tools bundle's env -// (ADR 0192), as gitea and umami are given theirs: the runtime's account may then read exactly it, and no -// other credential changes hands (issue 624). -test("the lab's tools bundle names its bound credential and binding in its env", () => { - const m = JSON.parse(readFileSync(join(import.meta.dirname, "..", "module.json"), "utf8")) as { - secrets: Record; binds: Record; - build: { artifacts: { name: string; env?: Record }[] }; - }; - const env = m.build.artifacts.find((a) => a.name === "code")?.env ?? {}; - assert.equal(env.MESH_LAB_FORGE_SECRET, m.secrets["npm-package-registry"]); - assert.equal(env.MESH_LAB_FORGE_BINDING, m.binds["npm-package-registry"]); -}); - // The next setup failure of novox/hq issue 624 (run-2026-10-13T13-15-26-634Z on shanks): npm 12 refused // mesh-tools' locked @novox/mesh-sdk with EALLOWREMOTE. The lab named the registry by its forge setting's // address (http://novox.internal:20000/…), the lock file holds the address the forge writes every tarball diff --git a/modules/lab/tools/index.ts b/modules/lab/tools/index.ts index 63c7c41f..20bae115 100644 --- a/modules/lab/tools/index.ts +++ b/modules/lab/tools/index.ts @@ -4,12 +4,10 @@ import { spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; -import { mkdtempSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { rmSync } from "node:fs"; import { - checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start, - stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, credentialPath, launchRun, + checkDir, cleanCredentials, clearChecks, forgeArgs, forgeConfig, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, start, + stop, tail, baseBuildLog, listBaseBuilds, startBaseBuild, launchRun, launchRebuild, BASE_MAX_AGE_DAYS, describeBase, readBaseImage, baseProblem, } from "./runs.js"; @@ -19,7 +17,6 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // ADR 0192): read from the env-file the mesh fills, at each call, so a changed setting is used // without restarting the runtime. MESH_LAB_FORGE itself still wins, for a hand-run instance. const forgeOf = (): string => (env.MESH_LAB_FORGE ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE")).replace(/\/+$/, ""); - // The forge credential the mesh binds and seals to the lab (novox/hq issue 624), read at each call. // The registry's public root the forge serves in the lab's binding (novox/hq ADR 0346), read at each call. const bindingOf = (): string | undefined => env.MESH_LAB_FORGE_BINDING ?? (wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING") || undefined); @@ -28,9 +25,9 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { const registry = await npmRegistry(forge, fetch, root.root); return registry.problem && root.problem ? { ...registry, problem: `${registry.problem}; ${root.problem}` } : registry; }; - const credentialOf = (forge: string): string | undefined => forgeCredential(forge, - env.MESH_LAB_FORGE_BINDING ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_BINDING"), - env.MESH_LAB_FORGE_SECRET ?? wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_SECRET")); + // The forge account's git configuration the mesh writes for the lab's clones (forgeConfig), read at each call. + const forgeGitOf = (): string | undefined => + forgeConfig(env.MESH_LAB_FORGE_GIT ?? (wordIn(env.MESH_LAB_ENV_FILE, "MESH_LAB_FORGE_GIT") || undefined)); return [ { name: "lab_check", @@ -50,7 +47,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { if (clone.status !== 0) return { ok: false, output: [String(clone.stderr).trim(), ...leftovers].join("\n") }; spawnSync("npm", ["ci", "--no-audit", "--no-fund", "--loglevel=error"], { cwd: dir, encoding: "utf8" }); check = spawnSync("node", ["--experimental-strip-types", "src/cli.ts", "check"], { cwd: dir, encoding: "utf8" }); - problems = setupProblems(forge, `${dir}/setup`, credentialOf(forge)); + problems = setupProblems(forge, `${dir}/setup`, forgeGitOf()); // And the address a run names the package registry by (npmRegistry): said when the registry does not say it. const registry = await registryOf(forge); if (registry.problem) problems.push(registry.problem); @@ -60,10 +57,8 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every // repository it clones answers on the forge, and every one it installs has a lock file where the run // installs from. - // A credential a stopped, crashed or rebooted run or rebuild left behind is removed, and said; one still in - // use by a running run or rebuild is not. - const busy = [running(work)?.id, buildingBase(work)?.id].filter((id): id is string => !!id); - const left = cleanCredentials(work, busy.map((id) => credentialPath(work, id))); + // A credential line a run or rebuild of before the forge account left behind is removed, and said. + const left = cleanCredentials(work); const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run or rebuild left behind: ${left.join(", ")}` : ""; // **And how old the base image is** (novox/hq issue 698): past the bound, missing, or of an age that cannot // be read is a problem here, though lab_run refuses only the first. @@ -106,7 +101,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // **One job at a time, not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698, // review 256): the registry asked first, then the check and the start with nothing awaited between. const launched = await launchRun(work, { registry: () => registryOf(forge), base: () => readBaseImage() }, - (registry) => start(work, forge, tests, refs, credentialOf(forge), registry)); + (registry) => start(work, forge, tests, refs, forgeGitOf(), registry)); if (!launched.started) return launched; return { started: true, run: launched.job, base: describeBase(launched.base), registry: launched.registry, ...(launched.problem ? { problem: launched.problem } : {}) }; @@ -137,7 +132,7 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { // Not under a run, nor beside another rebuild: the registry asked first, then the check and the start with // nothing awaited between (review 256). const launched = await launchRebuild(work, { registry: () => registryOf(forge), base: () => before }, - (registry) => startBaseBuild(work, forge, credentialOf(forge), registry)); + (registry) => startBaseBuild(work, forge, forgeGitOf(), registry)); if (launched.started === false) return { ...reading, started: false, reason: launched.reason }; return { ...reading, started: true, build: launched.job }; }, @@ -174,14 +169,12 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] { * answer for, and a checkout it installs with nothing installable (installPlan). Empty when a run's setup * would go through. */ -export function setupProblems(forge: string, dir: string, credential?: string): string[] { +export function setupProblems(forge: string, dir: string, forgeGit?: string): string[] { const problems: string[] = []; spawnSync("rm", ["-rf", dir]); - // The credential a run would offer, in a file of its own outside the check's clones, removed after. - const keep = mkdtempSync(join(tmpdir(), "lab-check-")); - const file = writeCredential(join(keep, "git-credentials"), credential); + // The forge account's configuration a run's clones go through (forgeConfig). const git = (args: string[]) => - spawnSync("sh", ["-c", `git ${credentialArgs(file)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], { + spawnSync("sh", ["-c", `git ${forgeArgs(forgeGit)}${args.map((a) => `'${a.replace(/'/g, `'\\''`)}'`).join(" ")}`], { encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, }); try { @@ -192,7 +185,7 @@ export function setupProblems(forge: string, dir: string, credential?: string): const said = remote.stderr.trim().split("\n")[0] ?? ""; problems.push(`a run checks out ${repo}, and ${url} could not be read` + (said ? `: ${said}` : ": no main branch") + - (credential ? "" : " — the lab holds no forge credential (its npm-package-registry binding and secret)")); + (forgeGit ? "" : " — the lab holds no forge account yet (its forge-account binding, written as forge.gitconfig in its state)")); continue; } if (!NPM_REPOSITORIES.includes(repo)) continue; @@ -209,7 +202,6 @@ export function setupProblems(forge: string, dir: string, credential?: string): } } } finally { - rmSync(keep, { recursive: true, force: true }); spawnSync("rm", ["-rf", dir]); } return problems; diff --git a/modules/lab/tools/runs.ts b/modules/lab/tools/runs.ts index a253c069..b1795d42 100644 --- a/modules/lab/tools/runs.ts +++ b/modules/lab/tools/runs.ts @@ -163,68 +163,31 @@ export async function npmRegistry(forge: string, get: typeof fetch = fetch, publ } /** - * The forge credential a run's clones may present, as a git credential-store line — - * scheme://user:password@host[:port] — for the forge's own origin, or undefined when the lab holds none. + * The forge account's git configuration a run's clones go through, or undefined when the mesh has not written it. * - * **The mesh's credential for the forge, the way the build modules hold it** (novox/hq issue 624): the lab - * requires the package registry, the forge's provisioner binds it an account (`as`) and seals that - * account's password, and the forge that answers npm is the forge that hosts the repositories. Private - * repositories (mesh-tools-go) are cloned with it; public ones clone as before. Written to a file git reads - * through its credential store, never into a URL, an argument, the run's script or its log, and offered - * only to the forge's own scheme, host and port. + * **The mesh's account on the forge, the way the build modules hold it** (novox/hq issue 624, issue 1966): the lab + * requires `forge-account`, the forge's provider registers the account's key (`~/.ssh/mesh_ed25519`) for it, and + * the mesh writes into the lab's state a git configuration that sends every clone of the forge's address over ssh + * to the forge, with an ssh configuration that offers that key and takes only the forge's host key. Private + * repositories (mesh-tools-go) clone with it; no password is bound, sealed or written for a run. */ -export function forgeCredential(forge: string, bindingFile: string, secretFile: string): string | undefined { - if (!forge || !bindingFile || !secretFile) return undefined; - let as = ""; - let secret = ""; +export function forgeConfig(file: string | undefined): string | undefined { + if (!file) return undefined; try { - as = String((JSON.parse(readFileSync(bindingFile, "utf8")) as { as?: string }).as ?? ""); - secret = readFileSync(secretFile, "utf8").trim(); + return /^\s*insteadOf\s*=\s*\S/m.test(readFileSync(file, "utf8")) ? file : undefined; } catch { return undefined; } - if (!as || !secret) return undefined; - let origin: URL; - try { - origin = new URL(forge); - } catch { - return undefined; - } - const line = new URL(`${origin.protocol}//${origin.host}`); - line.username = as; - line.password = secret; - return line.toString().replace(/\/$/, ""); } -/** Where a run keeps its forge credential: beside its directory, never in it, and removed when it ends. */ +/** Where a run of before kept its forge credential line, beside its directory: removed wherever one is left. */ export function credentialPath(work: string, id: string): string { return join(work, `.${id}.git-credentials`); } -/** writeCredential puts a credential-store line where git reads it, readable by this account alone. */ -export function writeCredential(path: string, line: string | undefined): string | undefined { - if (!line) return undefined; - writeFileSync(path, `${line}\n`, { mode: 0o600 }); - return path; -} - -/** - * credentialGuard is the shell that removes the credential however the script ends — its end, an error, - * or a stop's SIGTERM to the run's process group, which kills the wrapper before its own cleanup runs - * (review 196 on mesh-catalog #272). The script also removes it as soon as its clones are done. - */ -export function credentialGuard(path: string | undefined): string { - if (!path) return ""; - const rm = `rm -f ${shellQuote(path)}`; - return `trap ${shellQuote(rm)} EXIT -trap ${shellQuote(`${rm}; exit 143`)} TERM -trap ${shellQuote(`${rm}; exit 130`)} INT -trap ${shellQuote(`${rm}; exit 129`)} HUP`; -} - /** * cleanCredentials removes every run's and every base rebuild's credential file left in the work directory - * but the ones named — what a crash or a reboot left behind, which no trap could remove. Answers what it + * but the ones named — what runs of before the forge account, a crash or a reboot left behind. Answers what it * removed. */ export function cleanCredentials(work: string, keep?: string | string[]): string[] { @@ -272,10 +235,10 @@ export function clearChecks(work: string): string[] { return said; } -/** The arguments that have git offer the credential in that file, and only that one. */ -export function credentialArgs(path: string | undefined): string { +/** The arguments that have git read the forge account's configuration (forgeConfig) for one command. */ +export function forgeArgs(path: string | undefined): string { if (!path) return ""; - return `-c credential.helper= -c ${shellQuote(`credential.helper=store --file=${path}`)} `; + return `-c include.path=${shellQuote(path)} `; } export interface RunStatus { @@ -462,7 +425,7 @@ const shellQuote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`; * at this run's own tree, so what is built and claimed is exactly what was checked out. */ export function script(work: string, id: string, forge: string, tests: string[], refs: Record, - credential?: string, registry = `${forge}${NPM_PATH}`): string { + forgeGit?: string, registry = `${forge}${NPM_PATH}`): string { const dir = runDir(work, id); const setState = (state: string) => `node -e ${shellQuote( @@ -471,7 +434,7 @@ export function script(work: string, id: string, forge: string, tests: string[], const clones = REPOSITORIES.map((repo) => { const ref = refs[repo] ?? "main"; return [ - `git ${credentialArgs(credential)}clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`, + `git ${forgeArgs(forgeGit)}clone --quiet --depth 50 --branch ${shellQuote(ref)} ${shellQuote(`${forge}/novox/${repo}.git`)} ${shellQuote(join(dir, repo))}`, `echo "${repo} $(git -C ${shellQuote(join(dir, repo))} rev-parse HEAD)" >> ${shellQuote(join(dir, "commits.txt"))}`, ].join("\n"); }).join("\n"); @@ -479,11 +442,9 @@ export function script(work: string, id: string, forge: string, tests: string[], return `set -euo pipefail # A repository the forge will not show fails its clone at once, rather than waiting on a prompt. export GIT_TERMINAL_PROMPT=0 -${credentialGuard(credential)} cd ${shellQuote(dir)} ${setState("checking-out")} ${clones} -${credential ? `# The forge credential is for the clones, and nothing after them runs with it on disk.\nrm -f ${shellQuote(credential)}` : ""} node -e ${shellQuote( `const fs=require("fs");const f=${JSON.stringify(join(dir, "status.json"))};const s=JSON.parse(fs.readFileSync(f,"utf8"));s.commits=Object.fromEntries(fs.readFileSync(${JSON.stringify(join(dir, "commits.txt"))},"utf8").trim().split("\\n").map(l=>l.split(" ")));fs.writeFileSync(f,JSON.stringify(s,null,2))`, )} @@ -516,16 +477,15 @@ node --experimental-strip-types src/cli.ts suite ${tests.map(shellQuote).join(" /** start begins a run and returns at once with its status. */ export function start(work: string, forge: string, tests: string[], refs: Record, - credential?: string, registry?: string): RunStatus { + forgeGit?: string, registry?: string): RunStatus { const id = `run-${new Date().toISOString().replace(/[:.]/g, "-")}`; const dir = runDir(work, id); mkdirSync(dir, { recursive: true }); const status: RunStatus = { id, state: "checking-out", started: new Date().toISOString(), tests, refs }; writeFileSync(statusPath(work, id), JSON.stringify(status, null, 2)); - // What a crashed or rebooted run left behind goes before this run writes its own. + // A credential line a run of before left behind goes before this run starts. cleanCredentials(work); - const credentials = writeCredential(credentialPath(work, id), credential); - writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, credentials, registry), { mode: 0o700 }); + writeFileSync(join(dir, "run.sh"), script(work, id, forge, tests, refs, forgeGit, registry), { mode: 0o700 }); // The wrapper records how the run ended, then removes the checkouts and keeps the log and status: a // run's tree is its own, and the next run starts from fresh ones (novox/hq ADR 0172). @@ -533,7 +493,6 @@ export function start(work: string, forge: string, tests: string[], refs: Record node -e ${shellQuote( `const f=${JSON.stringify(statusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));if(s.state!=="stopped"){s.state=process.argv[1]==="0"?"passed":"failed"};s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, )} "$code" -rm -f ${shellQuote(credentialPath(work, id))} cd ${shellQuote(dir)} && rm -rf ${REPOSITORIES.map(shellQuote).join(" ")} bin`; const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" }); child.unref(); @@ -715,14 +674,12 @@ export function baseBuildLog(work: string, id: string, lines = 40): string { * baseScript is what one rebuild runs: mesh-lab's main from the forge, installed the way a run installs it, * then its `base build`, which replaces the image and reads back that it works before publishing it. */ -export function baseScript(dir: string, forge: string, credential?: string, registry = `${forge}${NPM_PATH}`): string { +export function baseScript(dir: string, forge: string, forgeGit?: string, registry = `${forge}${NPM_PATH}`): string { const lab = join(dir, "mesh-lab"); return `set -euo pipefail export GIT_TERMINAL_PROMPT=0 -${credentialGuard(credential)} cd ${shellQuote(dir)} -git ${credentialArgs(credential)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)} -${credential ? `rm -f ${shellQuote(credential)}` : ""} +git ${forgeArgs(forgeGit)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)} echo "mesh-lab $(git -C ${shellQuote(lab)} rev-parse HEAD)" printf '%s\\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))} export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))} @@ -733,21 +690,19 @@ node --experimental-strip-types src/cli.ts base build } /** startBaseBuild rebuilds the base image in the background and answers at once with the build. */ -export function startBaseBuild(work: string, forge: string, credential?: string, registry?: string): BaseBuild { +export function startBaseBuild(work: string, forge: string, forgeGit?: string, registry?: string): BaseBuild { const id = `base-${new Date().toISOString().replace(/[:.]/g, "-")}`; const dir = join(work, id); mkdirSync(dir, { recursive: true }); const build: BaseBuild = { id, state: "building", started: new Date().toISOString() }; writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2)); - // What a crashed or rebooted run or rebuild left behind goes before this one writes its own. + // A credential line a run or rebuild of before left behind goes before this one starts. cleanCredentials(work); - const credentials = writeCredential(credentialPath(work, id), credential); - writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, credentials, registry), { mode: 0o700 }); + writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, forgeGit, registry), { mode: 0o700 }); const wrapper = `bash ${shellQuote(join(dir, "build.sh"))} > ${shellQuote(join(dir, "build.log"))} 2>&1; code=$? node -e ${shellQuote( `const f=${JSON.stringify(buildStatusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=process.argv[1]==="0"?"built":"failed";s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`, )} "$code" -rm -f ${shellQuote(credentialPath(work, id))} rm -rf ${shellQuote(join(dir, "mesh-lab"))}`; const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" }); child.unref(); diff --git a/modules/mesh-build-go/README.md b/modules/mesh-build-go/README.md index ff16837f..74d6519b 100644 --- a/modules/mesh-build-go/README.md +++ b/modules/mesh-build-go/README.md @@ -135,11 +135,30 @@ that vendors), `go build -ldflags "-s -w -X main.builtFor=" -o .mesh-bui module's directory, by the installed go (`GOTOOLCHAIN=local`) from the module cache the fetch filled and no network (`GOPROXY=off`). The binary is named as the build agent names it. The output directory is packed with the build runner's `Pack`, as the build agent packs; two build nodes on the same toolchain give the same digest for the same tree (ADR 0477 §6). The step ends with the artifact, the binary, the digest, the size and -the reference of the archive: once the build ended, this process pushes it to the artifact store (novox/hq ADR 0498 §1 §2, -the build runner's `buildstep.PushBlob`) under `/` by its digest, a candidate's as a trunk build's, and answers -`artifact-store:////blobs/`; a push that fails fails the step `not stored: : `. The packed bytes never stay on the node. The store's address is `${dir:state}/registry`, placed by the -manifest, which requires `artifact-store`. The go build runs shut off in bubblewrap (ADR 0448, `isolate.go`): no +the reference of the archive: once the build ended, this process publishes it through the seat `oci-artifact-registry` +(design 51 *Publishing*; `publish.go`): its verbs `publish` and `publish-part`, which the manifest `invokes`, with mesh-sdk's +sender (`publish.Publish`), under `/` by its digest and of kind `bundle` or `archive`, a candidate's as a +trunk build's, in chunks, only what the holder lacks; the holder asks the builder and says `published` or `refused` on the +seat's events. The step answers the seat's form `oci-artifact-registry//@` (ADR 0605 §9); the +builder writes the older form beside it while a counted node-engine reads only that (ADR 0624); a publish the holder refuses, or a +seat nothing holds, fails the step `not stored: : `. The packed bytes never stay on the node. +A build the builder numbers later is published late: the module consumes the builder's `version.numbered` and, for a +bundle or an archive with a digest, publishes that held digest under its version, naming its one part by digest and +sending nothing (`latepublish.go`); a holder that no longer holds the digest fails the event, which is offered again. Every +ask of a seat a build module makes (a publish, a late publish, an image's copy) is narrated by `buildstep/narrate.go`: +asked, waiting on the seat every five minutes while it has not answered, stored or refused, each the ask's phase on its +running step key (logged outside an ask). +A contracts module (a package artifact naming `generated-from`) has its generated Go files regenerated with the seat +generator this module is built with and compared (`buildstep/generated.go`): the unit level of `build_go_test` fails +naming each file that differs, is missing or was not written by the generator (its `out-of-step`), and a build ask +answers the same files in its answer's `out-of-step`, a failed build's too, so a trunk build's end can raise +`generated-types-out-of-step` where no test step runs. Its TypeScript files are mesh-build-node's to compare. + +The module writes into no store itself and requires no `artifact-store`. A Go package (kind `package`, language `go`; `gopackage.go`) is +the directory its one path names packed as Go's module zip, every file under `@/`, its digest the +zip's (no `h1:` sum is answered: nothing carries it on); a trunk build the builder gave a version publishes it through +`go-package-registry` under its Go module path and that version (the manifest `invokes` both verbs) and answers +`@`; a candidate's is packed under `v0.0.0` and publishes nothing. The go build runs shut off in bubblewrap (ADR 0448, `isolate.go`): no network, none of the account, the caches beneath a throwaway layer unless the step is told `publish` (a trunk build). An artifact that is neither a Go bundle nor an archive fails the step naming it. @@ -154,9 +173,16 @@ and xz, read in Go as the build agent reads it, issue 1590); it runs no command - The build runner (the pull of asks, the step lifecycle, the toolchain guard, the fetch, the parsers of `go test -json` and `node --test`, `Pack`, the conformance kit and the `mesh-pack` command) is this module's package `git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep`, a Go module of its own in `buildstep/`. mesh-build-node - and mesh-build-image require it and replace it to the sibling directory, since it is the same repository. What the delivery + and mesh-build-image require it and replace it to the sibling directory, since it is the same repository. It is declared + as this module's Go package artifact `buildstep`, series `0.1` (ADR 0606): a user in another repository accepts its + series, never a commit. What the delivery parts read of a step (the failure contract, the fetch request, the stale-step sweep, the step id) stays in mesh-sdk's `buildstep`. +- The seat generator (design 26 *Contracts modules*) is the package `buildstep/seatgen` and the command `buildstep/cmd/seatgen`, + one program writing a contracts module's Go and TypeScript (`seatgen `; `seatgen -check ` names every path out of + step and runs the client and artifact checks). Its version is buildstep's, so the build input of a compare is the + mesh-build-go (or mesh-build-node) build itself: a new generator is a new build module and a new fingerprint. A contracts + repository runs it as `go run git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/cmd/seatgen@ -check .`. - `build_go_imports` (a tool, and the last step of a build ask that built an artifact) answers the mesh packages the module's Go programs import: `go list -deps` of each Go bundle's program, tests left out, the packages of the forge outside the module's own directory (a sibling package of its own repository counts), sorted and each once. A build ask carries them as diff --git a/modules/mesh-build-go/buildstep/askrun.go b/modules/mesh-build-go/buildstep/askrun.go index 60bc385e..bd5cd599 100644 --- a/modules/mesh-build-go/buildstep/askrun.go +++ b/modules/mesh-build-go/buildstep/askrun.go @@ -29,6 +29,10 @@ type AskSteps struct { // {"imports": [...]}, sorted, from the workspace the build ran in. A build that built an artifact asks it once, at // its end; nil reports none. Imports StepFunc + // Compare regenerates a contracts module's generated files (a package artifact naming generated-from) and answers + // {"out-of-step": [...]} (CompareStep): a build ask runs it once on the fetched workspace and answers the paths in + // its out-of-step, a failed build's too, so a trunk build's end names them where no test step runs. Nil compares none. + Compare StepFunc // Toolchain is the toolchain the module builds with now; a build ask naming another is declined (Declining). Nil checks nothing // (an image or an archive names none). Toolchain func(ctx context.Context) (delivery.ToolchainUsed, error) @@ -156,6 +160,24 @@ func (s AskSteps) build(ctx context.Context, job delivery.BuildJob) (any, error) } } f = got + if s.Compare != nil { + Phase(ctx, "compare the generated files") + out, err := s.Compare(ctx, map[string]any{"workspace": f.Workspace, "module": job.Path}) + if err != nil { + return nil, fmt.Errorf("the generated files of %s: %w", job.Module, err) + } + r, err := raw(out) + if err != nil { + return nil, err + } + var read struct { + OutOfStep []string `json:"out-of-step"` + } + if err := json.Unmarshal(r, &read); err != nil { + return nil, fmt.Errorf("the compare of %s answered what is not a list: %s", job.Module, r) + } + answer.OutOfStep = read.OutOfStep + } } var ( out any @@ -175,10 +197,21 @@ func (s AskSteps) build(ctx context.Context, job delivery.BuildJob) (any, error) if job.Publish { args["publish"] = true } + // The seat it is published through and the version the builder gave it (design 51 *Publishing*, CAT-BG2). + if a.Seat != "" { + args["seat"] = a.Seat + } + if a.Version != "" { + args["version"] = a.Version + } Phase(ctx, "build "+a.Artifact) out, err = s.Build(ctx, args) } if err != nil { + // The compare's out-of-step is answered with the failure, so the build's end still names it. + if len(answer.OutOfStep) > 0 { + return answer, fmt.Errorf("%s: %w", a.Artifact, err) + } return nil, fmt.Errorf("%s: %w", a.Artifact, err) } r, err := raw(out) diff --git a/modules/mesh-build-go/buildstep/askrun_test.go b/modules/mesh-build-go/buildstep/askrun_test.go index b0fce397..fb75f23f 100644 --- a/modules/mesh-build-go/buildstep/askrun_test.go +++ b/modules/mesh-build-go/buildstep/askrun_test.go @@ -70,7 +70,8 @@ func ask(job any) delivery.StepAsk { } // 1887/A3: a build ask fetches once, builds each artifact from the workspace with publish, copies a pinned one without a -// fetch, and answers each artifact's digest and reference. +// fetch, and answers each artifact's digest and reference; an artifact's seat and the version the builder gave it reach +// its build (CAT-BG2). func TestRunAskBuilds(t *testing.T) { r := &recorder{fetch: map[string]any{"workspace": "/w/1"}} steps := r.steps() @@ -85,7 +86,7 @@ func TestRunAskBuilds(t *testing.T) { // Issue 1585: the ask's package versions reach the fetch, and the ask's own fetch arguments stay as they were. a := ask(delivery.BuildJob{Module: "m", Path: "modules/m", Publish: true, Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}, - Artifacts: []delivery.AskArtifact{{Artifact: "base", From: "registry/b@sha256:00"}, {Artifact: "bundle"}, {Artifact: "lib"}}}) + Artifacts: []delivery.AskArtifact{{Artifact: "base", From: "registry/b@sha256:00"}, {Artifact: "bundle"}, {Artifact: "lib", Seat: "go-package-registry", Version: "v0.1.15"}}}) a.Packages = delivery.PackageVersions{"git.novox.be/novox/mesh-sdk/go": runCommit} out, err := RunAsk(context.Background(), steps, a) if err != nil { @@ -98,7 +99,7 @@ func TestRunAskBuilds(t *testing.T) { want := `copy {"artifact":"base","from":"registry/b@sha256:00","module":"m"} fetch {"commit":"` + runCommit + `","packages":{"git.novox.be/novox/mesh-sdk/go":"` + runCommit + `"},"repository":"novox/x"} build {"artifact":"bundle","module":"modules/m","publish":true,"workspace":"/w/1"} -build {"artifact":"lib","module":"modules/m","publish":true,"workspace":"/w/1"} +build {"artifact":"lib","module":"modules/m","publish":true,"seat":"go-package-registry","version":"v0.1.15","workspace":"/w/1"} imports {"module":"modules/m","workspace":"/w/1"}` if got != want { t.Fatalf("calls:\n%s\nwant:\n%s", got, want) diff --git a/modules/mesh-build-go/buildstep/blob.go b/modules/mesh-build-go/buildstep/blob.go deleted file mode 100644 index 127c6ace..00000000 --- a/modules/mesh-build-go/buildstep/blob.go +++ /dev/null @@ -1,170 +0,0 @@ -package buildstep - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "syscall" - "time" -) - -// RegistryWait is how long a push waits for an artifact store that refuses connections, longer than its nightly -// collection holds it still (about a minute and a half, issue 457); RegistryFirstPause is the first pause, doubling -// to RegistryMostPause. Variables so a test shortens them. -var ( - RegistryWait = 5 * time.Minute - RegistryFirstPause = time.Second - RegistryMostPause = 10 * time.Second -) - -// BlobReference is where a pushed bundle or archive is, as a done key names it and the controller delivers from -// (novox/hq ADR 0498 clause 1): artifact-store:////blobs/, the store's address left out -// (issue 102). -func BlobReference(module, artifact, digest string) string { - return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest -} - -// PushBlob puts body into the artifact store at registry (host:port) as a blob of repository (/) by -// its digest (`sha256:` and hex), from the build module's own process, after the build step ended (novox/hq ADR 0498 -// clauses 1 and 2): a blob the store already holds is done without an upload; otherwise it is uploaded (POST -// blobs/uploads/, then PUT with the digest). A store refusing connections is waited out up to RegistryWait; any other -// refusal fails, naming the store's status and its own words, for the caller's `not stored:` error. Bytes that are not -// their digest are refused before anything is sent. -func PushBlob(ctx context.Context, client *http.Client, registry, repository, digest string, body []byte) error { - sum := sha256.Sum256(body) - if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { - return fmt.Errorf("the blob is %s, not the digest %s it was to be pushed as", got, digest) - } - if client == nil { - client = http.DefaultClient - } - base := "http://" + registry + "/v2/" + repository + "/blobs/" - var present bool - if err := waitForStore(ctx, registry, func() error { - var err error - present, err = blobPresent(ctx, client, base+digest) - return err - }); err != nil { - return err - } - if present { - return nil - } - var location string - if err := waitForStore(ctx, registry, func() error { - var err error - location, err = startUpload(ctx, client, base+"uploads/") - return err - }); err != nil { - return err - } - target, err := url.Parse(base) - if err != nil { - return err - } - loc, err := target.Parse(location) - if err != nil { - return fmt.Errorf("the store answered the upload's location %q, which is not a URL: %w", location, err) - } - q := loc.Query() - q.Set("digest", digest) - loc.RawQuery = q.Encode() - req, err := http.NewRequestWithContext(ctx, http.MethodPut, loc.String(), bytes.NewReader(body)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/octet-stream") - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("putting %s of %s: %w", digest, repository, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusCreated { - return storeSaid(resp, "putting "+digest+" of "+repository) - } - return nil -} - -func blobPresent(ctx context.Context, client *http.Client, at string) (bool, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodHead, at, nil) - if err != nil { - return false, err - } - resp, err := client.Do(req) - if err != nil { - return false, err - } - resp.Body.Close() - switch resp.StatusCode { - case http.StatusOK: - return true, nil - case http.StatusNotFound: - return false, nil - } - return false, fmt.Errorf("asking the store for %s: it answered %s", at, resp.Status) -} - -func startUpload(ctx context.Context, client *http.Client, at string) (string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodPost, at, nil) - if err != nil { - return "", err - } - resp, err := client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusAccepted { - return "", storeSaid(resp, "starting an upload at "+at) - } - location := resp.Header.Get("Location") - if location == "" { - return "", fmt.Errorf("starting an upload at %s: the store named no location", at) - } - return location, nil -} - -// storeSaid is the store's refusal: its status and the first of its own words. -func storeSaid(resp *http.Response, what string) error { - words, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) - return fmt.Errorf("%s: the store answered %s: %s", what, resp.Status, strings.TrimSpace(string(words))) -} - -// refused is whether an error is a connection refused. -func refused(err error) bool { - return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused")) -} - -// waitForStore runs try, and while it fails because the store refuses connections, tries again with a growing pause -// until RegistryWait has passed. A refusal is the one error waited for: nothing was sent, so trying again does nothing -// twice. -func waitForStore(ctx context.Context, registry string, try func() error) error { - err := try() - if !refused(err) { - return err - } - started, pause := time.Now(), RegistryFirstPause - for { - left := RegistryWait - time.Since(started) - if left <= 0 { - return fmt.Errorf("the artifact store at %s refused every connection for %s, longer than its nightly collection holds it still, so it is down, not paused: %w", - registry, time.Since(started).Round(time.Millisecond), err) - } - select { - case <-ctx.Done(): - return fmt.Errorf("stopped while waiting for the artifact store at %s: %w (last: %v)", registry, ctx.Err(), err) - case <-time.After(min(pause, left)): - } - pause = min(pause*2, RegistryMostPause) - if err = try(); !refused(err) { - return err - } - } -} diff --git a/modules/mesh-build-go/buildstep/blob_test.go b/modules/mesh-build-go/buildstep/blob_test.go deleted file mode 100644 index b77afb0c..00000000 --- a/modules/mesh-build-go/buildstep/blob_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package buildstep - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "io" - "net" - "net/http" - "net/http/httptest" - "strings" - "sync" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -func digestOfBytes(b []byte) string { - h := sha256.Sum256(b) - return "sha256:" + hex.EncodeToString(h[:]) -} - -// registry is an artifact store as far as a blob push needs one: HEAD a blob, start an upload, PUT it with its digest. -type registry struct { - mu sync.Mutex - blobs map[string][]byte - calls []string - put int // the status a PUT answers, 201 when 0 -} - -func (r *registry) ServeHTTP(w http.ResponseWriter, req *http.Request) { - r.mu.Lock() - defer r.mu.Unlock() - r.calls = append(r.calls, req.Method+" "+req.URL.Path) - switch { - case req.Method == http.MethodHead && strings.Contains(req.URL.Path, "/blobs/sha256:"): - if _, ok := r.blobs[req.URL.Path[strings.LastIndex(req.URL.Path, "/")+1:]]; ok { - w.WriteHeader(http.StatusOK) - return - } - w.WriteHeader(http.StatusNotFound) - case req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/blobs/uploads/"): - w.Header().Set("Location", strings.TrimSuffix(req.URL.Path, "/")+"/u1?state=x") - w.WriteHeader(http.StatusAccepted) - case req.Method == http.MethodPut && strings.Contains(req.URL.Path, "/blobs/uploads/u1"): - if r.put != 0 { - w.WriteHeader(r.put) - _, _ = io.WriteString(w, `{"errors":[{"code":"DIGEST_INVALID","message":"provided digest did not match uploaded content"}]}`) - return - } - body, _ := io.ReadAll(req.Body) - d := req.URL.Query().Get("digest") - if req.URL.Query().Get("state") != "x" || d != digestOfBytes(body) { - w.WriteHeader(http.StatusBadRequest) - return - } - r.blobs[d] = body - w.WriteHeader(http.StatusCreated) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } -} - -// A done build's bundle or archive is pushed to the artifact store by its build module (novox/hq ADR 0498 clauses 1 -// and 2): a blob the store already holds is done with no upload; one it lacks is uploaded by digest; its reference is -// the artifact-store form the builder's key and the controller check. -func TestABlobIsPushedOnceByItsDigest(t *testing.T) { - reg := ®istry{blobs: map[string][]byte{}} - srv := httptest.NewServer(reg) - defer srv.Close() - addr := strings.TrimPrefix(srv.URL, "http://") - body := []byte("the packed bundle") - d := digestOfBytes(body) - if err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", d, body); err != nil { - t.Fatal(err) - } - if string(reg.blobs[d]) != string(body) { - t.Fatalf("the store holds %q", reg.blobs[d]) - } - reg.calls = nil - if err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", d, body); err != nil { - t.Fatal(err) - } - if len(reg.calls) != 1 || !strings.HasPrefix(reg.calls[0], "HEAD ") { - t.Fatalf("a blob the store holds was uploaded again: %v", reg.calls) - } - ref := BlobReference("mail", "code", d) - if ref != "artifact-store://mail/code/blobs/"+d { - t.Fatalf("BlobReference = %q", ref) - } - b := delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("4", 64), - Artifacts: []delivery.BuiltArtifact{{Name: "code", Kind: "bundle", Digest: d, Reference: ref}}} - if _, err := delivery.EncodeBuilderBuild(b); err != nil { - t.Fatalf("the key refuses the reference BlobReference gives: %v", err) - } -} - -// A store that refuses the upload fails the push naming the store's own words; bytes that are not the digest are -// refused before anything is sent. -func TestAPushTheStoreRefusesSaysTheStoresWords(t *testing.T) { - reg := ®istry{blobs: map[string][]byte{}, put: http.StatusBadRequest} - srv := httptest.NewServer(reg) - defer srv.Close() - addr := strings.TrimPrefix(srv.URL, "http://") - body := []byte("x") - err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", digestOfBytes(body), body) - if err == nil || !strings.Contains(err.Error(), "provided digest did not match") || !strings.Contains(err.Error(), "400") { - t.Fatalf("a refused upload said %v", err) - } - reg.calls = nil - err = PushBlob(context.Background(), srv.Client(), addr, "mail/code", digestOfBytes([]byte("other")), body) - if err == nil || len(reg.calls) != 0 { - t.Fatalf("bytes that are not their digest: %v, calls %v", err, reg.calls) - } -} - -// A store refusing connections (its nightly collection holds it still) is waited out, up to RegistryWait (issue 457). -func TestAPushWaitsOutAStoreRefusingConnections(t *testing.T) { - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - addr := l.Addr().String() - l.Close() // refusing now - reg := ®istry{blobs: map[string][]byte{}} - go func() { - time.Sleep(300 * time.Millisecond) - l2, err := net.Listen("tcp", addr) - if err != nil { - return - } - _ = http.Serve(l2, reg) - }() - beforeWait, beforePause := RegistryWait, RegistryFirstPause - RegistryWait, RegistryFirstPause = 3*time.Second, 100*time.Millisecond - defer func() { RegistryWait, RegistryFirstPause = beforeWait, beforePause }() - body := []byte("late") - if err := PushBlob(context.Background(), http.DefaultClient, addr, "mail/code", digestOfBytes(body), body); err != nil { - t.Fatalf("a store that came back was not waited for: %v", err) - } - - l3, _ := net.Listen("tcp", "127.0.0.1:0") - gone := l3.Addr().String() - l3.Close() - RegistryWait = 300 * time.Millisecond - err = PushBlob(context.Background(), http.DefaultClient, gone, "mail/code", digestOfBytes(body), body) - if err == nil || !strings.Contains(err.Error(), "refused every connection") { - t.Fatalf("a store that stayed down said %v", err) - } -} diff --git a/modules/mesh-build-go/buildstep/cmd/seatgen/main.go b/modules/mesh-build-go/buildstep/cmd/seatgen/main.go new file mode 100644 index 00000000..9cd41717 --- /dev/null +++ b/modules/mesh-build-go/buildstep/cmd/seatgen/main.go @@ -0,0 +1,78 @@ +// Command seatgen writes, or with -check compares, a contracts module's generated Go and TypeScript. +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "sort" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/seatgen" +) + +func main() { + check := flag.Bool("check", false, "compare instead of writing: name every path out of step") + goModule := flag.String("go-module", "git.novox.be/novox/mesh-contracts/go", "the generated Go tree's module path") + tsPackage := flag.String("ts-package", "@novox/mesh-contracts", "the generated TypeScript package's name") + flag.Parse() + dir := flag.Arg(0) + m, err := seatgen.Load(dir, *goModule, *tsPackage) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + files, err := seatgen.Generate(m) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if *check { + out, err := seatgen.Compare(dir, files) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + // The clients beside the contracts (ADR 0627 §3) and module.json's Go and npm package artifacts (ADR 0625 §1, §3). + for _, check := range []func(string, *seatgen.Module) ([]string, error){seatgen.CheckClients, seatgen.CheckArtifacts} { + more, err := check(dir, m) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + out = append(out, more...) + } + for _, p := range out { + fmt.Println(p) + } + if len(out) > 0 { + os.Exit(1) + } + return + } + for _, root := range seatgen.GeneratedRoots { + _ = os.RemoveAll(filepath.Join(dir, root, "src")) + } + // Each seat's npm package's sources too (ADR 0625 §1), so a file no definition writes any more is gone. + stale, _ := filepath.Glob(filepath.Join(dir, "ts", "seats", "*", "src")) + for _, p := range stale { + _ = os.RemoveAll(p) + } + paths := make([]string, 0, len(files)) + for p := range files { + paths = append(paths, p) + } + sort.Strings(paths) + for _, p := range paths { + full := filepath.Join(dir, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := os.WriteFile(full, files[p], 0o644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + } + fmt.Println(len(paths), "files") +} diff --git a/modules/mesh-build-go/buildstep/generated.go b/modules/mesh-build-go/buildstep/generated.go new file mode 100644 index 00000000..488dc1ec --- /dev/null +++ b/modules/mesh-build-go/buildstep/generated.go @@ -0,0 +1,132 @@ +package buildstep + +// **Regenerate and compare** (CAT-BG5, CAT-BN2; novox/hq ADR 0608 §5, design 26 *Contracts modules*). A module whose +// package artifact names `generated-from` holds contracts only: every file under its generated roots is written by the +// seat generator (package seatgen beside this file, command cmd/seatgen) from the definitions in that directory. A build module regenerates them with the generator +// it was built with (its version is this package's requirement, so a new generator is a new build module and a new +// fingerprint of what it builds) and names each file out of step: one that differs, one missing, one under a generated +// root the generator did not write. The test step fails on any; a trunk build's ask answers them in its out-of-step +// (delivery.BuildAnswer.OutOfStep), so the builder's build end raises generated-types-out-of-step even where no test +// step runs. Each build module compares its own roots: mesh-build-go `go`, mesh-build-node `ts`. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/seatgen" +) + +// GeneratedRootGo and GeneratedRootTS are the generated roots of a contracts module, one per build module. +const ( + GeneratedRootGo = "go" + GeneratedRootTS = "ts" +) + +var goModuleLine = regexp.MustCompile(`(?m)^module\s+(\S+)`) + +// OutOfStep is every file under root, a path from the module's directory, that the generator writes otherwise than the +// module holds it, for each `generated-from` its package artifacts name, sorted and each once; none for a module naming +// none. The generated Go tree's module path is read from /go/go.mod and the TypeScript package's name from +// /ts/package.json, as the contracts module states them. +func OutOfStep(moduleDir, root string) ([]string, error) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil // no manifest here: nothing names generated-from + } + if err != nil { + return nil, fmt.Errorf("the module's manifest cannot be read for its generated files: %w", err) + } + var m catalogue.Manifest + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("the module's manifest does not parse: %w", err) + } + froms := map[string]bool{} + if m.Build != nil { + for _, a := range m.Build.Artifacts { + if a.GeneratedFrom != "" { + froms[filepath.Clean(a.GeneratedFrom)] = true + } + } + } + seen := map[string]bool{} + for from := range froms { + if filepath.IsAbs(from) || from == ".." || strings.HasPrefix(from, "../") { + return nil, fmt.Errorf("generated-from %q is not inside the module", from) + } + dir := filepath.Join(moduleDir, from) + goModule, tsPackage, err := generatedNames(dir) + if err != nil { + return nil, err + } + g, err := seatgen.Load(dir, goModule, tsPackage) + if err != nil { + return nil, fmt.Errorf("the seat definitions in %s do not load: %w", from, err) + } + files, err := seatgen.Generate(g) + if err != nil { + return nil, fmt.Errorf("the seat definitions in %s do not generate: %w", from, err) + } + out, err := seatgen.Compare(dir, files) + if err != nil { + return nil, err + } + for _, line := range out { + p, _, _ := strings.Cut(line, ": ") + if p == root || strings.HasPrefix(p, root+"/") { + seen[filepath.ToSlash(filepath.Join(from, p))] = true + } + } + } + paths := make([]string, 0, len(seen)) + for p := range seen { + paths = append(paths, p) + } + sort.Strings(paths) + return paths, nil +} + +func generatedNames(dir string) (goModule, tsPackage string, err error) { + if raw, err := os.ReadFile(filepath.Join(dir, GeneratedRootGo, "go.mod")); err == nil { + if m := goModuleLine.FindSubmatch(raw); m != nil { + goModule = string(m[1]) + } + } + if raw, err := os.ReadFile(filepath.Join(dir, GeneratedRootTS, "package.json")); err == nil { + var p struct { + Name string `json:"name"` + } + if json.Unmarshal(raw, &p) == nil { + tsPackage = p.Name + } + } + if goModule == "" || tsPackage == "" { + return "", "", errors.New("a contracts module names its generated Go module in go/go.mod and its TypeScript package in ts/package.json; " + + filepath.Base(dir) + " names " + map[bool]string{true: "not the TypeScript package", false: "not the Go module"}[goModule != ""]) + } + return goModule, tsPackage, nil +} + +// CompareStep is AskSteps.Compare for a build module comparing root: it reads {"workspace", "module"}, the workspace's +// tree where tree says it is (the build module's own layout), and answers {"out-of-step": [...]}. +func CompareStep(root string, tree func(workspace string) string) StepFunc { + return func(_ context.Context, args map[string]any) (any, error) { + ws, _ := args["workspace"].(string) + mod, _ := args["module"].(string) + if ws == "" { + return nil, Refuse("the compare names no workspace") + } + paths, err := OutOfStep(filepath.Join(tree(ws), filepath.FromSlash(mod)), root) + if err != nil { + return nil, err + } + return map[string]any{"out-of-step": paths}, nil + } +} diff --git a/modules/mesh-build-go/buildstep/generated_test.go b/modules/mesh-build-go/buildstep/generated_test.go new file mode 100644 index 00000000..369466e3 --- /dev/null +++ b/modules/mesh-build-go/buildstep/generated_test.go @@ -0,0 +1,134 @@ +package buildstep + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/seatgen" +) + +// contracts writes a contracts module of one seat, its generated files as the generator writes them, and answers its +// directory and the generated paths. +func contracts(t *testing.T, manifest string) (string, []string) { + t.Helper() + dir := t.TempDir() + for p, text := range map[string]string{ + "module.json": manifest, + "seats/index.json": `{"format":1,"seats":["one-store"]}`, + "seats/one-store/seat.json": `{"format":1,"seat":"one-store","scope":"mesh","grain":"seat-list","decision":"ADR 1","verbs":[{"name":"query","input":{"type":"object"}}]}`, + } { + writeContract(t, filepath.Join(dir, p), text) + } + m, err := seatgen.Load(dir, "example.org/c/go", "@example/c") + if err != nil { + t.Fatal(err) + } + files, err := seatgen.Generate(m) + if err != nil { + t.Fatal(err) + } + var paths []string + for p, b := range files { + writeContract(t, filepath.Join(dir, p), string(b)) + paths = append(paths, p) + } + // The names the compare reads, where the contracts module states them; a generated go.mod or package.json is kept. + if _, ok := files["go/go.mod"]; !ok { + writeContract(t, filepath.Join(dir, "go/go.mod"), "module example.org/c/go\n") + } + if _, ok := files["ts/package.json"]; !ok { + writeContract(t, filepath.Join(dir, "ts/package.json"), `{"name":"@example/c"}`) + } + return dir, paths +} + +func writeContract(t *testing.T, path, text string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(text), 0o644); err != nil { + t.Fatal(err) + } +} + +// **CAT-BG5, CAT-BN2**: a contracts module's generated files are regenerated and compared, each build module its own +// root; a file that differs, one missing and one the generator did not write are each named; a clean tree and a module +// naming no generated-from name none; a build ask answers them in its out-of-step. +func TestAContractsModulesGeneratedFilesAreComparedWithTheGenerator(t *testing.T) { + marked := `{"module":"c","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"],"generated-from":"."}]}}` + dir, paths := contracts(t, marked) + var goFile, tsFile string + for _, p := range paths { + if strings.HasPrefix(p, "go/") && strings.HasSuffix(p, ".go") && goFile == "" { + goFile = p + } + if strings.HasPrefix(p, "ts/") && strings.HasSuffix(p, ".ts") && tsFile == "" { + tsFile = p + } + } + if goFile == "" || tsFile == "" { + t.Fatalf("the generator wrote no Go or no TypeScript file: %v", paths) + } + for _, root := range []string{GeneratedRootGo, GeneratedRootTS} { + if got, err := OutOfStep(dir, root); err != nil || len(got) != 0 { + t.Fatalf("a clean tree, root %s: %v %v", root, got, err) + } + } + writeContract(t, filepath.Join(dir, goFile), "package changed\n") + writeContract(t, filepath.Join(dir, "go/extra.go"), "package extra\n") + writeContract(t, filepath.Join(dir, tsFile), "// changed\n") + got, err := OutOfStep(dir, GeneratedRootGo) + if want := sorted(goFile, "go/extra.go"); err != nil || !reflect.DeepEqual(got, want) { + t.Errorf("Go root: %v %v, want %v", got, err, want) + } + if got, _ := OutOfStep(dir, GeneratedRootTS); !reflect.DeepEqual(got, []string{tsFile}) { + t.Errorf("TypeScript root: %v, want %s alone", got, tsFile) + } + _ = os.Remove(filepath.Join(dir, goFile)) + if got, _ := OutOfStep(dir, GeneratedRootGo); !reflect.DeepEqual(got, sorted(goFile, "go/extra.go")) { + t.Errorf("a missing file is not named: %v", got) + } + plain, _ := contracts(t, `{"module":"p","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"]}]}}`) + writeContract(t, filepath.Join(plain, "go/extra.go"), "package extra\n") + if got, err := OutOfStep(plain, GeneratedRootGo); err != nil || len(got) != 0 { + t.Errorf("a module naming no generated-from compared: %v %v", got, err) + } + + // The build ask answers them beside what it built, and with a failed build too. + built := 0 + steps := AskSteps{Compare: CompareStep(GeneratedRootGo, func(ws string) string { return ws }), + Fetch: func(context.Context, map[string]any) (any, error) { + return map[string]any{"workspace": filepath.Dir(dir)}, nil + }, + Build: func(context.Context, map[string]any) (any, error) { + if built++; built > 1 { + return nil, errors.New("the second artifact does not build") + } + return map[string]any{"digest": "sha256:aa"}, nil + }} + job := delivery.BuildJob{Module: "c", Path: filepath.Base(dir), Fetch: map[string]any{"repository": "novox/c"}, + Artifacts: []delivery.AskArtifact{{Artifact: "go"}}} + out, err := RunAsk(context.Background(), steps, ask(job)) + if a, ok := out.(delivery.BuildAnswer); err != nil || !ok || !reflect.DeepEqual(a.OutOfStep, sorted(goFile, "go/extra.go")) || len(a.Artifacts) != 1 { + t.Errorf("the build ask's out-of-step: %+v %v", out, err) + } + job.Artifacts = append(job.Artifacts, delivery.AskArtifact{Artifact: "go-other"}) + built = 0 + out, err = RunAsk(context.Background(), steps, ask(job)) + if a, ok := out.(delivery.BuildAnswer); err == nil || !ok || len(a.OutOfStep) != 2 { + t.Errorf("a failed build answered no out-of-step: %+v %v", out, err) + } +} + +func sorted(paths ...string) []string { + sort.Strings(paths) + return paths +} diff --git a/modules/mesh-build-go/buildstep/go.mod b/modules/mesh-build-go/buildstep/go.mod index d61305e9..f7f7be73 100644 --- a/modules/mesh-build-go/buildstep/go.mod +++ b/modules/mesh-build-go/buildstep/go.mod @@ -8,6 +8,14 @@ require ( ) require ( + git.novox.be/novox/mesh-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/gopackageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/npmpackageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/ociartifactregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.0 // indirect diff --git a/modules/mesh-build-go/buildstep/latepublish.go b/modules/mesh-build-go/buildstep/latepublish.go new file mode 100644 index 00000000..cadb6b7e --- /dev/null +++ b/modules/mesh-build-go/buildstep/latepublish.go @@ -0,0 +1,91 @@ +package buildstep + +// **The late publish** (novox/hq ADR 0617 §1, design 51 *The versions*, *Publishing*; CAT-BG2, CAT-BI1): a trunk build +// whose number the builder did not know yet publishes its digest unnumbered; the builder says version.numbered once it +// numbers that fingerprint, and the build module publishes the held digest again under that version. Nothing travels: +// the publish names the one part by its digest, which the holder holds, so it answers held and tags the version (the +// holder asks builder_versions first, ADR 0605 §4). The bytes never stayed on the node, so a holder that no longer holds +// the digest is said and the event offered again. Every instance may hear the same event; the publish is the same call +// under the same transfer name, so a second one is answered held. One place for every build module: mesh-build-go's +// bundles and archives on oci-artifact-registry, mesh-build-image's images on container-image-registry. + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "slices" + "time" + + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/parts" +) + +// NumberedPattern is the event type a build module consumes for the late publish, as its manifest's `consumes` names it. +const NumberedPattern = delivery.BuilderModule + "." + delivery.VersionNumberedEvent + +// errNotHere is a held part's Open: its bytes did not stay on the node. +var errNotHere = errors.New("the built bytes did not stay on the node; the holder must hold the digest") + +// LatePublish publishes a numbered build's held digest under its version on seat, from the builder's version.numbered +// body, when its kind is one of kinds. It answers false, nil for an event that is not the module's to publish: another +// kind, another seat (an event naming none is the artifact seat's, as the builder wrote it before images), or a version +// numbered with no digest. +func LatePublish(ctx context.Context, call SeatCall, body []byte, seat string, kinds ...string) (bool, error) { + n, err := delivery.DecodeVersionNumbered(body) + if err != nil { + return false, err + } + on := n.Seat + if on == "" { + on = ArtifactSeat + } + if n.Digest == "" || on != seat || !slices.Contains(kinds, n.Kind) { + return false, nil + } + in := reg.PublishInput{Name: n.Name, Kind: n.Kind, Version: n.Version, Tag: n.Tag, Digest: n.Digest, + Fingerprint: n.Fingerprint, Commit: n.Commit, Transfer: TransferName(n.Name, "v"+n.Version, n.Digest)} + sent, err := OnSeat(ctx, seat, "late publish of "+n.Name+" "+n.Version+" at "+n.Digest, func() (parts.Sent, error) { + return PublishParts(ctx, call, seat, in, []parts.Part{{Digest: n.Digest, + Open: func() (io.ReadCloser, error) { return nil, errNotHere }}}) + }, nil) + if err != nil { + return false, fmt.Errorf("the late publish of %s@%s (%s): %w", n.Name, n.Version, n.Digest, err) + } + if len(sent.Parts) > 0 { + return false, fmt.Errorf("the late publish of %s@%s sent parts it should not have", n.Name, n.Version) + } + return true, nil +} + +// HearNumbered subscribes late to the builder's version.numbered, asking again until the runtime takes it, and says +// each publish and each failure; an error is the event's, which is offered again. +func HearNumbered(ctx context.Context, late func(ctx context.Context, body []byte) (bool, error)) { + handle := func(e stdio.Envelope) error { + did, err := late(ctx, e.Body) + if err != nil { + log.Printf("%v", err) + return err + } + if did { + log.Printf("published a numbered build's held digest under its version (%s)", e.Headers["x-event-id"]) + } + return nil + } + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.Subscribe(NumberedPattern, handle) + if err == nil { + log.Printf("listening for the builder's %s", NumberedPattern) + return + } + log.Printf("not hearing the builder's %s yet (%v); asking again in %s", NumberedPattern, err, wait) + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + } +} diff --git a/modules/mesh-build-go/buildstep/narrate.go b/modules/mesh-build-go/buildstep/narrate.go new file mode 100644 index 00000000..6e5cfbc9 --- /dev/null +++ b/modules/mesh-build-go/buildstep/narrate.go @@ -0,0 +1,106 @@ +package buildstep + +// **A build narrates its asks of a seat** (CAT-NR1; design 51 *Publishing*): asked, waiting on the seat (said again every +// NarrateEvery while it waits), stored, refused. Each line is the ask's phase (phase.go): logged with the ask's id and +// written to its running step key, so the builder and the delivery view read from the bus where a publish or a copy +// stands; outside an ask's run (a late publish, a tool called by hand) it is logged. One place for every build module: +// PublishOne and PublishFiles narrate through it, and a step that waits on a seat's event (an image's copy) uses +// NarrateSeat itself. + +import ( + "context" + "fmt" + "log" + "time" +) + +// NarrateEvery is how often a build says again that it still waits on a seat. +var NarrateEvery = 5 * time.Minute + +// narrateTicks is the clock of the waiting lines; a test hands its own. +var narrateTicks = func(every time.Duration) (<-chan time.Time, func()) { + t := time.NewTicker(every) + return t.C, t.Stop +} + +// SeatNarration is one ask of a build on a seat, said as it moves. +type SeatNarration struct { + ctx context.Context + seat, what string + asked time.Time + tick <-chan time.Time + stop func() +} + +// NarrateSeat says that the build asked seat for what (a verb and its object, such as "publish of nats/server") and +// starts the waiting lines; Stored or Refused ends it. +func NarrateSeat(ctx context.Context, seat, what string) *SeatNarration { + n := &SeatNarration{ctx: ctx, seat: seat, what: what, asked: now()} + n.tick, n.stop = narrateTicks(NarrateEvery) + n.say("asked %s: %s", seat, what) + return n +} + +// Waiting is the clock a waiting step selects on beside its outcome; on each tick it calls StillWaiting. +func (n *SeatNarration) Waiting() <-chan time.Time { return n.tick } + +// StillWaiting says that the build still waits on the seat, and since when. +func (n *SeatNarration) StillWaiting() { + n.say("waiting on %s: %s, %s since asked", n.seat, n.what, now().Sub(n.asked).Round(time.Second)) +} + +// Stored says that the seat holds what the build asked it to keep (how: where, or what travelled). +func (n *SeatNarration) Stored(how string) { + n.stop() + if how != "" { + how = " (" + how + ")" + } + n.say("stored on %s: %s%s", n.seat, n.what, how) +} + +// Refused says that the seat did not keep it, and why. +func (n *SeatNarration) Refused(err error) { + n.stop() + n.say("refused by %s: %s: %v", n.seat, n.what, err) +} + +func (n *SeatNarration) say(format string, a ...any) { + line := fmt.Sprintf(format, a...) + if r, ok := n.ctx.Value(phaseKey{}).(*phases); ok && r != nil { + r.enter(line) + return + } + log.Print(line) +} + +// OnSeat runs call as one narrated ask of the build on seat: asked, waiting every NarrateEvery while call runs, then +// stored (how(answer)) or refused (its error). +func OnSeat[T any](ctx context.Context, seat, what string, call func() (T, error), how func(T) string) (T, error) { + n := NarrateSeat(ctx, seat, what) + type answer struct { + v T + err error + } + done := make(chan answer, 1) + go func() { + v, err := call() + done <- answer{v, err} + }() + for { + select { + case a := <-done: + if a.err != nil { + n.Refused(a.err) + } else { + h := "" + if how != nil { + h = how(a.v) + } + n.Stored(h) + } + return a.v, a.err + case <-n.Waiting(): + n.StillWaiting() + } + } +} diff --git a/modules/mesh-build-go/buildstep/narrate_test.go b/modules/mesh-build-go/buildstep/narrate_test.go new file mode 100644 index 00000000..aa0cf34b --- /dev/null +++ b/modules/mesh-build-go/buildstep/narrate_test.go @@ -0,0 +1,107 @@ +package buildstep + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// CAT-NR1: a build's publish on a seat says asked, waiting on the seat (on every tick of NarrateEvery while the holder +// has not answered), and stored or refused, each as the ask's phase written to its running step key. +func TestABuildNarratesItsPublishOnASeat(t *testing.T) { + digest := "sha256:" + strings.Repeat("c", 64) + for name, c := range map[string]struct { + answer error + ticks int + want []string + }{ + "stored": {want: []string{ + "asked go-package-registry: publish of mod 1.2.3 at " + digest, + "stored on go-package-registry: publish of mod 1.2.3 at " + digest + " (0 of its 1 parts sent, the rest held)"}}, + "refused": {answer: errors.New("the holder is down"), want: []string{ + "asked go-package-registry: publish of mod 1.2.3 at " + digest, + "refused by go-package-registry: publish of mod 1.2.3 at " + digest + ": go-package-registry's publish of mod: the holder is down"}}, + "waiting twice, then stored": {ticks: 2, want: []string{ + "asked go-package-registry: publish of mod 1.2.3 at " + digest, + "waiting on go-package-registry: publish of mod 1.2.3 at " + digest + ", 5m0s since asked", + "waiting on go-package-registry: publish of mod 1.2.3 at " + digest + ", 10m0s since asked", + "stored on go-package-registry: publish of mod 1.2.3 at " + digest + " (0 of its 1 parts sent, the rest held)"}}, + } { + t.Run(name, func(t *testing.T) { + clock := time.Date(2026, 11, 4, 16, 0, 0, 0, time.UTC) + var mu sync.Mutex + oldNow, oldTicks := now, narrateTicks + now = func() time.Time { mu.Lock(); defer mu.Unlock(); return clock } + tick := make(chan time.Time) + stopped := make(chan struct{}) + narrateTicks = func(every time.Duration) (<-chan time.Time, func()) { + if every != 5*time.Minute { + t.Errorf("the waiting lines come every %s, want 5m", every) + } + return tick, func() { close(stopped) } + } + t.Cleanup(func() { now, narrateTicks = oldNow, oldTicks }) + + var said []string + wrote := make(chan struct{}, 16) + step := delivery.Step{ID: "a1", Tool: "build", Node: "ace", State: delivery.StepRunning, Started: clock} + ph := &phases{key: "ace.a1", step: step, put: func(key string, value any) (uint64, error) { + var s delivery.Step + _ = json.Unmarshal(value.(json.RawMessage), &s) + phasesMu.Lock() + said = append(said, s.Phase) + phasesMu.Unlock() + wrote <- struct{}{} + return 1, nil + }} + release := make(chan struct{}) + call := func(key string, body any) (json.RawMessage, error) { + <-release + if c.answer != nil { + return nil, c.answer + } + return json.Marshal(reg.PublishAnswer{Held: true, Transfer: "t"}) + } + ctx := context.WithValue(context.Background(), phaseKey{}, ph) + done := make(chan error, 1) + go func() { done <- PublishOne(ctx, call, GoSeat, "mod", "package", "1.2.3", digest, "t", []byte("x")) }() + line := func() { + select { + case <-wrote: + case <-time.After(5 * time.Second): + t.Fatal("the publish said no line") + } + } + line() // asked, at the clock's start + for i := 0; i < c.ticks; i++ { + mu.Lock() + clock = clock.Add(5 * time.Minute) + mu.Unlock() + tick <- clock + line() // its waiting line + } + close(release) + err := <-done + if (err != nil) != (c.answer != nil) { + t.Fatalf("PublishOne = %v", err) + } + <-stopped + phasesMu.Lock() + defer phasesMu.Unlock() + // Each line is the step's phase, written to its key, in order. + if fmt.Sprint(said) != fmt.Sprint(c.want) { + t.Errorf("phases =\n%s\nwant\n%s", strings.Join(said, "\n"), strings.Join(c.want, "\n")) + } + }) + } +} + +var phasesMu sync.Mutex diff --git a/modules/mesh-build-go/buildstep/publish.go b/modules/mesh-build-go/buildstep/publish.go new file mode 100644 index 00000000..3fee43d8 --- /dev/null +++ b/modules/mesh-build-go/buildstep/publish.go @@ -0,0 +1,116 @@ +package buildstep + +// The publish of a done build's file through a registry seat (novox/hq design 51 *Publishing*, ADR 0611; CAT-BG1, +// CAT-BN3): from the build module's own process, after the build step ended, through the seat's verbs publish and +// publish-part, which each build module's manifest `invokes`, in chunks, only what the holder lacks. The holder checks +// the build with the builder and says published or refused on the seat's events. One place for every build module: it +// binds each seat to its client in mesh-contracts (ADR 0627 §1), over mesh-sdk's seat-neutral sender of parts (§2). + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + imageclient "git.novox.be/novox/mesh-contracts/go/clients/containerimageregistryclient" + goclient "git.novox.be/novox/mesh-contracts/go/clients/gopackageregistryclient" + npmclient "git.novox.be/novox/mesh-contracts/go/clients/npmpackageregistryclient" + ociclient "git.novox.be/novox/mesh-contracts/go/clients/ociartifactregistryclient" + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + goseat "git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry" + npmseat "git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry" + ociseat "git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry" + "git.novox.be/novox/mesh-sdk/go/parts" +) + +// The registry seats a build module publishes on. +const ( + ArtifactSeat = ociseat.Seat + ImageSeat = reg.Seat + GoSeat = goseat.Seat + NpmSeat = npmseat.Seat +) + +// SeatCall asks one verb of a seat, `seat:.`, with a body, and answers its JSON (the bundle's stdio.Ask). +type SeatCall = func(key string, body any) (json.RawMessage, error) + +// TransferName is a transfer's name, the build module's own: the name, a qualifier and the digest's head, safe as a +// transfer id. +func TransferName(name, qualifier, digest string) string { + head := strings.TrimPrefix(digest, "sha256:") + if len(head) > 16 { + head = head[:16] + } + return strings.NewReplacer("/", "-", ".", "-", " ", "-", "*", "-", ">", "-", "@", "-").Replace(name + "-" + qualifier + "-" + head) +} + +// PublishOne publishes body, of digest, as name (at version, when it has one) of kind on seat, in one part. +func PublishOne(ctx context.Context, call SeatCall, seat, name, kind, version, digest, transfer string, body []byte) error { + in := reg.PublishInput{Name: name, Kind: kind, Version: version, Digest: digest, Transfer: transfer} + _, err := OnSeat(ctx, seat, publishWhat(in), func() (parts.Sent, error) { + return PublishParts(ctx, call, seat, in, []parts.Part{{Digest: digest, Size: int64(len(body)), + Open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(body)), nil }}}) + }, SentHow(1)) + return err +} + +// publishWhat names a publish in a narration: its name, its version when it has one, and its digest. +func publishWhat(in reg.PublishInput) string { + what := "publish of " + in.Name + if in.Version != "" { + what += " " + in.Version + } + return what + " at " + in.Digest +} + +// SentHow says what of a publish of parts travelled, for its stored line. +func SentHow(n int) func(parts.Sent) string { + return func(s parts.Sent) string { + return fmt.Sprintf("%d of its %d parts sent, the rest held", len(s.Parts), n) + } +} + +// PublishParts publishes in with ps as its parts on seat, through that seat's client: in's fields are the seat's own +// (the four registry seats share the publish verb's input, design 51), the parts are written by the client. +func PublishParts(ctx context.Context, call SeatCall, seat string, in reg.PublishInput, ps []parts.Part) (parts.Sent, error) { + switch seat { + case ImageSeat: + s, err := imageclient.Publish(ctx, imageclient.Call(call), in, ps) + return s.Sent, err + case ArtifactSeat: + s, err := ociclient.Publish(ctx, ociclient.Call(call), ociseat.PublishInput{Name: in.Name, Kind: in.Kind, Version: in.Version, + Tag: in.Tag, Digest: in.Digest, Fingerprint: in.Fingerprint, Commit: in.Commit, Transfer: in.Transfer}, ps) + return s.Sent, err + case GoSeat: + s, err := goclient.Publish(ctx, goclient.Call(call), goseat.PublishInput{Name: in.Name, Kind: in.Kind, Version: in.Version, + Tag: in.Tag, Digest: in.Digest, Fingerprint: in.Fingerprint, Commit: in.Commit, Transfer: in.Transfer}, ps) + return s.Sent, err + case NpmSeat: + s, err := npmclient.Publish(ctx, npmclient.Call(call), npmseat.PublishInput{Name: in.Name, Kind: in.Kind, Version: in.Version, + Tag: in.Tag, Digest: in.Digest, Fingerprint: in.Fingerprint, Commit: in.Commit, Transfer: in.Transfer}, ps) + return s.Sent, err + } + return parts.Sent{}, fmt.Errorf("%s is no registry seat a build publishes on", seat) +} + +// File is one part of a publish kept on disk: an image's config, layer or manifest, by its digest and size. +type File struct { + Digest string + Size int64 + Path string +} + +// PublishFiles publishes in with files as its parts (an image's config, layers and manifest, CAT-BI1): the holder +// answers the parts it lacks, and only those are read and sent. +func PublishFiles(ctx context.Context, call SeatCall, seat string, in reg.PublishInput, files []File) (parts.Sent, error) { + ps := make([]parts.Part, 0, len(files)) + for _, f := range files { + ps = append(ps, parts.Part{Digest: f.Digest, Size: f.Size, Open: func() (io.ReadCloser, error) { return os.Open(f.Path) }}) + } + return OnSeat(ctx, seat, publishWhat(in), func() (parts.Sent, error) { + return PublishParts(ctx, call, seat, in, ps) + }, SentHow(len(files))) +} diff --git a/modules/mesh-build-go/buildstep/seatgen/clients.go b/modules/mesh-build-go/buildstep/seatgen/clients.go new file mode 100644 index 00000000..13e85ac3 --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/clients.go @@ -0,0 +1,217 @@ +package seatgen + +import ( + "encoding/json" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + re "regexp" + "sort" + "strconv" + "strings" + + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// ClientRoot is where a seat's hand-written client packages live (ADR 0625 §4, §5; ADR 0627): one directory per +// client, `go/clients/client`, a Go module of its own (ADR 0625 §3), so the seat set's importers +// never import a client and a client's import of mesh-sdk forms no cycle. Compare leaves the tree to CheckClients. +const ClientRoot = "go/clients" + +// RuntimeModule is mesh-sdk's Go module: the one module beside its own seat's contract a client may import (ADR 0625 +// §3, §4). +const RuntimeModule = "git.novox.be/novox/mesh-sdk/go" + +// ClientGoModule is the Go module of a seat's client, beside its contract's. +func ClientGoModule(root, seat string) string { return root + "/clients/" + pkgName(seat) + "client" } + +// a module's tool address (`.`) or call (`.` written with an underscore). +var toolAddress = re.MustCompile(`^[a-z][a-z0-9-]*\.[a-z0-9]+_[a-z0-9_]+$`) + +// CheckClients is ADR 0627 §3's catalog check of the clients, and ADR 0625 §5's rule for the files beside them: every +// directory under ClientRoot is the client of one seat defined whole, a Go module of its own at ClientGoModule; it +// imports only the standard library, its own seat's contract module and mesh-sdk's runtime (never another seat's +// contract or client); and no string in it names a module's subject (`mesh.mod.`), a module's tool, or another seat +// (its name, its subjects, its `seat:` call). Each problem is named once, sorted. +func CheckClients(dir string, m *Module) ([]string, error) { + var out []string + entries, err := os.ReadDir(filepath.Join(dir, ClientRoot)) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + bySeatPkg := map[string]string{} + for name, d := range m.Seats { + if d.Grain == seatdef.Whole { + bySeatPkg[pkgName(name)+"client"] = name + } + } + for _, e := range entries { + at := ClientRoot + "/" + e.Name() + if !e.IsDir() { + out = append(out, at+": a file under the clients' root, in no client") + continue + } + seat, ok := bySeatPkg[e.Name()] + if !ok { + out = append(out, at+": the client of no seat defined whole (a client is `client`)") + continue + } + mod, err := os.ReadFile(filepath.Join(dir, at, "go.mod")) + want := "module " + ClientGoModule(m.GoModule, seat) + if err != nil || strings.SplitN(string(mod), "\n", 2)[0] != want { + out = append(out, at+"/go.mod: not "+strconv.Quote(want)+": a client is a Go module of its own (ADR 0625 §3)") + } + problems, err := checkClient(dir, at, seat, m) + if err != nil { + return nil, err + } + out = append(out, problems...) + } + sort.Strings(out) + return out, nil +} + +func checkClient(dir, at, seat string, m *Module) ([]string, error) { + var out []string + own := SeatGoModule(m.GoModule, seat) + others := map[string]bool{} + for name := range m.Seats { + if name != seat { + others[name] = true + } + } + namesOther := func(s string) bool { + if others[s] { + return true + } + for name := range others { + if strings.HasPrefix(s, "mesh.seat."+name+".") || strings.HasPrefix(s, "seat:"+name+".") { + return true + } + } + return false + } + err := filepath.WalkDir(filepath.Join(dir, at), func(p string, e os.DirEntry, err error) error { + if err != nil || e.IsDir() || !strings.HasSuffix(p, ".go") { + return err + } + r := rel(dir, p) + f, err := parser.ParseFile(token.NewFileSet(), p, nil, parser.SkipObjectResolution) + if err != nil { + return err + } + for _, im := range f.Imports { + path, _ := strconv.Unquote(im.Path.Value) + switch { + case !strings.Contains(strings.SplitN(path, "/", 2)[0], "."): + case path == own: + case path == RuntimeModule || strings.HasPrefix(path, RuntimeModule+"/"): + case strings.HasPrefix(path, m.GoModule+"/clients/"): + out = append(out, r+": imports "+path+", another seat's client (ADR 0625 §4)") + case path == m.GoModule || strings.HasPrefix(path, m.GoModule+"/"): + out = append(out, r+": imports "+path+", not its own seat's contract (ADR 0625 §4)") + default: + out = append(out, r+": imports "+path+": a client uses only its seat's contract and mesh-sdk's runtime (ADR 0625 §3)") + } + } + ast.Inspect(f, func(n ast.Node) bool { + lit, ok := n.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return true + } + v, err := strconv.Unquote(lit.Value) + if err != nil { + return true + } + switch { + case strings.HasPrefix(v, "mesh.mod."): + out = append(out, fmt.Sprintf("%s: names a module's subject %q (ADR 0625 §4)", r, v)) + case toolAddress.MatchString(v): + out = append(out, fmt.Sprintf("%s: names a module's tool %q (ADR 0625 §4)", r, v)) + case namesOther(v): + out = append(out, fmt.Sprintf("%s: names another seat %q (ADR 0625 §4)", r, v)) + } + return true + }) + return nil + }) + return out, err +} + +// TSSetArtifact is the seat set's npm package artifact, at ts/. +const TSSetArtifact = "lib" + +// CheckArtifacts is the check of module.json's package artifacts against the tree (ADR 0625 §1, §3), per language. +// Go: the root module is the artifact `go` at `go`, each seat defined whole the artifact `go-` +// at its module's directory, each client present the artifact `go-client` at its directory. +// npm: the seat set's package is the artifact `lib` at `ts`, each seat defined whole the artifact +// `ts-` at TSSeatDir. No other package artifact of either language. Each problem is named once, +// sorted. +func CheckArtifacts(dir string, m *Module) ([]string, error) { + raw, err := os.ReadFile(filepath.Join(dir, "module.json")) + if err != nil { + return nil, err + } + var mf struct { + Build struct { + Artifacts []struct { + Name string `json:"name"` + Kind string `json:"kind"` + Language string `json:"language"` + Paths []string `json:"paths"` + } `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &mf); err != nil { + return nil, fmt.Errorf("module.json: %w", err) + } + want := map[string]map[string]string{ + "go": {"go": "go"}, + "typescript": {TSSetArtifact: "ts"}, + } + for name, d := range m.Seats { + if d.Grain == seatdef.Whole { + want["go"]["go-"+pkgName(name)] = "go/seats/" + pkgName(name) + want["typescript"]["ts-"+pkgName(name)] = TSSeatDir(name) + } + } + if entries, err := os.ReadDir(filepath.Join(dir, ClientRoot)); err == nil { + for _, e := range entries { + if e.IsDir() { + want["go"]["go-"+e.Name()] = ClientRoot + "/" + e.Name() + } + } + } + said := map[string]string{"go": "Go", "typescript": "npm"} + var out []string + seen := map[string]bool{} + for _, a := range mf.Build.Artifacts { + paths, lang := want[a.Language] + if a.Kind != "package" || !lang { + continue + } + seen[a.Language+"/"+a.Name] = true + path, ok := paths[a.Name] + switch { + case !ok: + out = append(out, "module.json: the "+said[a.Language]+" package artifact "+a.Name+" is no seat's module, no client's and not the seat set's") + case len(a.Paths) != 1 || a.Paths[0] != path: + out = append(out, fmt.Sprintf("module.json: the artifact %s's paths are %v, not [%s]", a.Name, a.Paths, path)) + } + } + for lang, paths := range want { + for name, path := range paths { + if !seen[lang+"/"+name] { + out = append(out, "module.json: no "+said[lang]+" package artifact "+name+" for "+path) + } + } + } + sort.Strings(out) + return out, nil +} diff --git a/modules/mesh-build-go/buildstep/seatgen/clients_test.go b/modules/mesh-build-go/buildstep/seatgen/clients_test.go new file mode 100644 index 00000000..9a979589 --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/clients_test.go @@ -0,0 +1,148 @@ +package seatgen + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// ADR 0627 §3's catalog check of a client, ADR 0625 §5's rule for what is hand-written, and module.json's Go package +// artifacts against the tree (ADR 0625 §1, §3): each row writes one tree and names what the checks must say of it. +func TestAClientUsesOnlyItsSeatAndTheRuntime(t *testing.T) { + m, err := Load("testdata/contracts", "example.org/contracts/go", "@example/contracts") + if err != nil { + t.Fatal(err) + } + files, err := Generate(m) + if err != nil { + t.Fatal(err) + } + const good = `package aregistryclient + +import ( + "context" + + "example.org/contracts/go/seats/aregistry" + "git.novox.be/novox/mesh-sdk/go/parts" +) + +func Publish(ctx context.Context) string { _ = parts.MaxChunk; return "seat:" + aregistry.Seat + ".publish" } +` + const ts = `,{"name":"lib","kind":"package","language":"typescript","paths":["ts"]}, +{"name":"ts-aregistry","kind":"package","language":"typescript","paths":["ts/seats/aregistry"]}, +{"name":"ts-bregistry","kind":"package","language":"typescript","paths":["ts/seats/bregistry"]}` + artifacts := func(extra string) string { + return `{"build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"]}, +{"name":"go-aregistry","kind":"package","language":"go","paths":["go/seats/aregistry"]}, +{"name":"go-bregistry","kind":"package","language":"go","paths":["go/seats/bregistry"]}` + ts + extra + `]}}` + } + clientArtifact := `,{"name":"go-aregistryclient","kind":"package","language":"go","paths":["go/clients/aregistryclient"]}` + for _, c := range []struct { + name string + tree map[string]string + module string + want []string + }{ + {"no client: the seats alone", nil, artifacts(""), nil}, + {"a client of its own seat over the runtime", map[string]string{ + "go/clients/aregistryclient/go.mod": "module example.org/contracts/go/clients/aregistryclient\n", + "go/clients/aregistryclient/client.go": good, + }, artifacts(clientArtifact), nil}, + {"a client with no artifact, and an artifact of no module", map[string]string{ + "go/clients/aregistryclient/go.mod": "module example.org/contracts/go/clients/aregistryclient\n", + "go/clients/aregistryclient/client.go": good, + }, artifacts(`,{"name":"go-extra","kind":"package","language":"go","paths":["go/extra"]}`), + []string{"module.json: no Go package artifact go-aregistryclient", "module.json: the Go package artifact go-extra is no seat's"}}, + {"a seat's npm package with no artifact, one at another path, and one of no package", nil, + `{"build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"]}, +{"name":"go-aregistry","kind":"package","language":"go","paths":["go/seats/aregistry"]}, +{"name":"go-bregistry","kind":"package","language":"go","paths":["go/seats/bregistry"]}, +{"name":"lib","kind":"package","language":"typescript","paths":["ts"]}, +{"name":"ts-aregistry","kind":"package","language":"typescript","paths":["ts"]}, +{"name":"ts-onestore","kind":"package","language":"typescript","paths":["ts/seats/onestore"]}]}}`, + []string{"module.json: no npm package artifact ts-bregistry", "module.json: the artifact ts-aregistry's paths are [ts], not [ts/seats/aregistry]", + "module.json: the npm package artifact ts-onestore is no seat's"}}, + {"a client of no seat defined whole, and one not a module of its own", map[string]string{ + "go/clients/onestoreclient/x.go": "package onestoreclient\n", + "go/clients/aregistryclient/go.mod": "module example.org/contracts/go\n", + }, artifacts(clientArtifact + `,{"name":"go-onestoreclient","kind":"package","language":"go","paths":["go/clients/onestoreclient"]}`), + []string{"go/clients/aregistryclient/go.mod: not", "go/clients/onestoreclient: the client of no seat defined whole"}}, + {"a client using another seat, its client, a module's subject and tool", map[string]string{ + "go/clients/aregistryclient/go.mod": "module example.org/contracts/go/clients/aregistryclient\n", + "go/clients/aregistryclient/client.go": `package aregistryclient + +import ( + "example.org/contracts/go/seats/bregistry" + "example.org/contracts/go/clients/bregistryclient" + "github.com/some/thing" +) + +var a, b, c, d, e = bregistry.Seat, bregistryclient.X, thing.Y, "mesh.mod.holder.event.x", "seat:b-registry.publish" +var f = "records.records_search" +`, + }, artifacts(clientArtifact), []string{ + `client.go: imports example.org/contracts/go/clients/bregistryclient, another seat's client`, + `client.go: imports example.org/contracts/go/seats/bregistry, not its own seat's contract`, + `client.go: imports github.com/some/thing: a client uses only`, + `client.go: names a module's subject "mesh.mod.holder.event.x"`, + `client.go: names a module's tool "records.records_search"`, + `client.go: names another seat "seat:b-registry.publish"`, + }}, + } { + dir := t.TempDir() + write := func(p, s string) { + full := filepath.Join(dir, p) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(s), 0o644); err != nil { + t.Fatal(err) + } + } + for p, b := range files { + write(p, string(b)) + } + for p, s := range c.tree { + write(p, s) + } + write("module.json", c.module) + var got []string + cmp, err := Compare(dir, files) + if err != nil { + t.Fatal(err) + } + got = append(got, cmp...) + for _, check := range []func(string, *Module) ([]string, error){CheckClients, CheckArtifacts} { + more, err := check(dir, m) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + got = append(got, more...) + } + if len(got) != len(c.want) { + t.Errorf("%s: said %d problems, want %d:\n%s", c.name, len(got), len(c.want), strings.Join(got, "\n")) + continue + } + for _, w := range c.want { + found := false + for _, g := range got { + found = found || strings.Contains(g, w) + } + if !found { + t.Errorf("%s: no problem says %q:\n%s", c.name, w, strings.Join(got, "\n")) + } + } + } + // A hand-written file outside a client is still out of step (ADR 0625 §5). + dir := t.TempDir() + for p, b := range files { + full := filepath.Join(dir, p) + _ = os.MkdirAll(filepath.Dir(full), 0o755) + _ = os.WriteFile(full, b, 0o644) + } + _ = os.WriteFile(filepath.Join(dir, "go/seats/aregistry/helper.go"), []byte("package aregistry\n"), 0o644) + if out, _ := Compare(dir, files); len(out) != 1 || !strings.Contains(out[0], "helper.go: not written by the generator") { + t.Errorf("a hand-written file beside a contract: %v", out) + } +} diff --git a/modules/mesh-build-go/buildstep/seatgen/gogen.go b/modules/mesh-build-go/buildstep/seatgen/gogen.go new file mode 100644 index 00000000..f7715b64 --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/gogen.go @@ -0,0 +1,688 @@ +package seatgen + +import ( + "bytes" + "encoding/json" + "fmt" + "go/format" + "sort" + "strconv" + "strings" + + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +const header = "// Code generated by seatgen from the seat definitions; DO NOT EDIT.\n\n" + +func goFiles(m *Module, out map[string][]byte) error { + // The seat set is the contracts module's root Go module; each seat defined whole is a Go module of its own beside + // it, so a change of one seat moves only its importers, whose fingerprint counts a Go module's go.mod and go.sum + // (ADR 0625 §1, §2). A seat's module imports nothing: the parts it includes are written into it whole. + out["go/go.mod"] = goMod(m.GoModule) + set, err := goSeatList(m) + if err != nil { + return err + } + out["go/seats/seats.go"] = set + for _, name := range m.Order { + d := m.Seats[name] + if d.Grain != seatdef.Whole { + continue + } + src, test, err := goWhole(m, d) + if err != nil { + return fmt.Errorf("seat %s: %w", name, err) + } + dir := "go/seats/" + pkgName(name) + "/" + out[dir+"go.mod"] = goMod(SeatGoModule(m.GoModule, name)) + out[dir+"contract.go"] = src + if test != nil { + out[dir+"contract_test.go"] = test + } + } + return nil +} + +// SeatGoModule is the Go module of a seat defined whole, beside the seat set's. +func SeatGoModule(root, seat string) string { return root + "/seats/" + pkgName(seat) } + +func goMod(path string) []byte { return []byte("module " + path + "\n\ngo 1.26.0\n") } + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func gofmt(b *bytes.Buffer) ([]byte, error) { + src, err := format.Source(b.Bytes()) + if err != nil { + return nil, fmt.Errorf("the generated Go does not parse: %w\n%s", err, b.String()) + } + return src, nil +} + +// goSeatList is go/seats/seats.go: the seat list's types, every seat's name and the compiled list in reading order. +func goSeatList(m *Module) ([]byte, error) { + var b bytes.Buffer + b.WriteString(header) + b.WriteString(`// Package seats is the mesh's seat list, generated from every seat's definition (design 26 *The seat set has one +// source*): what each seat is, where it is held, the provision it delivers, its verbs with their inputs, what it +// accepts and emits, what it receives and the paths it owns. mesh-sdk's runtime reads it; a seat its holder declares +// itself has a name here and no entry in the list. +package seats + +// Seat is one role the mesh defines. +type Seat struct { + Name string + Scope string + Grain string + Replicated bool + Delivers string + Accepts []string + Emits []string + PulledByKind bool + Yields bool + Serves []Verb + Receives []Receivable + Paths []Path + HeldByTheEngine bool + Decision string +} + +// Verb is one verb a seat serves: its input and answer as JSON Schema. +type Verb struct { + Name string + Description string + Input map[string]any + Output map[string]any + Optional bool + Replaces []string +} + +// Receivable is one kind of contribution a seat receives. +type Receivable struct { + Kind string + Comment string + Dirs bool + Shape *Shape + Offered bool +} + +// Shape is a kind given as data. Its checks are the runtime's. +type Shape struct { + Places []PlaceField + Order string + Examples []map[string]any + Text bool +} + +// PlaceField is one field a holder places by, and the values it takes. +type PlaceField struct { + Field string + Values []string +} + +// Path is one path on its machine a seat's holder owns. +type Path struct { + Path string + StepsBackFor string +} + +// The name of every seat defined. +const ( +`) + for _, name := range m.Order { + fmt.Fprintf(&b, "\t%s = %q\n", ident(name), name) + } + b.WriteString(")\n\n// Names is every seat defined, in reading order, the ones their holders declare included.\nfunc Names() []string {\n\treturn []string{") + for _, name := range m.Order { + b.WriteString(ident(name) + ", ") + } + b.WriteString("}\n}\n\n// List is the seat list, in reading order: a new copy each call.\nfunc List() []Seat {\n\treturn []Seat{\n") + for _, name := range m.Order { + d := m.Seats[name] + if d.HolderDeclares { + continue + } + b.WriteString(goSeat(d)) + } + b.WriteString("\t}\n}\n") + return gofmt(&b) +} + +func goSeat(d seatdef.Definition) string { + var b strings.Builder + fmt.Fprintf(&b, "{Name: %s, Scope: %q, Grain: %q, Decision: %q", ident(d.Seat), d.Scope, d.Grain, d.Decision) + if d.Replicated { + b.WriteString(", Replicated: true") + } + if d.Delivers != "" { + fmt.Fprintf(&b, ", Delivers: %q", d.Delivers) + } + if len(d.Accepts) > 0 { + b.WriteString(", Accepts: " + goLit(anyStrings(seatdef.MessageNames(d.Accepts)))) + } + if len(d.Events) > 0 { + b.WriteString(", Emits: " + goLit(anyStrings(seatdef.MessageNames(d.Events)))) + } + if d.PulledByKind { + b.WriteString(", PulledByKind: true") + } + if d.Yields { + b.WriteString(", Yields: true") + } + if d.HeldByTheEngine { + b.WriteString(", HeldByTheEngine: true") + } + if len(d.Verbs) > 0 { + b.WriteString(",\n\t\tServes: []Verb{\n") + for _, v := range d.Verbs { + fmt.Fprintf(&b, "\t\t\t{Name: %q, Description: %q", v.Name, v.Description) + if v.Optional { + b.WriteString(", Optional: true") + } + if v.Input != nil { + b.WriteString(",\n\t\t\t\tInput: " + goLit(resolveRefs(v.Input, d.Defs))) + } + if v.Answer != nil { + b.WriteString(",\n\t\t\t\tOutput: " + goLit(resolveRefs(v.Answer, d.Defs))) + } + if len(v.Replaces) > 0 { + b.WriteString(", Replaces: " + goLit(anyStrings(v.Replaces))) + } + b.WriteString("},\n") + } + b.WriteString("\t\t}") + } + if len(d.Receives) > 0 { + b.WriteString(",\n\t\tReceives: []Receivable{") + for _, r := range d.Receives { + fmt.Fprintf(&b, "{Kind: %q, Comment: %q", r.Kind, r.Comment) + if r.Dirs { + b.WriteString(", Dirs: true") + } + if r.Offered { + b.WriteString(", Offered: true") + } + if s := r.Shape; s != nil { + b.WriteString(", Shape: &Shape{") + if len(s.Places) > 0 { + b.WriteString("Places: []PlaceField{") + for _, p := range s.Places { + fmt.Fprintf(&b, "{Field: %q, Values: %s}, ", p.Field, goLit(anyStrings(p.Values))) + } + b.WriteString("}, ") + } + if s.Order != "" { + fmt.Fprintf(&b, "Order: %q, ", s.Order) + } + if len(s.Examples) > 0 { + b.WriteString("Examples: []map[string]any{") + for _, e := range s.Examples { + b.WriteString(goLit(e) + ", ") + } + b.WriteString("}, ") + } + if s.Text { + b.WriteString("Text: true") + } + b.WriteString("}") + } + b.WriteString("}, ") + } + b.WriteString("}") + } + if len(d.Paths) > 0 { + b.WriteString(", Paths: []Path{") + for _, p := range d.Paths { + fmt.Fprintf(&b, "{Path: %q, StepsBackFor: %q}, ", p.Path, p.StepsBackFor) + } + b.WriteString("}") + } + b.WriteString("},\n") + return b.String() +} + +func anyStrings(s []string) []any { + out := make([]any, len(s)) + for i, v := range s { + out[i] = v + } + return out +} + +// goLit is a JSON value as a Go literal, in the form encoding/json decodes it, but for a list of strings alone, +// which is a []string as the seat list wrote it: a map's keys sorted, a number a float64. +func goLit(v any) string { + switch x := v.(type) { + case nil: + return "nil" + case string: + return strconv.Quote(x) + case bool: + return strconv.FormatBool(x) + case float64: + return "float64(" + strconv.FormatFloat(x, 'g', -1, 64) + ")" + case int: + return "float64(" + strconv.Itoa(x) + ")" + case json.Number: + return "float64(" + x.String() + ")" + case []string: + return goLit(anyStrings(x)) + case []any: + allStrings := len(x) > 0 + for _, e := range x { + if _, ok := e.(string); !ok { + allStrings = false + } + } + var b strings.Builder + if allStrings { + b.WriteString("[]string{") + } else { + b.WriteString("[]any{") + } + for i, e := range x { + if i > 0 { + b.WriteString(", ") + } + b.WriteString(goLit(e)) + } + b.WriteString("}") + return b.String() + case map[string]any: + var b strings.Builder + b.WriteString("map[string]any{") + for i, k := range sortedKeys(x) { + if i > 0 { + b.WriteString(", ") + } + b.WriteString(strconv.Quote(k) + ": " + goLit(x[k])) + } + b.WriteString("}") + return b.String() + } + // Any other value (a typed list from a definition read in Go) goes through JSON once. + raw, _ := json.Marshal(v) + var back any + _ = json.Unmarshal(raw, &back) + return goLit(back) +} + +// declareAll declares a definition's types: its own named types, then each verb's input and answer, each accept, +// event, holder event, record and provision; a name given by a part (in fromPart) is an alias of the part's. +func declareAll(d seatdef.Definition, fromPart map[string]string) (*typer, error) { + tp := newTyper() + decl := func(name, desc string, s map[string]any) error { + if s == nil { + return nil + } + if _, ok := fromPart[name]; ok { + tp.taken[name] = true + return nil + } + return tp.declare(name, desc, s) + } + for _, n := range d.DefNames() { + if err := decl(n, "", d.Defs[n]); err != nil { + return nil, err + } + } + for _, v := range d.Verbs { + if err := decl(ident(v.Name)+"Input", "is what "+v.Name+" takes.", v.Input); err != nil { + return nil, err + } + if err := decl(ident(v.Name)+"Answer", "is what "+v.Name+" answers.", v.Answer); err != nil { + return nil, err + } + } + for _, a := range d.Accepts { + if err := decl(ident(a.Name)+"Accept", "is the body of "+a.Name+".", a.Body); err != nil { + return nil, err + } + } + for _, e := range d.Events { + if err := decl(ident(e.Name)+"Event", "is the body of the event "+e.Name+".", e.Body); err != nil { + return nil, err + } + } + for _, e := range d.HolderEvents { + if err := decl(ident(e.Name)+"HolderEvent", "is the body of the holder's event "+e.Name+".", e.Body); err != nil { + return nil, err + } + } + for _, r := range d.Records { + if err := decl(ident(r.Name)+"Record", "is one value of the record "+r.Name+".", r.Value); err != nil { + return nil, err + } + } + for _, p := range d.Provisions { + if err := decl(ident(p.Name)+"Form", "is the form of the provision "+p.Name+".", p.Form); err != nil { + return nil, err + } + } + return tp, nil +} + +func oneLine(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return "no description." + } + return strings.ReplaceAll(s, "\n", " ") +} + +func writeNames(b *bytes.Buffer, prefix string, names []string) { + if len(names) == 0 { + return + } + b.WriteString("const (\n") + for _, n := range names { + fmt.Fprintf(b, "\t%s%s = %q\n", prefix, ident(n), n) + } + b.WriteString(")\n\n") +} + +func writeList(b *bytes.Buffer, fn, prefix, what string, names []string) { + if len(names) == 0 { + return + } + fmt.Fprintf(b, "// %s is %s: a new copy each call.\nfunc %s() []string {\n\treturn []string{", fn, what, fn) + for i, n := range names { + if i > 0 { + b.WriteString(", ") + } + b.WriteString(prefix + ident(n)) + } + b.WriteString("}\n}\n\n") +} + +// goWhole is a whole seat's package: its names, subjects and types, and the test of its fixtures. +func goWhole(m *Module, d seatdef.Definition) ([]byte, []byte, error) { + fromPart := map[string]string{} + tp, err := declareAll(d, fromPart) + if err != nil { + return nil, nil, err + } + pkg := pkgName(d.Seat) + var b bytes.Buffer + b.WriteString(header) + fmt.Fprintf(&b, "// Package %s is the contract of the seat %q, defined whole (seats/%s/seat.json): %s\npackage %s\n\n", + pkg, d.Seat, d.Seat, oneLine(d.Description), pkg) + fmt.Fprintf(&b, "// Seat is the seat's name.\nconst Seat = %q\n\n", d.Seat) + writeNames(&b, "Verb", seatdef.VerbNames(d.Verbs)) + writeNames(&b, "Accept", seatdef.MessageNames(d.Accepts)) + writeNames(&b, "Event", seatdef.MessageNames(d.Events)) + writeList(&b, "Verbs", "Verb", "every verb the seat serves, in its definition's order", seatdef.VerbNames(d.Verbs)) + writeList(&b, "Events", "Event", "every event published on the seat's subject", seatdef.MessageNames(d.Events)) + if f := d.Family; f != nil { + fmt.Fprintf(&b, "// Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects.\nconst (\n\tFamily = %q\n\tFamilyMaxAgeDays = %d\n)\n\n", f.Name, f.MaxAgeDays) + } + for _, e := range d.Events { + subj := seatdef.EventSubject(d.Seat, e.Name) + if e.ByCaller { + fmt.Fprintf(&b, "// Subject%s is where %s is published for one caller.\nfunc Subject%s(caller string) string { return %q + caller }\n\n", + ident(e.Name), e.Name, ident(e.Name), subj+".") + } else { + fmt.Fprintf(&b, "// Subject%s is where %s is published.\nconst Subject%s = %q\n\n", ident(e.Name), e.Name, ident(e.Name), subj) + } + } + for _, a := range d.Accepts { + subj := seatdef.AcceptSubject(d.Seat, a.Name) + if a.ByCaller { + fmt.Fprintf(&b, "// AcceptSubject%s is where one caller sends %s.\nfunc AcceptSubject%s(caller string) string { return %q + caller }\n\n", + ident(a.Name), a.Name, ident(a.Name), subj+".") + } else { + fmt.Fprintf(&b, "// AcceptSubject%s is where %s is sent.\nconst AcceptSubject%s = %q\n\n", ident(a.Name), a.Name, ident(a.Name), subj) + } + } + if len(d.HolderEvents) > 0 { + b.WriteString("// The holder's own events, listed beside the seat and published under the holder's subjects (ADR 0620 §2).\nconst (\n") + for _, e := range d.HolderEvents { + fmt.Fprintf(&b, "\tHolderEvent%s = %q\n", ident(e.Name), e.Name) + } + b.WriteString(")\n\n") + for _, e := range d.HolderEvents { + before, after, _ := strings.Cut(e.Subject, "") + fmt.Fprintf(&b, "// HolderSubject%s is where the holder publishes %s", ident(e.Name), e.Name) + if e.Family != "" { + fmt.Fprintf(&b, ", kept in %s", e.Family) + } + fmt.Fprintf(&b, ".\nfunc HolderSubject%s(holder string) string { return %q + holder + %q }\n\n", ident(e.Name), before, after) + } + } + if len(d.Records) > 0 { + writeNames(&b, "Record", recordNames(d.Records)) + } + if err := goTypes(&b, tp, map[string]*typ{}); err != nil { + return nil, nil, err + } + src, err := gofmt(&b) + if err != nil { + return nil, nil, err + } + test, err := goFixtures(m, d, pkg, tp, fromPart) + return src, test, err +} + +func recordNames(rs []seatdef.Record) []string { + out := make([]string, len(rs)) + for i, r := range rs { + out[i] = r.Name + } + return out +} + +// goTypes writes every declared type; also are the kinds of types declared elsewhere (a part's), by name. +func goTypes(b *bytes.Buffer, tp *typer, also map[string]*typ) error { + kinds := map[string]*typ{} + for k, v := range also { + kinds[k] = v + } + for _, t := range tp.types { + kinds[t.name] = t.t + } + for _, t := range tp.types { + desc := strings.TrimSpace(t.desc) + if desc == "" { + desc = "is a type of the contract." + } else if !strings.HasPrefix(desc, "is ") { + desc = "is " + strings.TrimSuffix(desc, ".") + "." + } + fmt.Fprintf(b, "// %s %s\n", t.name, oneLine(desc)) + if t.t.kind != "struct" { + fmt.Fprintf(b, "type %s = %s\n\n", t.name, goType(t.t, kinds, true)) + goEnum(b, t.name, t.name, t.t) + continue + } + fmt.Fprintf(b, "type %s struct {\n", t.name) + for _, f := range t.t.fields { + if f.desc != "" { + fmt.Fprintf(b, "\t// %s\n", oneLine(f.desc)) + } + tag := f.json + if !f.required { + tag += ",omitempty" + } + fmt.Fprintf(b, "\t%s %s `json:%q`\n", ident(f.json), goType(f.t, kinds, f.required), tag) + } + b.WriteString("}\n\n") + for _, f := range t.t.fields { + goEnum(b, t.name+"'s "+f.json, t.name+ident(f.json), f.t) + } + } + return nil +} + +// goEnum writes a string's enum as one constant per value, named prefix and the value (RemoveInputReasonRetention), so a +// caller names a value the definition gives and never writes it by hand. +func goEnum(b *bytes.Buffer, what, prefix string, t *typ) { + if t.kind != "string" || len(t.enum) == 0 { + return + } + fmt.Fprintf(b, "// The values of %s.\nconst (\n", what) + for _, v := range t.enum { + fmt.Fprintf(b, "\t%s%s = %q\n", prefix, ident(v), v) + } + b.WriteString(")\n\n") +} + +func goType(t *typ, kinds map[string]*typ, required bool) string { + switch t.kind { + case "string": + return "string" + case "integer": + return "int64" + case "number": + return "float64" + case "boolean": + return "bool" + case "object": + return "map[string]any" + case "map": + return "map[string]" + goType(t.elem, kinds, true) + case "slice": + return "[]" + goType(t.elem, kinds, true) + case "ref": + if k, ok := kinds[t.ref]; ok && k.kind == "struct" && !required { + return "*" + t.ref + } + return t.ref + } + return "any" +} + +// goFixtures is the test that every fixture of the seat decodes into its type and reads back as its wire says, and +// that the subject it names is the one the contract gives. +func goFixtures(m *Module, d seatdef.Definition, pkg string, tp *typer, fromPart map[string]string) ([]byte, error) { + fx := m.Fixtures[d.Seat] + if len(fx) == 0 { + return nil, nil + } + has := map[string]bool{} + for _, t := range tp.types { + has[t.name] = true + } + for n := range fromPart { + has[n] = true + } + var b bytes.Buffer + b.WriteString(header) + fmt.Fprintf(&b, "package %s\n\nimport (\n\t\"encoding/json\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"testing\"\n)\n\n", pkg) + fmt.Fprintf(&b, "// TestFixtures reads every fixture of seats/%s/fixtures: each decodes into its type, a field the type does not\n"+ + "// know is dropped (ADR 0608 §6), and it reads back as its wire says; each subject named is the contract's.\n", d.Seat) + b.WriteString("func TestFixtures(t *testing.T) {\n\tcases := []struct {\n\t\tname, subject, given, wire string\n\t\tdecode func([]byte) (any, error)\n\t\tsubjectIs *regexp.Regexp\n\t}{\n") + for _, f := range fx { + kind, name, _ := f.Instance() + typeName, pattern, err := fixtureTarget(d, kind, name) + if err != nil { + return nil, fmt.Errorf("fixture %q: %w", f.Name, err) + } + given, _ := json.Marshal(f.Given) + wire := given + if f.Wire != nil { + wire, _ = json.Marshal(f.Wire) + } + decoder := "func(b []byte) (any, error) { var v map[string]any; err := json.Unmarshal(b, &v); return v, err }" + if has[typeName] { + decoder = fmt.Sprintf("func(b []byte) (any, error) { var v %s; err := json.Unmarshal(b, &v); return v, err }", typeName) + } + subjectIs := "nil" + if pattern != "" { + subjectIs = "regexp.MustCompile(" + strconv.Quote(pattern) + ")" + } else if f.Subject != "" { + return nil, fmt.Errorf("fixture %q names a subject, and a verb's message has none in the contract", f.Name) + } + if pattern != "" && f.Subject == "" { + return nil, fmt.Errorf("fixture %q of %s names no subject (design 26: each event's subject is named)", f.Name, f.Of) + } + fmt.Fprintf(&b, "\t\t{name: %q, subject: %q, given: %s, wire: %s,\n\t\t\tdecode: %s, subjectIs: %s},\n", + f.Name, f.Subject, strconv.Quote(string(given)), strconv.Quote(string(wire)), decoder, subjectIs) + } + b.WriteString(` } + for _, c := range cases { + v, err := c.decode([]byte(c.given)) + if err != nil { + t.Errorf("%s: does not decode: %v", c.name, err) + continue + } + back, err := json.Marshal(v) + if err != nil { + t.Errorf("%s: does not encode: %v", c.name, err) + continue + } + var got, want any + if err := json.Unmarshal(back, &got); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(c.wire), &want); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: reads back as %s, want %s", c.name, back, c.wire) + } + if c.subjectIs != nil && !c.subjectIs.MatchString(c.subject) { + t.Errorf("%s: subject %q is not the contract's %s", c.name, c.subject, c.subjectIs) + } + } +} +`) + return gofmt(&b) +} + +// fixtureTarget is the type a fixture decodes into and the pattern its subject must match, if it has one. +func fixtureTarget(d seatdef.Definition, kind, name string) (string, string, error) { + switch kind { + case "verb-input", "verb-answer": + for _, v := range d.Verbs { + if v.Name == name { + if kind == "verb-input" { + return ident(name) + "Input", "", nil + } + return ident(name) + "Answer", "", nil + } + } + case "accept", "event": + ms, subj := d.Accepts, seatdef.AcceptSubject(d.Seat, name) + suffix := "Accept" + if kind == "event" { + ms, subj, suffix = d.Events, seatdef.EventSubject(d.Seat, name), "Event" + } + for _, msg := range ms { + if msg.Name == name { + p := "^" + regexp(subj) + if msg.ByCaller { + p += `\.[A-Za-z0-9_-]+` + } + return ident(name) + suffix, p + "$", nil + } + } + case "holder-event": + for _, e := range d.HolderEvents { + if e.Name == name { + before, after, _ := strings.Cut(e.Subject, "") + p := "^" + regexp(before) + if strings.Contains(e.Subject, "") { + p += "[a-z0-9-]+" + regexp(after) + } + return ident(name) + "HolderEvent", p + "$", nil + } + } + } + return "", "", fmt.Errorf("the definition has no %s %q", kind, name) +} + +func regexp(s string) string { + var b strings.Builder + for _, r := range s { + if strings.ContainsRune(`\.+*?()|[]{}^$`, r) { + b.WriteRune('\\') + } + b.WriteRune(r) + } + return b.String() +} diff --git a/modules/mesh-build-go/buildstep/seatgen/layout_test.go b/modules/mesh-build-go/buildstep/seatgen/layout_test.go new file mode 100644 index 00000000..c7848ab6 --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/layout_test.go @@ -0,0 +1,144 @@ +package seatgen + +import ( + "encoding/json" + "go/parser" + "go/token" + re "regexp" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// TestEachSeatDefinedWholeIsAGoModuleOfItsOwn is ADR 0625's generator check (§1, §2, §3): the seat set is one package in +// the root Go module, each seat defined whole is a Go module of its own holding its package alone, the parts it +// includes are written into it, and no generated Go file imports anything outside the standard library. +func TestEachSeatDefinedWholeIsAGoModuleOfItsOwn(t *testing.T) { + m, err := Load("testdata/contracts", "example.org/contracts/go", "@example/contracts") + if err != nil { + t.Fatal(err) + } + files, err := Generate(m) + if err != nil { + t.Fatal(err) + } + mods := map[string]string{} + for p, b := range files { + if strings.HasPrefix(p, "go/") && strings.HasSuffix(p, "go.mod") { + mods[strings.TrimSuffix(p, "go.mod")] = strings.SplitN(string(b), "\n", 2)[0] + } + } + want := map[string]string{ + "go/": "module example.org/contracts/go", + "go/seats/aregistry/": "module example.org/contracts/go/seats/aregistry", + "go/seats/bregistry/": "module example.org/contracts/go/seats/bregistry", + } + for dir, line := range want { + if mods[dir] != line { + t.Errorf("%sgo.mod says %q, want %q", dir, mods[dir], line) + } + } + // Every seat the definitions mark whole has its own module, whatever the seat set says of it (ADR 0627: the seat + // set marks ten whole, and none is without its package). + for name, d := range m.Seats { + if d.Grain == seatdef.Whole && mods["go/seats/"+pkgName(name)+"/"] == "" { + t.Errorf("the seat %s is defined whole and has no Go module of its own", name) + } + } + if len(mods) != len(want) { + t.Errorf("Go modules %v, want %v", mods, want) + } + for p, b := range files { + if !strings.HasPrefix(p, "go/") || !strings.HasSuffix(p, ".go") { + continue + } + dir := p[:strings.LastIndex(p, "/")+1] + if dir != "go/seats/" && mods[dir] == "" { + t.Errorf("%s is in no seat's own Go module, nor the seat set's package", p) + } + f, err := parser.ParseFile(token.NewFileSet(), p, b, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, im := range f.Imports { + if path := strings.Trim(im.Path.Value, `"`); strings.Contains(strings.SplitN(path, "/", 2)[0], ".") { + t.Errorf("%s imports %s: a generated package imports nothing outside the standard library", p, path) + } + } + } + for _, seat := range []string{"aregistry", "bregistry"} { + src := string(files["go/seats/"+seat+"/contract.go"]) + if !strings.Contains(src, "type Ref struct") || !strings.Contains(src, "type PublishedEvent") { + t.Errorf("%s does not hold the part's types whole", seat) + } + // A string's enum is a constant per value, so a caller names the definition's value (A3t: the removal reason). + if !re.MustCompile(`RefReasonRetention\s+= "retention"`).MatchString(src) || !re.MustCompile(`RefReasonTagGone\s+= "tag-gone"`).MatchString(src) { + t.Errorf("%s does not name the part's enum values:\n%s", seat, src) + } + } + if !strings.Contains(string(files["go/seats/seats.go"]), "OneStore") { + t.Error("the seat set does not name the seat at the seat list's grain") + } + + // The npm side (ADR 0625 §1): the indexer counts an npm package a lock resolves, so each seat defined whole is an + // npm package of its own (`-` at ts/seats/), holding its parts' types whole and + // importing nothing outside itself; the seat set's package (ts/) holds the seat list alone. + pkgs := map[string]string{} + for p, b := range files { + if strings.HasPrefix(p, "ts/") && strings.HasSuffix(p, "package.json") { + var pj struct { + Name string `json:"name"` + } + if err := json.Unmarshal(b, &pj); err != nil { + t.Fatalf("%s: %v", p, err) + } + pkgs[strings.TrimSuffix(p, "package.json")] = pj.Name + } + } + wantPkgs := map[string]string{ + "ts/": "@example/contracts", + "ts/seats/aregistry/": "@example/contracts-a-registry", + "ts/seats/bregistry/": "@example/contracts-b-registry", + } + if len(pkgs) != len(wantPkgs) { + t.Errorf("npm packages %v, want %v", pkgs, wantPkgs) + } + for dir, name := range wantPkgs { + if pkgs[dir] != name { + t.Errorf("%spackage.json names %q, want %q", dir, pkgs[dir], name) + } + } + for p, b := range files { + if !strings.HasPrefix(p, "ts/") || !strings.HasSuffix(p, ".ts") { + continue + } + dir := "" + for d := range pkgs { + if strings.HasPrefix(p, d) && len(d) > len(dir) { + dir = d + } + } + if dir == "" || !strings.HasPrefix(p, dir+"src/") { + t.Errorf("%s is in no npm package's src", p) + } + for _, line := range strings.Split(string(b), "\n") { + i := strings.Index(line, ` from "`) + if l := strings.TrimSpace(line); i >= 0 && (strings.HasPrefix(l, "import") || strings.HasPrefix(l, "export")) { + from := strings.Trim(line[i+len(` from `):], `";`) + if !strings.HasPrefix(from, "./") { + t.Errorf("%s imports %s: a generated npm package imports only its own files", p, from) + } + } + } + } + for _, seat := range []string{"aregistry", "bregistry"} { + if !strings.Contains(string(files["ts/seats/"+seat+"/src/index.ts"]), "export type PublishedEvent = registry.PublishedEvent") || + !strings.Contains(string(files["ts/seats/"+seat+"/src/parts/registry.ts"]), "export type PublishedEvent") { + t.Errorf("the npm package of %s does not hold the part's types whole", seat) + } + } + if ix := string(files["ts/src/index.ts"]); strings.Contains(ix, "Registry") || !strings.Contains(ix, "seats") { + t.Errorf("the seat set's npm package exports a seat's contract or not the seat list:\n%s", ix) + } +} diff --git a/modules/mesh-build-go/buildstep/seatgen/load.go b/modules/mesh-build-go/buildstep/seatgen/load.go new file mode 100644 index 00000000..43eb6548 --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/load.go @@ -0,0 +1,210 @@ +// Package seatgen writes a contracts module's Go and TypeScript from its seat definitions (design 26 *The +// generator*): the seat list every reader compiles, and for each seat defined whole its names, subjects and message +// types, with a test of its fixtures. The same input gives the same bytes. It is the build modules' logic (the Go +// build module's for Go, the Node build module's for TypeScript); this copy wrote mesh-contracts' first files and is +// handed to them as it stands. +package seatgen + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// Module is a contracts module read from its directory. +type Module struct { + // GoModule is the Go module path of the generated Go tree (`/go`). + GoModule string + // TSPackage is the npm package's name. + TSPackage string + // Order is the seat list's reading order (seats/index.json). + Order []string + // Seats are every definition, resolved, by name. + Seats map[string]seatdef.Definition + // Raw are every definition as written, its parts not taken in. + Raw map[string]seatdef.Definition + // Parts are every part, by name. + Parts map[string]seatdef.Part + // Fixtures are each seat's fixtures, by seat, in file order. + Fixtures map[string][]seatdef.Fixture +} + +// Load reads a contracts module: `seats/index.json`, `seats//seat.json`, `seats//fixtures/*.json` +// and `parts/*.json`. A seat with a directory and not in the index, or in the index with no directory, is refused. +func Load(dir, goModule, tsPackage string) (*Module, error) { + m := &Module{GoModule: goModule, TSPackage: tsPackage, Seats: map[string]seatdef.Definition{}, + Raw: map[string]seatdef.Definition{}, Parts: map[string]seatdef.Part{}, Fixtures: map[string][]seatdef.Fixture{}} + raw, err := os.ReadFile(filepath.Join(dir, "seats", "index.json")) + if err != nil { + return nil, err + } + ix, err := seatdef.ParseIndex(raw) + if err != nil { + return nil, fmt.Errorf("seats/index.json: %w", err) + } + m.Order = ix.Seats + parts, _ := filepath.Glob(filepath.Join(dir, "parts", "*.json")) + sort.Strings(parts) + for _, p := range parts { + raw, err := os.ReadFile(p) + if err != nil { + return nil, err + } + part, err := seatdef.ParsePart(raw) + if err != nil { + return nil, fmt.Errorf("%s: %w", rel(dir, p), err) + } + if want := strings.TrimSuffix(filepath.Base(p), ".json"); part.Part != want { + return nil, fmt.Errorf("%s: names part %q", rel(dir, p), part.Part) + } + m.Parts[part.Part] = part + } + dirs, _ := filepath.Glob(filepath.Join(dir, "seats", "*", "seat.json")) + inIndex := map[string]bool{} + for _, s := range m.Order { + inIndex[s] = true + } + for _, p := range dirs { + name := filepath.Base(filepath.Dir(p)) + if !inIndex[name] { + return nil, fmt.Errorf("%s: seat %q is not in seats/index.json", rel(dir, p), name) + } + raw, err := os.ReadFile(p) + if err != nil { + return nil, err + } + d, err := seatdef.Parse(raw) + if err != nil { + return nil, fmt.Errorf("%s: %w", rel(dir, p), err) + } + if d.Seat != name { + return nil, fmt.Errorf("%s: names seat %q", rel(dir, p), d.Seat) + } + m.Raw[name] = d + resolved, err := seatdef.Resolve(d, m.Parts) + if err != nil { + return nil, fmt.Errorf("%s: %w", rel(dir, p), err) + } + m.Seats[name] = resolved + fx, _ := filepath.Glob(filepath.Join(filepath.Dir(p), "fixtures", "*.json")) + sort.Strings(fx) + for _, f := range fx { + raw, err := os.ReadFile(f) + if err != nil { + return nil, err + } + cases, err := seatdef.ParseFixtures(raw) + if err != nil { + return nil, fmt.Errorf("%s: %w", rel(dir, f), err) + } + m.Fixtures[name] = append(m.Fixtures[name], cases...) + } + } + for _, s := range m.Order { + if _, ok := m.Seats[s]; !ok { + return nil, fmt.Errorf("seats/index.json names %q, which has no seats/%s/seat.json", s, s) + } + } + return m, nil +} + +func rel(dir, p string) string { + if r, err := filepath.Rel(dir, p); err == nil { + return r + } + return p +} + +// Generate is every file the module's generated trees hold, by path from the module's root. +func Generate(m *Module) (map[string][]byte, error) { + out := map[string][]byte{} + if err := goFiles(m, out); err != nil { + return nil, err + } + if err := tsFiles(m, out); err != nil { + return nil, err + } + return out, nil +} + +// GeneratedRoots are the directories every file of which the generator writes: a file there it did not write is +// out of step (design 26: the compare step). +var GeneratedRoots = []string{"go", "ts"} + +// Compare is every path out of step between what the generator writes and the tree in dir: a file that differs, one +// missing, or one under a generated root the generator did not write, each named once, sorted. +func Compare(dir string, files map[string][]byte) ([]string, error) { + var out []string + for p, want := range files { + got, err := os.ReadFile(filepath.Join(dir, p)) + switch { + case os.IsNotExist(err): + out = append(out, p+": missing") + case err != nil: + return nil, err + case string(got) != string(want): + out = append(out, p+": differs") + } + } + for _, root := range GeneratedRoots { + err := filepath.WalkDir(filepath.Join(dir, root), func(p string, e os.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return filepath.SkipDir + } + return err + } + if e.IsDir() { + if e.Name() == "node_modules" || e.Name() == "dist" || rel(dir, p) == ClientRoot { + return filepath.SkipDir + } + return nil + } + r := rel(dir, p) + if _, ok := files[r]; !ok { + out = append(out, r+": not written by the generator") + } + return nil + }) + if err != nil { + return nil, err + } + } + sort.Strings(out) + return out, nil +} + +// pkgName is a seat's or part's Go package: its name without hyphens. +func pkgName(name string) string { return strings.ReplaceAll(name, "-", "") } + +// ident is a name in Go's exported form: mesh-broker → MeshBroker, publish-part → PublishPart, preview.written → +// PreviewWritten, local-name → LocalName, id → ID. +func ident(name string) string { + var b strings.Builder + for _, w := range strings.FieldsFunc(name, func(r rune) bool { return r == '-' || r == '_' || r == '.' || r == ' ' }) { + switch w { + case "id": + b.WriteString("ID") + case "url": + b.WriteString("URL") + case "mib": + b.WriteString("MiB") + default: + b.WriteString(strings.ToUpper(w[:1]) + w[1:]) + } + } + return b.String() +} + +// tsName is a name in TypeScript's camel form for a constant: publish-part → publishPart. +func tsName(name string) string { + id := ident(name) + if strings.HasPrefix(id, "ID") { + return "id" + id[2:] + } + return strings.ToLower(id[:1]) + id[1:] +} diff --git a/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/parts/registry.json b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/parts/registry.json new file mode 100644 index 00000000..01a76b78 --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/parts/registry.json @@ -0,0 +1,3 @@ +{"format":1,"part":"registry","verbs":[{"name":"publish","input":{"$ref":"#/defs/Ref"},"answer":{"type":"object"}}], + "events":[{"name":"published","body":{"$ref":"#/defs/Ref"}}], + "defs":{"Ref":{"type":"object","properties":{"digest":{"type":"string"},"reason":{"type":"string","enum":["tag-gone","retention"]}},"required":["digest"]}}} diff --git a/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/a-registry/seat.json b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/a-registry/seat.json new file mode 100644 index 00000000..73235370 --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/a-registry/seat.json @@ -0,0 +1,3 @@ +{"format":1,"seat":"a-registry","scope":"mesh","grain":"whole","decision":"ADR 1","includes":["registry"], + "verbs":[{"name":"hold","input":{"type":"object"},"answer":{"type":"object"}}], + "family":{"name":"a-registry.events","max-age-days":7}} diff --git a/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/b-registry/seat.json b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/b-registry/seat.json new file mode 100644 index 00000000..cff301db --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/b-registry/seat.json @@ -0,0 +1,3 @@ +{"format":1,"seat":"b-registry","scope":"mesh","grain":"whole","decision":"ADR 1","includes":["registry"], + "verbs":[{"name":"hold","input":{"type":"object"},"answer":{"type":"object"}}], + "family":{"name":"b-registry.events","max-age-days":7}} diff --git a/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/index.json b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/index.json new file mode 100644 index 00000000..5218cd34 --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/index.json @@ -0,0 +1 @@ +{"format":1,"seats":["one-store","a-registry","b-registry"]} diff --git a/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/one-store/seat.json b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/one-store/seat.json new file mode 100644 index 00000000..3e0501a6 --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/one-store/seat.json @@ -0,0 +1 @@ +{"format":1,"seat":"one-store","scope":"mesh","grain":"seat-list","decision":"ADR 1","verbs":[{"name":"query","input":{"type":"object"}}]} diff --git a/modules/mesh-build-go/buildstep/seatgen/tsgen.go b/modules/mesh-build-go/buildstep/seatgen/tsgen.go new file mode 100644 index 00000000..c8fea5cb --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/tsgen.go @@ -0,0 +1,313 @@ +package seatgen + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "strings" + + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +const tsHeader = "// Code generated by seatgen from the seat definitions; DO NOT EDIT.\n\n" + +// TSSeatDir is where a seat defined whole has its npm package (ADR 0625 §1): the indexer counts the npm package a +// lock resolves, so each such seat is a package of its own and a module is rebuilt only for the seats it uses. +func TSSeatDir(seat string) string { return "ts/seats/" + pkgName(seat) } + +// TSSeatPackage is the npm package name of a seat defined whole, beside the seat set's package root. +func TSSeatPackage(root, seat string) string { return root + "-" + seat } + +const tsConfig = `{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "declaration": true, + "outDir": "dist", + "rootDir": "src", + "strict": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*.ts"] +} +` + +// tsPackage writes one npm package's package.json and tsconfig.json at dir. +func tsPackage(out map[string][]byte, dir, name, description string) { + pkg := map[string]any{ + "name": name, "version": "0.1.0", "type": "module", + "description": description, + "exports": map[string]any{".": "./dist/index.js"}, + "files": []any{"dist"}, + "scripts": map[string]any{"build": "tsc", "prepare": "tsc"}, + "devDependencies": map[string]any{"typescript": "^5.6.0"}, + } + raw, _ := json.MarshalIndent(pkg, "", " ") + out[dir+"/package.json"] = append(raw, '\n') + out[dir+"/tsconfig.json"] = []byte(tsConfig) +} + +// tsFiles writes the npm side: the seat set's package at ts/ (the seat list alone, ADR 0625 §2) and one package per +// seat defined whole at TSSeatDir (ADR 0625 §1), holding the parts it includes whole and importing nothing outside it. +func tsFiles(m *Module, out map[string][]byte) error { + tsPackage(out, "ts", m.TSPackage, "The mesh's seat set, generated from every seat's definition.") + out["ts/src/index.ts"] = []byte(tsHeader + "export * as seats from \"./seats.js\";\n") + out["ts/src/seats.ts"] = tsSeatList(m) + partTypes := map[string]*typer{} + partFiles := map[string][]byte{} + for _, name := range sortedKeys(m.Parts) { + p := m.Parts[name] + d := seatdef.Definition{Grain: seatdef.Whole, Verbs: p.Verbs, Events: p.Events, Defs: p.Defs} + tp, err := declareAll(d, nil) + if err != nil { + return err + } + partTypes[name] = tp + var b bytes.Buffer + b.WriteString(tsHeader) + fmt.Fprintf(&b, "// The part %q several seats include whole: %s\n\n", p.Part, oneLine(p.Description)) + tsNames(&b, "verbs", seatdef.VerbNames(p.Verbs)) + tsNames(&b, "events", seatdef.MessageNames(p.Events)) + tsTypes(&b, tp) + partFiles[name] = b.Bytes() + } + for _, name := range m.Order { + d := m.Seats[name] + if d.Grain != seatdef.Whole { + continue + } + dir := TSSeatDir(name) + tsPackage(out, dir, TSSeatPackage(m.TSPackage, name), fmt.Sprintf("The contract of the seat %s, generated from its definition.", name)) + raw := m.Raw[name] + fromPart := map[string]string{} + for _, inc := range raw.Includes { + for _, t := range partTypes[inc].types { + fromPart[t.name] = inc + } + out[dir+"/src/parts/"+inc+".ts"] = partFiles[inc] + } + tp, err := declareAll(d, fromPart) + if err != nil { + return err + } + var b bytes.Buffer + b.WriteString(tsHeader) + fmt.Fprintf(&b, "// The contract of the seat %q, defined whole (seats/%s/seat.json): %s\n\n", d.Seat, d.Seat, oneLine(d.Description)) + for _, inc := range raw.Includes { + fmt.Fprintf(&b, "import type * as %s from \"./parts/%s.js\";\n\n", tsName(inc), inc) + } + fmt.Fprintf(&b, "export const seat = %q;\n\n", d.Seat) + tsNames(&b, "verbs", seatdef.VerbNames(d.Verbs)) + tsNames(&b, "accepts", seatdef.MessageNames(d.Accepts)) + tsNames(&b, "events", seatdef.MessageNames(d.Events)) + if f := d.Family; f != nil { + fmt.Fprintf(&b, "export const family = { name: %q, maxAgeDays: %d", f.Name, f.MaxAgeDays) + b.WriteString(" } as const;\n\n") + } + for _, e := range d.Events { + subj := seatdef.EventSubject(d.Seat, e.Name) + if e.ByCaller { + fmt.Fprintf(&b, "export function %sSubject(caller: string): string {\n return %q + caller;\n}\n\n", tsName(e.Name), subj+".") + } else { + fmt.Fprintf(&b, "export const %sSubject = %q;\n\n", tsName(e.Name), subj) + } + } + for _, a := range d.Accepts { + subj := seatdef.AcceptSubject(d.Seat, a.Name) + if a.ByCaller { + fmt.Fprintf(&b, "export function %sSubject(caller: string): string {\n return %q + caller;\n}\n\n", tsName(a.Name), subj+".") + } else { + fmt.Fprintf(&b, "export const %sSubject = %q;\n\n", tsName(a.Name), subj) + } + } + if len(d.HolderEvents) > 0 { + tsNames(&b, "holderEvents", holderNames(d.HolderEvents)) + for _, e := range d.HolderEvents { + before, after, _ := strings.Cut(e.Subject, "") + fmt.Fprintf(&b, "export function %sHolderSubject(holder: string): string {\n return %q + holder + %q;\n}\n\n", + tsName(e.Name), before, after) + } + } + for _, n := range sortedKeys(fromPart) { + fmt.Fprintf(&b, "export type %s = %s.%s;\n", n, tsName(fromPart[n]), n) + } + if len(fromPart) > 0 { + b.WriteString("\n") + } + tsTypes(&b, tp) + out[dir+"/src/index.ts"] = b.Bytes() + } + return nil +} + +func holderNames(es []seatdef.HolderEvent) []string { + out := make([]string, len(es)) + for i, e := range es { + out[i] = e.Name + } + return out +} + +func tsNames(b *bytes.Buffer, what string, names []string) { + if len(names) == 0 { + return + } + fmt.Fprintf(b, "export const %s = {\n", what) + for _, n := range names { + fmt.Fprintf(b, " %s: %q,\n", tsName(n), n) + } + b.WriteString("} as const;\n\n") +} + +func tsTypes(b *bytes.Buffer, tp *typer) { + for _, t := range tp.types { + if d := strings.TrimSpace(t.desc); d != "" { + fmt.Fprintf(b, "/** %s */\n", oneLine(d)) + } + if t.t.kind != "struct" { + fmt.Fprintf(b, "export type %s = %s;\n\n", t.name, tsType(t.t)) + continue + } + fmt.Fprintf(b, "export interface %s {\n", t.name) + for _, f := range t.t.fields { + if f.desc != "" { + fmt.Fprintf(b, " /** %s */\n", oneLine(f.desc)) + } + opt := "?" + if f.required { + opt = "" + } + fmt.Fprintf(b, " %s%s: %s;\n", strconv.Quote(f.json), opt, tsType(f.t)) + } + b.WriteString("}\n\n") + } +} + +func tsType(t *typ) string { + switch t.kind { + case "string": + if len(t.enum) > 0 { + parts := make([]string, len(t.enum)) + for i, e := range t.enum { + parts[i] = strconv.Quote(e) + } + return strings.Join(parts, " | ") + } + return "string" + case "integer", "number": + return "number" + case "boolean": + return "boolean" + case "object": + return "Record" + case "map": + return "Record" + case "slice": + el := tsType(t.elem) + if strings.Contains(el, "|") { + el = "(" + el + ")" + } + return el + "[]" + case "ref": + return t.ref + } + return "unknown" +} + +// tsSeatList is ts/src/seats.ts: the seat list as data, and every seat's name. +func tsSeatList(m *Module) []byte { + var b bytes.Buffer + b.WriteString(tsHeader) + b.WriteString(`// The mesh's seat list, generated from every seat's definition (design 26). + +export interface Verb { + name: string; + description: string; + input?: Record; + output?: Record; + optional?: boolean; + replaces?: string[]; +} + +export interface Seat { + name: string; + scope: "mesh" | "node"; + grain: "whole" | "seat-list"; + decision: string; + delivers?: string; + replicated?: boolean; + accepts?: string[]; + emits?: string[]; + pulledByKind?: boolean; + yields?: boolean; + heldByTheEngine?: boolean; + serves?: Verb[]; +} + +`) + b.WriteString("export const names = {\n") + for _, n := range m.Order { + fmt.Fprintf(&b, " %s: %q,\n", tsName(n), n) + } + b.WriteString("} as const;\n\nexport const list: readonly Seat[] = ") + var list []map[string]any + for _, n := range m.Order { + d := m.Seats[n] + if d.HolderDeclares { + continue + } + e := map[string]any{"name": d.Seat, "scope": d.Scope, "grain": d.Grain, "decision": d.Decision} + if d.Delivers != "" { + e["delivers"] = d.Delivers + } + if d.Replicated { + e["replicated"] = true + } + if len(d.Accepts) > 0 { + e["accepts"] = seatdef.MessageNames(d.Accepts) + } + if len(d.Events) > 0 { + e["emits"] = seatdef.MessageNames(d.Events) + } + if d.PulledByKind { + e["pulledByKind"] = true + } + if d.Yields { + e["yields"] = true + } + if d.HeldByTheEngine { + e["heldByTheEngine"] = true + } + var verbs []map[string]any + for _, v := range d.Verbs { + vm := map[string]any{"name": v.Name, "description": v.Description} + if v.Input != nil { + vm["input"] = resolveRefs(v.Input, d.Defs) + } + if v.Answer != nil { + vm["output"] = resolveRefs(v.Answer, d.Defs) + } + if v.Optional { + vm["optional"] = true + } + if len(v.Replaces) > 0 { + vm["replaces"] = v.Replaces + } + verbs = append(verbs, vm) + } + if len(verbs) > 0 { + e["serves"] = verbs + } + list = append(list, e) + } + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + _ = enc.Encode(list) + b.Truncate(b.Len() - 1) + b.WriteString(";\n") + return b.Bytes() +} diff --git a/modules/mesh-build-go/buildstep/seatgen/types.go b/modules/mesh-build-go/buildstep/seatgen/types.go new file mode 100644 index 00000000..9fa1507c --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/types.go @@ -0,0 +1,185 @@ +package seatgen + +import ( + "fmt" + "sort" + "strings" +) + +// A typ is one JSON Schema read as a type both languages write: the subset the definitions use (objects with +// properties, maps, arrays, strings, integers, numbers, booleans, references to a definition's own types). Anything +// else is `any`, so a schema the generator does not understand still decodes. +type typ struct { + kind string // string, integer, number, boolean, any, map, slice, struct, ref, object (free) + elem *typ // a map's value or a slice's item + ref string // a referenced type's name + fields []field + enum []string +} + +type field struct { + json string + required bool + desc string + t *typ +} + +// named is one type a seat's package declares: a struct, or an alias of another. +type named struct { + name string + desc string + t *typ +} + +// typer reads schemas into named types, naming each inline object by where it sits. +type typer struct { + types []named + taken map[string]bool +} + +func newTyper() *typer { return &typer{taken: map[string]bool{}} } + +// declare reads schema s as the type called name and declares it. +func (tp *typer) declare(name, desc string, s map[string]any) error { + if tp.taken[name] { + return fmt.Errorf("two types would be called %s", name) + } + tp.taken[name] = true + t, err := tp.read(name, s, true) + if err != nil { + return fmt.Errorf("%s: %w", name, err) + } + if desc == "" { + desc, _ = s["description"].(string) + } + tp.types = append(tp.types, named{name: name, desc: desc, t: t}) + return nil +} + +func (tp *typer) read(where string, s map[string]any, top bool) (*typ, error) { + if s == nil { + return &typ{kind: "any"}, nil + } + if r, ok := s["$ref"].(string); ok { + n, found := strings.CutPrefix(r, "#/defs/") + if !found || n == "" { + return nil, fmt.Errorf("$ref %q is not #/defs/", r) + } + return &typ{kind: "ref", ref: n}, nil + } + t, _ := s["type"].(string) + switch t { + case "string": + out := &typ{kind: "string"} + if e, ok := s["enum"].([]any); ok { + for _, v := range e { + if str, ok := v.(string); ok { + out.enum = append(out.enum, str) + } + } + } + return out, nil + case "integer", "number", "boolean": + return &typ{kind: t}, nil + case "array": + items, _ := s["items"].(map[string]any) + el, err := tp.inline(where+"Item", items) + if err != nil { + return nil, err + } + return &typ{kind: "slice", elem: el}, nil + case "object": + props, hasProps := s["properties"].(map[string]any) + if !hasProps { + if ap, ok := s["additionalProperties"].(map[string]any); ok { + el, err := tp.inline(where+"Value", ap) + if err != nil { + return nil, err + } + return &typ{kind: "map", elem: el}, nil + } + return &typ{kind: "object"}, nil + } + required := map[string]bool{} + switch r := s["required"].(type) { + case []any: + for _, v := range r { + if str, ok := v.(string); ok { + required[str] = true + } + } + case []string: + for _, v := range r { + required[v] = true + } + } + names := make([]string, 0, len(props)) + for k := range props { + names = append(names, k) + } + sort.Strings(names) + out := &typ{kind: "struct"} + for _, k := range names { + ps, _ := props[k].(map[string]any) + ft, err := tp.inline(where+ident(k), ps) + if err != nil { + return nil, err + } + desc, _ := ps["description"].(string) + out.fields = append(out.fields, field{json: k, required: required[k], desc: desc, t: ft}) + } + return out, nil + } + return &typ{kind: "any"}, nil +} + +// inline is a nested schema's type: an object with properties becomes a named type of its own, called where. +func (tp *typer) inline(where string, s map[string]any) (*typ, error) { + if s != nil { + if _, props := s["properties"].(map[string]any); props && s["type"] == "object" { + desc, _ := s["description"].(string) + if err := tp.declare(where, desc, s); err != nil { + return nil, err + } + return &typ{kind: "ref", ref: where}, nil + } + } + return tp.read(where, s, false) +} + +// resolveRefs is a schema with every reference to a definition's own type replaced by that type, for the seat list +// a reader shows: a caller reading a verb's input there never meets a reference it cannot follow. +func resolveRefs(s map[string]any, defs map[string]map[string]any) map[string]any { + if s == nil { + return nil + } + v := resolveValue(s, defs, 0) + m, _ := v.(map[string]any) + return m +} + +func resolveValue(v any, defs map[string]map[string]any, depth int) any { + if depth > 32 { + return v + } + switch x := v.(type) { + case map[string]any: + if r, ok := x["$ref"].(string); ok { + if d, ok := defs[strings.TrimPrefix(r, "#/defs/")]; ok { + return resolveValue(d, defs, depth+1) + } + } + out := make(map[string]any, len(x)) + for k, e := range x { + out[k] = resolveValue(e, defs, depth+1) + } + return out + case []any: + out := make([]any, len(x)) + for i, e := range x { + out[i] = resolveValue(e, defs, depth+1) + } + return out + } + return v +} diff --git a/modules/mesh-build-go/buildstep/step.go b/modules/mesh-build-go/buildstep/step.go index a265184e..ede19129 100644 --- a/modules/mesh-build-go/buildstep/step.go +++ b/modules/mesh-build-go/buildstep/step.go @@ -83,6 +83,12 @@ func finish(put Put, key string, step delivery.Step, result any, err error, logf step.State, step.Reason = delivery.StepRefused, err.Error() default: step.State, step.Reason = delivery.StepFailed, err.Error() + // A failed step may answer what it found on the way (a build's out-of-step files): kept on its end. + if result != nil { + if raw, merr := json.Marshal(result); merr == nil { + step.Result = raw + } + } } werr := writeEnd(put, key, step) if werr == nil { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/archive.go b/modules/mesh-build-go/cmd/mesh-build-go/archive.go index a25d6477..5c138815 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/archive.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/archive.go @@ -77,7 +77,7 @@ func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir if !within(realDir, realRoot) { return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module: it resolves to %s, and the module is %s", art.Name, art.From, realRoot, realDir) } - return m.packInto(ctx, workspace, module, name, art.Name, root) + return m.packInto(ctx, workspace, module, name, art.Name, "archive", root) } // pinnedArchive fetches the file an artifact is pinned to, checks it against the pin, lays it out as the artifact says @@ -111,7 +111,7 @@ func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name stri } else if err := unpackUpstream(file, laid, art.Within); err != nil { return nil, fmt.Errorf("%s: %s (%s): %w", art.Name, art.URL, art.Version, err) } - return m.packInto(ctx, workspace, module, name, art.Name, laid) + return m.packInto(ctx, workspace, module, name, art.Name, "archive", laid) } // pinnedFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes are still the diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index 5579cc6c..25dcb8eb 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "path/filepath" "time" "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" @@ -38,6 +39,7 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps { return buildstep.AskSteps{ Fetch: byName["build_go_fetch"], Build: byName["build_go_build"], Digest: byName["build_go_digest"], Test: byName["build_go_test"], Imports: byName[ImportsTool], + Compare: buildstep.CompareStep(buildstep.GeneratedRootGo, func(ws string) string { return filepath.Join(m.Work, ws, "tree") }), Toolchain: func(ctx context.Context) (delivery.ToolchainUsed, error) { installed, err := m.Probe(ctx) if err != nil { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build.go b/modules/mesh-build-go/cmd/mesh-build-go/build.go index acc4e7df..11ba1328 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/build.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/build.go @@ -34,23 +34,27 @@ type BuildResult struct { Binary string `json:"binary"` Digest string `json:"digest"` Size int `json:"size"` - // Reference is where the archive is in the artifact store (novox/hq ADR 0498 §1): - // artifact-store:////blobs/. + // Reference is where the archive is, in the form the seat it was published through answers (novox/hq ADR 0605 + // §9): oci-artifact-registry//@. The builder writes the older form beside it while a + // counted node-engine reads only that (ADR 0624 §1, §3). Reference string `json:"reference"` } func (m *Module) buildTool() StepTool { return StepTool{ Name: "build_go_build", - Description: "Build one Go bundle or archive artifact of a module of a fetched workspace with the build agent's recipe, by the " + - "installed Go and no network, and pack it. Answers a step id at once; the step ends on its key with the " + - "artifact's digest, size and the archive's path.", + Description: "Build one Go bundle, archive or Go package artifact of a module of a fetched workspace with the build agent's " + + "recipe, by the installed Go and no network, and pack it (a Go package as Go's module zip); a bundle or archive is " + + "published on oci-artifact-registry, a Go package given a version on go-package-registry under its module path. " + + "Answers a step id at once; the step ends on its key with the artifact's digest, size and reference.", Input: map[string]any{ "type": "object", "properties": map[string]any{ "workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"}, "module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"}, "artifact": map[string]any{"type": "string", "description": "the artifact's name in the module's manifest"}, + "version": map[string]any{"type": "string", "description": "the version the builder gave a package at the trunk's claim; absent for a candidate, whose package is not published"}, + "seat": map[string]any{"type": "string", "description": "the registry seat the artifact is published through, as the build ask names it"}, "publish": map[string]any{"type": "boolean", "description": "true for a merged trunk commit alone: the build writes the shared Go caches; without it (a candidate) they are read beneath a throwaway layer"}, }, "required": []string{"workspace", "module", "artifact"}, @@ -109,6 +113,11 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { if art == nil { return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", ")) } + if art.Kind == catalogue.ArtifactPackage && art.Language == "go" { + version, _ := args["version"].(string) + seat, _ := args["seat"].(string) + return m.buildGoPackage(ctx, workspace, dir, *art, version, seat) + } if art.Kind == "archive" { return m.buildArchive(ctx, workspace, module, manifest.Module, dir, *art) } @@ -149,7 +158,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { if err := m.insideModule(workspace, dir, out); err != nil { return nil, fmt.Errorf("the output directory of %s: %w", art.Name, err) } - res, err := m.packInto(ctx, workspace, module, manifest.Module, art.Name, out) + res, err := m.packInto(ctx, workspace, module, manifest.Module, art.Name, "bundle", out) if err != nil { return nil, err } @@ -157,20 +166,20 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { return res, nil } -// packInto packs a directory with mesh-sdk's Pack, as the build agent packs every artifact, and pushes the archive to -// the artifact store from this process, after the build step ended (novox/hq ADR 0498 §1 §2): a candidate's as a trunk -// build's, under / by its digest. The step answers the reference; a push that fails fails the step -// `not stored:`. The packed bytes never stay on the node: they are held in this process until the push answered (the -// lead's decision L9). -func (m *Module) packInto(ctx context.Context, workspace, module, name, artifact, root string) (BuildResult, error) { +// packInto packs a directory with mesh-sdk's Pack, as the build agent packs every artifact, and publishes the archive +// on the seat oci-artifact-registry from this process, after the build step ended (design 51 *Publishing*; publish.go): +// a candidate's as a trunk build's, under / by its digest, of kind (bundle or archive). The step +// answers the reference; a publish the seat refuses fails the step `not stored:`. The packed bytes never stay on the +// node: they are held in this process until the publish answered (the lead's decision L9). +func (m *Module) packInto(ctx context.Context, workspace, module, name, artifact, kind, root string) (BuildResult, error) { archive, digest, err := buildstep.Pack(root) if err != nil { return BuildResult{}, fmt.Errorf("packing %s: %w", artifact, err) } - if err := m.push(ctx, name+"/"+artifact, digest, archive); err != nil { + if err := m.publishArchive(ctx, name, artifact, kind, digest, archive); err != nil { return BuildResult{}, fmt.Errorf("%s%s: %v", delivery.NotStored, artifact, err) } - return BuildResult{Artifact: artifact, Digest: digest, Size: len(archive), Reference: buildstep.BlobReference(name, artifact, digest)}, nil + return BuildResult{Artifact: artifact, Digest: digest, Size: len(archive), Reference: catalogue.SeatReference(ArtifactSeat, name, artifact, digest)}, nil } // buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the installed go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 8644edd9..6679c5e0 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -8,6 +8,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "os" "path/filepath" @@ -310,15 +311,58 @@ func (m *Module) goEnv() []string { if m.GoModCache != "" { modcache = m.GoModCache } - return append([]string{ + env := []string{ "GOTOOLCHAIN=local", "GOFLAGS=-mod=mod", "GOMODCACHE=" + modcache, "GOCACHE=" + filepath.Join(m.Cache, "gobuild"), - "GOPRIVATE=git.novox.be", - "GIT_TERMINAL_PROMPT=0", - "GIT_SSH_COMMAND=ssh -o BatchMode=yes", - }, m.threadEnv()...) + } + if proxy := m.goSeat(); proxy != "" { + // The Go seat first, the public proxy second, the forge's git last (CAT-BG3, E11: direct goes in the first + // change after mesh-sdk's and mesh-contracts' first numbered versions are in the Go seat); no GOPRIVATE, and + // the checksum database off for the forge's paths, which go.sum and the builder's h1: digest guard instead. + env = append(env, "GOPROXY="+proxy+","+PublicGoProxy+",direct", "GONOSUMDB="+ForgeGoPaths, "GOPRIVATE=") + } else { + env = append(env, "GOPRIVATE="+ForgeGoPaths) + } + return append(append(env, "GIT_TERMINAL_PROMPT=0", "GIT_SSH_COMMAND=ssh -o BatchMode=yes"), m.threadEnv()...) +} + +// PublicGoProxy is Go's public module proxy; ForgeGoPaths the module paths the forge serves. +const ( + PublicGoProxy = "https://proxy.golang.org" + ForgeGoPaths = "git.novox.be" +) + +// goSeat is the Go seat's proxy URL from its binding (`at`, and `serves` scheme, port and go-path), empty when the +// module holds no binding of it or the binding names no address. +func (m *Module) goSeat() string { + if m.GoSeatFile == "" { + return "" + } + raw, err := os.ReadFile(m.GoSeatFile) + if err != nil { + return "" + } + var b struct { + At string `json:"at"` + Serves struct { + Scheme string `json:"scheme"` + Port int `json:"port"` + GoPath string `json:"go-path"` + } `json:"serves"` + } + if json.Unmarshal(raw, &b) != nil || b.At == "" || b.Serves.GoPath == "" { + return "" + } + scheme := b.Serves.Scheme + if scheme == "" { + scheme = "http" + } + if b.Serves.Port == 0 { + return fmt.Sprintf("%s://%s%s", scheme, b.At, b.Serves.GoPath) + } + return fmt.Sprintf("%s://%s:%d%s", scheme, b.At, b.Serves.Port, b.Serves.GoPath) } // goEnvFor is goEnv for a command run in dir: a directory that vendors (vendor/modules.txt) is read from its vendor diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go index e86e1559..84ac2c46 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -145,6 +145,20 @@ func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) { } } } + // CAT-BG3: with the Go seat bound, the fetch goes through it first, Go's public proxy second and the forge's git + // last, with no GOPRIVATE and the checksum database off for the forge's paths. + seat := filepath.Join(t.TempDir(), "go-package-registry.json") + if err := os.WriteFile(seat, []byte(`{"at":"forge.mesh","serves":{"scheme":"http","port":3000,"go-path":"/api/packages/novox/go"}}`), 0o600); err != nil { + t.Fatal(err) + } + m.GoSeatFile = seat + env := strings.Join(m.goEnv(), " ") + for _, want := range []string{"GOPROXY=http://forge.mesh:3000/api/packages/novox/go,https://proxy.golang.org,direct", "GONOSUMDB=git.novox.be", "GOPRIVATE= "} { + if !strings.Contains(env+" ", want) { + t.Errorf("with the Go seat bound the environment lacks %q: %s", want, env) + } + } + m.GoSeatFile = "" if downloads != 2 || verifies != 2 { t.Errorf("%d downloads and %d verifies, want 2 each (go.mod and sub/dir/go.mod):\n%s", downloads, verifies, lines) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go index 89ff15c8..f98fce5a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go @@ -10,6 +10,7 @@ import ( "encoding/json" "os" "path/filepath" + "slices" "strings" "testing" @@ -143,10 +144,10 @@ func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) { // template in the files/ archive (novox/hq ADR 0349 §2). func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) { m := readForgeManifest(t) - // artifact-store: the module pushes a done build's archive itself (ADR 0498 §2). // node-tools: the module pulls its asks through the tool runner's seat pull (1911). - if len(m.Requires) != 4 || m.Requires[0] != "forge-account" || m.Requires[1] != "go-toolchain" || m.Requires[2] != "artifact-store" || m.Requires[3] != "node-tools" { - t.Errorf("requires = %v, want [forge-account go-toolchain artifact-store node-tools]", m.Requires) + // go-package-registry: the Go seat's read side, the fetch's first source (design 51, CAT-BG3, E2). + if want := []any{"forge-account", "go-toolchain", "go-package-registry", "node-tools"}; !slices.Equal(m.Requires, want) { + t.Errorf("requires = %v, want %v", m.Requires, want) } keeps := false for _, s := range m.State { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/gopackage.go b/modules/mesh-build-go/cmd/mesh-build-go/gopackage.go new file mode 100644 index 00000000..6201fe29 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/gopackage.go @@ -0,0 +1,89 @@ +package main + +// The build of a Go package (CAT-BG2; novox/hq design 51 *Publishing*, ADR 0605 §3, §9): the directory the artifact +// names (its one path, else the module's own) packed as Go's module zip, every file under @/, +// as Go's module proxy protocol serves it; its digest is the zip's sha256 (no h1: sum is answered: nothing in the contract carries +// it on, the lead's ruling on A3n). A +// trunk build given a version by the builder publishes it through go-package-registry under its Go module path (the +// name the forge's Go seat takes, the lead's ruling 3 on A3k) and that version; a candidate's publishes nothing. The +// zip is made in this process, after nothing ran: packing a tree runs no code of it. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" + modzip "golang.org/x/mod/zip" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// GoPackageSeat is the registry seat a Go package is published on. +const GoPackageSeat = "go-package-registry" + +// candidateVersion is the version a candidate's zip is laid out under: no version was given, and nothing is published. +const candidateVersion = "v0.0.0" + +func (m *Module) buildGoPackage(ctx context.Context, workspace, dir string, art catalogue.Artifact, version, seat string) (BuildResult, error) { + root := dir + switch len(art.Paths) { + case 0: + case 1: + root = filepath.Join(dir, filepath.FromSlash(strings.TrimSuffix(art.Paths[0], "/"))) + default: + return BuildResult{}, fmt.Errorf("the Go package %s names %d paths; a Go package is one directory holding its go.mod", art.Name, len(art.Paths)) + } + if err := m.insideModule(workspace, dir, root); err != nil { + return BuildResult{}, fmt.Errorf("the directory of %s: %w", art.Name, err) + } + raw, err := os.ReadFile(filepath.Join(root, "go.mod")) + if err != nil { + return BuildResult{}, fmt.Errorf("the Go package %s has no go.mod in %s: %w", art.Name, art.Paths, err) + } + path := modfile.ModulePath(raw) + if path == "" { + return BuildResult{}, fmt.Errorf("the go.mod of the Go package %s names no module", art.Name) + } + if version != "" && seat != GoPackageSeat { + return BuildResult{}, fmt.Errorf("the Go package %s is given the version %s and the seat %q: a Go package is published through %s", art.Name, version, seat, GoPackageSeat) + } + zipVersion := candidateVersion + if version != "" { + zipVersion = "v" + strings.TrimPrefix(version, "v") + } + file, err := os.CreateTemp(filepath.Join(m.Work, workspace), "gopackage-*.zip") + if err != nil { + return BuildResult{}, err + } + defer os.Remove(file.Name()) // the packed file never stays on the node (the lead's decision L9) + defer file.Close() + if err := modzip.CreateFromDir(file, module.Version{Path: path, Version: zipVersion}, root); err != nil { + return BuildResult{}, fmt.Errorf("packing the Go package %s (%s@%s): %w", art.Name, path, zipVersion, err) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return BuildResult{}, err + } + body, err := io.ReadAll(file) + if err != nil { + return BuildResult{}, err + } + d := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(d[:]) + res := BuildResult{Artifact: art.Name, Digest: digest, Size: len(body)} + if version == "" { + return res, nil + } + if err := m.publishOn(ctx, seat, path, "package", version, digest, transferName(path, version, digest), body); err != nil { + return BuildResult{}, fmt.Errorf("%s%s: %v", delivery.NotStored, art.Name, err) + } + res.Reference = path + "@" + version + return res, nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/latepublish.go b/modules/mesh-build-go/cmd/mesh-build-go/latepublish.go new file mode 100644 index 00000000..66c6d712 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/latepublish.go @@ -0,0 +1,27 @@ +package main + +// The late publish (CAT-BG2): a numbered bundle or archive's held digest published under its version, by the build +// modules' one late publish (buildstep/latepublish.go). + +import ( + "context" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// NumberedPattern is the event type this module consumes for the late publish, as its manifest's `consumes` names it. +const NumberedPattern = buildstep.NumberedPattern + +// LatePublish publishes a numbered bundle or archive's held digest under its version on the artifact seat; a Go package +// publishes at its build, an image is mesh-build-image's. +func (m *Module) LatePublish(ctx context.Context, body []byte) (bool, error) { + call := m.Call + if call == nil { + call = stdio.Ask + } + return buildstep.LatePublish(ctx, call, body, ArtifactSeat, "bundle", "archive") +} + +// hearNumbered hears the builder's version.numbered for the late publish. +func (m *Module) hearNumbered(ctx context.Context) { buildstep.HearNumbered(ctx, m.LatePublish) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/levels.go b/modules/mesh-build-go/cmd/mesh-build-go/levels.go index beb540fe..a6266f88 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/levels.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/levels.go @@ -72,6 +72,9 @@ type TestResult struct { // JudgedBy is the program that ran the level (ADR 0535 §5): "installed", or "candidate " under the // runner workspace. JudgedBy string `json:"judged_by,omitempty"` + // OutOfStep are the generated files of a contracts module in the workspace out of step with what the generator writes + // (CAT-BG5, buildstep.OutOfStep): the unit level fails on any, Why naming each. Absent when none is. + OutOfStep []string `json:"out-of-step,omitempty"` // By is the build module and the SDK it was built with (sdkstep.By): the checker names it when a failure hides // its cause. By string `json:"by,omitempty"` diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 78b3ec25..68c88622 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -54,7 +54,7 @@ func main() { m.Home = u.HomeDir } m.Run = StepRunner(m.Home) - m.RegistryFile = m.Home + "/registry" + m.GoSeatFile = m.Home + "/go-package-registry.json" m.Settings = m.Home + "/settings.json" m.ReadKept = readOf(stdio.State(KeptState)).Get m.ReadManifest = askManifest @@ -82,6 +82,8 @@ func main() { if m.Node != "" { // The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again. go func() { _ = puller.Pull(ctx) }() + // The builder's version.numbered, for the late publish of a held digest (latepublish.go). + go m.hearNumbered(ctx) } if err := stdio.Serve("", append(m.Tools(steps), m.SeatTools(puller)...)); err != nil { logf("%v", err) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go index f8c5e158..c199de78 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go @@ -75,6 +75,17 @@ func TestManifestDeclaresTheAccountTheCapabilityAndThePackages(t *testing.T) { if want := []string{"bubblewrap", "git", "openssh", "postgresql"}; !slices.Equal(packages, want) { t.Errorf("packages = %v, want %v", packages, want) } + // buildstep, which mesh-build-node and mesh-build-image require, is a Go package of this module, numbered in + // series 0.1 (CAT-BG6; ADR 0606): its users accept a series, never a commit. + declared := false + for _, a := range m.Build.Artifacts { + if a.Name == "buildstep" { + declared = a.Kind == "package" && a.Language == "go" && slices.Equal(a.Paths, []string{"buildstep"}) && a.Series == "0.1" + } + } + if !declared { + t.Errorf("artifacts %+v declare no Go package buildstep in series 0.1", m.Build.Artifacts) + } } // The work and cache directories are outside the home (ADR 0466 §1), owned by the account. @@ -159,21 +170,32 @@ func TestTheModuleDeclaresBubblewrapAndTheCapabilityItNeeds(t *testing.T) { } } -// 1587-S1 (ADR 0498 §2): the module requires the artifact store and is told its address in a file of its home, which no -// run step is given (a run step has no network either, ADR 0448). -func TestTheModuleRequiresTheArtifactStoreAndIsToldItsAddress(t *testing.T) { +// 1587-S1, CAT-BG1 (design 51 *Publishing*, ADR 0614 §2): the module publishes a done archive through the seat +// oci-artifact-registry's verbs, which it invokes (the served-verb edge orders it after the holder), and no longer +// requires the artifact store or is told its address: nothing writes into the store but its holder. +func TestTheModuleInvokesTheArtifactSeatsVerbsAndWritesNoStore(t *testing.T) { m := readManifest(t) - if !slices.Contains(m.Requires, "artifact-store") { - t.Errorf("requires %v lack artifact-store", m.Requires) + if slices.Contains(m.Requires, "artifact-store") { + t.Errorf("requires %v still name artifact-store", m.Requires) } - r := resource(t, m, "registry") - // The template (files/registry) is read below; decoding leaves it unread in the resource. - if r["type"] != "file" || r["path"] != "${dir:state}/registry" || r["template"] != "files/registry" { - t.Errorf("the registry resource is %v", r) + for _, seat := range []string{ArtifactSeat, GoPackageSeat} { // a Go package's too (CAT-BG2) + for _, verb := range []string{"publish", "publish-part"} { + if !slices.Contains(m.Invokes, "seat:"+seat+"."+verb) { + t.Errorf("invokes %v lack the seat %s's %s", m.Invokes, seat, verb) + } + } } - raw, err := os.ReadFile(filepath.Join(moduleDir, "files", "registry")) - if err != nil || strings.TrimSpace(string(raw)) != "${seat:mesh-artifact-store:reach}" { - t.Errorf("files/registry is %q (%v), want the store seat's reach", raw, err) + // The late publish hears the builder's version.numbered (latepublish.go). + if !slices.Contains(m.Consumes, NumberedPattern) { + t.Errorf("consumes %v lack %s", m.Consumes, NumberedPattern) + } + for _, r := range m.Resources { + if r["id"] == "registry" { + t.Errorf("the store's address is still placed: %v", r) + } + } + if _, err := os.Stat(filepath.Join(moduleDir, "files", "registry")); err == nil { + t.Error("files/registry is still shipped") } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 38e2f843..b17384e6 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -67,10 +67,13 @@ type Module struct { // Inside is the integration level's own process, started shut off in bubblewrap by a test step (level.go): its // commands run as they are, never shut off a second time. Inside bool - // RegistryFile holds the artifact store's address (host:port), placed by the manifest; Push puts a blob there - // (buildstep.PushBlob when nil; a test's fake store). - RegistryFile string - Push func(ctx context.Context, registry, repository, digest string, body []byte) error + // GoSeatFile is the binding of the provision go-package-registry, the Go seat's read side (design 51 *Go + // packages*): bound, the mesh's Go modules are fetched through it first, Go's public proxy second, and the forge's + // git over the account's ssh last (Go's direct, kept for this milestone only); unbound, as before, directly. + GoSeatFile string + // Call asks a seat's verb over the bus: the publish of a done archive (publish.go). stdio.Ask when nil; a test's + // fake holder. + Call callFunc // Executable is this program's path, which the integration level starts again shut off; os.Executable when empty. Executable string // NodeToolchains is where the Node build module keeps its installed toolchains, bound read-only into the diff --git a/modules/mesh-build-go/cmd/mesh-build-go/publish.go b/modules/mesh-build-go/cmd/mesh-build-go/publish.go new file mode 100644 index 00000000..993a045f --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/publish.go @@ -0,0 +1,42 @@ +package main + +// The publish of a done build's archive (novox/hq design 51 *Publishing*, ADR 0611; CAT-BG1): from this module's own +// process, after the build step ended, through the seat oci-artifact-registry's verbs publish and publish-part, which +// the manifest `invokes`, with the build modules' one sender (buildstep.PublishOne): the archive in chunks, only what the holder +// lacks. No build step reaches the store, and this module no longer writes into it directly (push.go is gone): the +// holder checks the build with the builder and says published or refused on the seat's events. + +import ( + "context" + "encoding/json" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" +) + +// ArtifactSeat is the registry seat a bundle's or an archive's archive is published on. +const ArtifactSeat = buildstep.ArtifactSeat + +// transferName is the transfer's name, this module's own (buildstep.TransferName). +func transferName(module, artifact, digest string) string { + return buildstep.TransferName(module, artifact, digest) +} + +// publishArchive publishes body, of digest, as / of kind (bundle or archive) on the artifact seat. +func (m *Module) publishArchive(ctx context.Context, module, artifact, kind, digest string, body []byte) error { + return m.publishOn(ctx, ArtifactSeat, module+"/"+artifact, kind, "", digest, transferName(module, artifact, digest), body) +} + +// publishOn publishes body through buildstep.PublishOne: through Call when a test set it, else the runtime's request +// over the bus (stdio.Ask). +func (m *Module) publishOn(ctx context.Context, seat, name, kind, version, digest, transfer string, body []byte) error { + call := m.Call + if call == nil { + call = stdio.Ask + } + return buildstep.PublishOne(ctx, call, seat, name, kind, version, digest, transfer, body) +} + +// callFunc is the runtime's request and reply over the bus, as stdio.Ask makes it. +type callFunc = func(key string, body any) (json.RawMessage, error) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go b/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go new file mode 100644 index 00000000..221a9bd2 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go @@ -0,0 +1,286 @@ +package main + +import ( + "archive/zip" + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back. +var pushed sync.Map + +// fakeStore is the seat oci-artifact-registry's holder as m's publish meets it (publish.go): publish opens a transfer +// and answers the part it lacks, publish-part takes the chunks in sequence, and the last one stores the whole part +// when it hashes to its digest. Each blob is kept by name and digest; fail, when set, is the holder's refusal at the +// last chunk; asked are the addresses m called. +type fakeStore struct { + mu sync.Mutex + blobs map[string][]byte // "@" + fail error + open map[string]registry.PublishInput + chunks map[string][]byte + asked []string + // published are the publishes whose last chunk was stored, in order. + published []registry.PublishInput +} + +func withStore(t *testing.T, m *Module) *fakeStore { + t.Helper() + s := &fakeStore{blobs: map[string][]byte{}, open: map[string]registry.PublishInput{}, chunks: map[string][]byte{}} + m.Call = func(key string, body any) (json.RawMessage, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.asked = append(s.asked, key) + raw, _ := json.Marshal(body) + seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".") + if seat != ArtifactSeat && seat != GoPackageSeat { + return nil, errors.New("nats: no responders available for request") + } + switch verb { + case "publish": + var in registry.PublishInput + _ = json.Unmarshal(raw, &in) + if (seat == ArtifactSeat) != (in.Kind == "bundle" || in.Kind == "archive") || in.Transfer == "" || strings.ContainsAny(in.Transfer, "/. *>") { + t.Errorf("publish of kind %q, transfer %q", in.Kind, in.Transfer) + } + s.open[in.Transfer] = in + // A digest held under the name already travels no more (the late publish, CAT-BG2). + if _, held := s.blobs[in.Name+"@"+in.Digest]; held { + s.published = append(s.published, in) + return json.Marshal(registry.PublishAnswer{Transfer: in.Transfer, Held: true}) + } + return json.Marshal(registry.PublishAnswer{Transfer: in.Transfer, Lacks: []string{in.Digest}}) + case "publish-part": + var c registry.PublishPartInput + _ = json.Unmarshal(raw, &c) + data, _ := base64.StdEncoding.DecodeString(c.Data) + s.chunks[c.Transfer] = append(s.chunks[c.Transfer], data...) + if !c.Last { + return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part}) + } + in, whole := s.open[c.Transfer], s.chunks[c.Transfer] + delete(s.chunks, c.Transfer) + if s.fail != nil { + return nil, s.fail + } + sum := sha256.Sum256(whole) + if "sha256:"+hex.EncodeToString(sum[:]) != in.Digest { + t.Errorf("published %s with bytes of another digest", in.Digest) + } + s.blobs[in.Name+"@"+in.Digest] = whole + s.published = append(s.published, in) + pushed.Store(in.Digest, whole) + return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part, Complete: true}) + } + return nil, errors.New("nats: no responders available for request") + } + return s +} + +// archiveOf is the archive a build pushed, by its digest. +func archiveOf(t *testing.T, res BuildResult) []byte { + t.Helper() + raw, ok := pushed.Load(res.Digest) + if !ok { + t.Fatalf("nothing was pushed under %s", res.Digest) + } + return raw.([]byte) +} + +// 1587-S2 (ADR 0498 §1): a done build answers a reference of the stated form whose tail is the build's digest, and the +// store holds the archive there; a bundle and an archive alike. +func TestADoneBuildAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) { + m, states, _ := buildModule(t, manifestWith(goBundle), "the program") + store := withStore(t, m) + res, st := runBuild(t, m, states, "tools") + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + if want := "oci-artifact-registry/demo/tools@" + res.Digest; res.Reference != want { + t.Errorf("reference %q, want %q", res.Reference, want) + } + if store.blobs["demo/tools@"+res.Digest] == nil || store.asked[0] != "seat:oci-artifact-registry.publish" { + t.Errorf("the seat was asked %v and holds %d blobs, not demo/tools@%s", store.asked, len(store.blobs), res.Digest) + } + a, states2, _ := archiveModule(t, "") + withStore(t, a) + arch, st := runBuild(t, a, states2, "files") + if st.State != delivery.StepPassed || arch.Reference != "oci-artifact-registry/demo/files@"+arch.Digest { + t.Errorf("the archive artifact: %s %q, reference %q", st.State, st.Reason, arch.Reference) + } +} + +// 1587-S3 (ADR 0498 §2), CAT-BG1: a publish the seat's holder refuses fails the step `not stored: : `, and so does a seat nothing holds. +func TestARefusedPushFailsTheStepNotStored(t *testing.T) { + m, states, _ := buildModule(t, manifestWith(goBundle), "the program") + store := withStore(t, m) + store.fail = errors.New("refused: the forge did not take it: disk full") + if _, st := runBuild(t, m, states, "tools"); st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, delivery.NotStored+"tools: ") || !strings.Contains(st.Reason, "disk full") { + t.Errorf("the step ended %s: %q", st.State, st.Reason) + } + m.Call = func(string, any) (json.RawMessage, error) { + return nil, errors.New("nats: no responders available for request") + } + if _, st := runBuild(t, m, states, "tools"); st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, delivery.NotStored+"tools: ") || + !strings.Contains(st.Reason, "oci-artifact-registry") { + t.Errorf("no holder: the step ended %s: %q", st.State, st.Reason) + } +} + +// 1587-S4 (the lead's decision L9): the packed file is not left on the node once its push answered, a candidate's +// included: the workspace holds no archive, pushed or refused. +func TestThePackedFileLeavesTheNodeOnceItsPushAnswered(t *testing.T) { + for _, fail := range []error{nil, errors.New("refused")} { + m, states, _ := buildModule(t, manifestWith(goBundle), "the program") + withStore(t, m).fail = fail + runBuild(t, m, states, "tools") + var archives []string + filepath.WalkDir(m.Work, func(p string, d os.DirEntry, err error) error { + if err == nil && strings.HasSuffix(p, ".tar.gz") { + archives = append(archives, p) + } + return nil + }) + if len(archives) != 0 { + t.Errorf("push answered %v: archives left on the node: %v", fail, archives) + } + } +} + +// CAT-BG2 (REG S23, ADR 0605 §3, §9; the lead's ruling 3 on A3k): a Go package is built as Go's module zip of the +// directory its artifact names, every file under @/, its digest the zip's. +// A trunk build given a version publishes it through go-package-registry under its Go module path and that version, +// and answers @; a candidate's (no version) publishes nothing and answers its digest alone. A +// directory with no go.mod fails the step naming it. +func TestAGoPackageIsBuiltAsAModuleZipAndPublishedUnderItsModulePath(t *testing.T) { + const pkg = `{"name":"go","kind":"package","language":"go","paths":["go"],"series":"0.1"}` + m, states, _ := buildModule(t, manifestWith(pkg), "unused") + src := filepath.Join(m.Work, "ws1", "tree", "modules", "demo", "go") + for name, body := range map[string]string{"go.mod": "module git.example/demo/go\n\ngo 1.26\n", "a.go": "package demo\n", + "vendor/x/x.go": "package x\n"} { + if err := os.MkdirAll(filepath.Dir(filepath.Join(src, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + run := func(args map[string]any) (BuildResult, delivery.Step) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == "build_go_build" { + if _, err := tool.Run(args); err != nil { + t.Fatal(err) + } + } + } + st := states.waitEnd(t) + var res BuildResult + if st.State == delivery.StepPassed { + _ = json.Unmarshal(st.Result, &res) + } + return res, st + } + store := withStore(t, m) + args := map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "go"} + cand, st := run(args) + if st.State != delivery.StepPassed || !strings.HasPrefix(cand.Digest, "sha256:") || cand.Reference != "" || len(store.asked) != 0 { + t.Fatalf("a candidate's Go package: %s %q, %+v, asked %v", st.State, st.Reason, cand, store.asked) + } + args["version"], args["seat"] = "v0.1.15", GoPackageSeat + res, st := run(args) + if st.State != delivery.StepPassed || res.Reference != "git.example/demo/go@v0.1.15" { + t.Fatalf("a trunk Go package: %s %q, %+v", st.State, st.Reason, res) + } + if len(store.published) != 1 { + t.Fatalf("published %+v", store.published) + } + in := store.published[0] + if in.Name != "git.example/demo/go" || in.Version != "v0.1.15" || in.Kind != "package" || in.Digest != res.Digest { + t.Errorf("the publish: %+v", in) + } + z, err := zip.NewReader(bytes.NewReader(store.blobs[in.Name+"@"+in.Digest]), int64(len(store.blobs[in.Name+"@"+in.Digest]))) + if err != nil { + t.Fatal(err) + } + var names []string + for _, f := range z.File { + names = append(names, f.Name) + } + if strings.Join(names, " ") != "git.example/demo/go@v0.1.15/a.go git.example/demo/go@v0.1.15/go.mod" { + t.Errorf("the zip holds %v", names) + } + if err := os.Remove(filepath.Join(src, "go.mod")); err != nil { + t.Fatal(err) + } + if _, st := run(args); st.State != delivery.StepFailed || !strings.Contains(st.Reason, "go.mod") { + t.Errorf("no go.mod: %s %q", st.State, st.Reason) + } +} + +// CAT-BG2 (design 51 *The versions*): the builder's version.numbered for a bundle or an archive publishes the held +// digest under its version, sending nothing; a digest the holder no longer holds fails the event (offered again), +// and what is not this module's to publish is let pass. +func TestTheNumberedEventPublishesAHeldDigestUnderItsVersion(t *testing.T) { + digest := "sha256:" + strings.Repeat("ab", 32) + other := "sha256:" + strings.Repeat("cd", 32) + numbered := func(kind, seat, digest string) []byte { + raw, err := delivery.EncodeVersionNumbered(delivery.VersionNumbered{Name: "mod/tools", Kind: kind, Version: "0.1.4", + Module: "mod", Fingerprint: "fp", Commit: "c0", Digest: digest, Seat: seat}) + if err != nil { + t.Fatal(err) + } + return raw + } + for _, c := range []struct { + name string + body []byte + did, fail bool + }{ + {"a held bundle is tagged", numbered("bundle", ArtifactSeat, digest), true, false}, + {"an archive with no seat named goes to the artifact seat", numbered("archive", "", digest), true, false}, + {"a digest the holder lacks fails, sending nothing", numbered("bundle", ArtifactSeat, other), false, true}, + {"a Go package is not published late", numbered("package", GoPackageSeat, digest), false, false}, + {"an image is mesh-build-image's", numbered("image", "container-image-registry", digest), false, false}, + {"no digest, nothing to publish", numbered("bundle", ArtifactSeat, ""), false, false}, + {"an unreadable body fails", []byte(`{"name":"mod/tools"}`), false, true}, + } { + t.Run(c.name, func(t *testing.T) { + m := &Module{} + s := withStore(t, m) + s.blobs["mod/tools@"+digest] = []byte("held") + did, err := m.LatePublish(t.Context(), c.body) + if did != c.did || (err != nil) != c.fail { + t.Fatalf("did %v, err %v; want did %v, fail %v", did, err, c.did, c.fail) + } + for _, k := range s.asked { + if strings.HasSuffix(k, ".publish-part") { + t.Errorf("a late publish sent a part: %v", s.asked) + } + } + if !c.did { + return + } + if len(s.published) != 1 { + t.Fatalf("published %v", s.published) + } + p := s.published[0] + if p.Name != "mod/tools" || p.Version != "0.1.4" || p.Digest != digest || p.Fingerprint != "fp" || p.Commit != "c0" { + t.Errorf("published %+v", p) + } + }) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/push.go b/modules/mesh-build-go/cmd/mesh-build-go/push.go deleted file mode 100644 index fb468a03..00000000 --- a/modules/mesh-build-go/cmd/mesh-build-go/push.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -// The push of a done build's archive (novox/hq ADR 0498 §1 §2; issue 1587): from this module's own process, after the -// build step ended, with mesh-sdk's one client (buildstep.PushBlob), to the artifact store whose address the manifest -// binds into RegistryFile. No build step reaches the store: a run step has no network (ADR 0448). - -import ( - "context" - "fmt" - "net/http" - "os" - "strings" - "time" - - "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" -) - -// registry is the artifact store's address, host:port, as the manifest bound it into RegistryFile. -func (m *Module) registry() (string, error) { - raw, err := os.ReadFile(m.RegistryFile) - if err != nil { - return "", fmt.Errorf("the artifact store's address cannot be read: %w", err) - } - addr := strings.TrimSpace(string(raw)) - if addr == "" || strings.Contains(addr, "${") || strings.HasPrefix(addr, ":") || strings.HasSuffix(addr, ":") { - return "", fmt.Errorf("the artifact store's address in %s is %q, not host:port", m.RegistryFile, addr) - } - return addr, nil -} - -// push puts body into the artifact store as a blob of repository (/) by its digest: Push when a test -// set it, else buildstep.PushBlob. -func (m *Module) push(ctx context.Context, repository, digest string, body []byte) error { - addr, err := m.registry() - if err != nil { - return err - } - if m.Push != nil { - return m.Push(ctx, addr, repository, digest, body) - } - return buildstep.PushBlob(ctx, &http.Client{Timeout: 10 * time.Minute}, addr, repository, digest, body) -} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/push_test.go b/modules/mesh-build-go/cmd/mesh-build-go/push_test.go deleted file mode 100644 index b830db1b..00000000 --- a/modules/mesh-build-go/cmd/mesh-build-go/push_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package main - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "os" - "path/filepath" - "strings" - "sync" - "testing" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -// pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back. -var pushed sync.Map - -// fakeStore makes m push to a store of the test's own: its address in RegistryFile, each blob kept by repository and -// digest; fail, when set, is the store's refusal. -type fakeStore struct { - mu sync.Mutex - blobs map[string][]byte // "@" - fail error - addr string -} - -func withStore(t *testing.T, m *Module) *fakeStore { - t.Helper() - s := &fakeStore{blobs: map[string][]byte{}} - m.RegistryFile = filepath.Join(t.TempDir(), "registry") - if err := os.WriteFile(m.RegistryFile, []byte("store.test:5000\n"), 0o644); err != nil { - t.Fatal(err) - } - m.Push = func(ctx context.Context, registry, repository, digest string, body []byte) error { - s.mu.Lock() - defer s.mu.Unlock() - s.addr = registry - if s.fail != nil { - return s.fail - } - sum := sha256.Sum256(body) - if "sha256:"+hex.EncodeToString(sum[:]) != digest { - t.Errorf("pushed %s with bytes of another digest", digest) - } - s.blobs[repository+"@"+digest] = append([]byte(nil), body...) - pushed.Store(digest, s.blobs[repository+"@"+digest]) - return nil - } - return s -} - -// archiveOf is the archive a build pushed, by its digest. -func archiveOf(t *testing.T, res BuildResult) []byte { - t.Helper() - raw, ok := pushed.Load(res.Digest) - if !ok { - t.Fatalf("nothing was pushed under %s", res.Digest) - } - return raw.([]byte) -} - -// 1587-S2 (ADR 0498 §1): a done build answers a reference of the stated form whose tail is the build's digest, and the -// store holds the archive there; a bundle and an archive alike. -func TestADoneBuildAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) { - m, states, _ := buildModule(t, manifestWith(goBundle), "the program") - store := withStore(t, m) - res, st := runBuild(t, m, states, "tools") - if st.State != delivery.StepPassed { - t.Fatalf("the step ended %s: %s", st.State, st.Reason) - } - if want := "artifact-store://demo/tools/blobs/" + res.Digest; res.Reference != want { - t.Errorf("reference %q, want %q", res.Reference, want) - } - if store.addr != "store.test:5000" || store.blobs["demo/tools@"+res.Digest] == nil { - t.Errorf("the store at %q holds %d blobs, not demo/tools@%s", store.addr, len(store.blobs), res.Digest) - } - a, states2, _ := archiveModule(t, "") - withStore(t, a) - arch, st := runBuild(t, a, states2, "files") - if st.State != delivery.StepPassed || arch.Reference != "artifact-store://demo/files/blobs/"+arch.Digest { - t.Errorf("the archive artifact: %s %q, reference %q", st.State, st.Reason, arch.Reference) - } -} - -// 1587-S3 (ADR 0498 §2): a push the store refuses fails the step `not stored: : `, and so -// does a store whose address the manifest did not place. -func TestARefusedPushFailsTheStepNotStored(t *testing.T) { - m, states, _ := buildModule(t, manifestWith(goBundle), "the program") - store := withStore(t, m) - store.fail = errors.New("store.test:5000 refused the blob: 500 Internal Server Error disk full") - if _, st := runBuild(t, m, states, "tools"); st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, delivery.NotStored+"tools: ") || !strings.Contains(st.Reason, "disk full") { - t.Errorf("the step ended %s: %q", st.State, st.Reason) - } - m.RegistryFile = filepath.Join(t.TempDir(), "absent") - if _, st := runBuild(t, m, states, "tools"); st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, delivery.NotStored+"tools: ") { - t.Errorf("no address: the step ended %s: %q", st.State, st.Reason) - } -} - -// 1587-S4 (the lead's decision L9): the packed file is not left on the node once its push answered, a candidate's -// included: the workspace holds no archive, pushed or refused. -func TestThePackedFileLeavesTheNodeOnceItsPushAnswered(t *testing.T) { - for _, fail := range []error{nil, errors.New("refused")} { - m, states, _ := buildModule(t, manifestWith(goBundle), "the program") - withStore(t, m).fail = fail - runBuild(t, m, states, "tools") - var archives []string - filepath.WalkDir(m.Work, func(p string, d os.DirEntry, err error) error { - if err == nil && strings.HasSuffix(p, ".tar.gz") { - archives = append(archives, p) - } - return nil - }) - if len(archives) != 0 { - t.Errorf("push answered %v: archives left on the node: %v", fail, archives) - } - } -} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/store_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/store_integration_test.go index 1a2fa7ae..a9212974 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/store_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/store_integration_test.go @@ -3,12 +3,16 @@ package main import ( + "bytes" "context" "crypto/sha256" "encoding/hex" + "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" + "net/url" "os" "os/exec" "path/filepath" @@ -19,6 +23,8 @@ import ( "github.com/distribution/distribution/v3/registry/handlers" _ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" + registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -34,23 +40,68 @@ func realStore(t *testing.T, m *Module) string { srv := httptest.NewServer(handlers.NewApp(context.Background(), config)) t.Cleanup(srv.Close) addr := strings.TrimPrefix(srv.URL, "http://") - m.RegistryFile = filepath.Join(t.TempDir(), "registry") - if err := os.WriteFile(m.RegistryFile, []byte(addr+"\n"), 0o644); err != nil { - t.Fatal(err) + // The seat's holder as distribution holds it: the publish through the seat's verbs (publish.go, played by withStore), + // each whole part then stored in the store under / by its digest. + s := withStore(t, m) + inner := m.Call + m.Call = func(key string, body any) (json.RawMessage, error) { + raw, err := inner(key, body) + var a registry.PublishPartAnswer + if err != nil || !strings.HasSuffix(key, ".publish-part") || json.Unmarshal(raw, &a) != nil || !a.Complete { + return raw, err + } + s.mu.Lock() + defer s.mu.Unlock() + for k, b := range s.blobs { + name, digest, _ := strings.Cut(k, "@") + if err := uploadBlob(srv.URL, name, digest, b); err != nil { + return nil, err + } + delete(s.blobs, k) + } + return raw, nil } - m.Push = nil return addr } -// resolve reads a done artifact's reference from the store, as the controller will: artifact-store:/// -// /blobs/ is /v2///blobs/. +// uploadBlob is the holder's own write into its store: an upload opened, then its bytes put by their digest (the +// distribution protocol's two requests), as distribution stores a part it took whole. +func uploadBlob(base, name, digest string, body []byte) error { + resp, err := http.Post(base+"/v2/"+name+"/blobs/uploads/", "", nil) + if err != nil { + return err + } + resp.Body.Close() + loc, err := url.Parse(resp.Header.Get("Location")) + if err != nil || resp.StatusCode != http.StatusAccepted { + return fmt.Errorf("the store opened no upload of %s: %s %v", digest, resp.Status, err) + } + b, _ := url.Parse(base) + u := b.ResolveReference(loc) + q := u.Query() + q.Set("digest", digest) + u.RawQuery = q.Encode() + req, _ := http.NewRequest(http.MethodPut, u.String(), bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/octet-stream") + put, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + put.Body.Close() + if put.StatusCode != http.StatusCreated { + return fmt.Errorf("the store did not take %s: %s", digest, put.Status) + } + return nil +} + +// resolve reads a done artifact's reference from the store as a node-engine does (catalogue.Routed): the seat's form +// oci-artifact-registry//@ is /v2///blobs/ (ADR 0605 §9). func resolve(t *testing.T, addr string, a delivery.BuiltArtifact) []byte { t.Helper() - path, ok := strings.CutPrefix(a.Reference, "artifact-store://") - if !ok { - t.Fatalf("%s's reference %q is not the artifact store's", a.Name, a.Reference) + if _, _, ok := catalogue.InRegistrySeat(a.Reference); !ok { + t.Fatalf("%s's reference %q is not in the form a registry seat answers", a.Name, a.Reference) } - resp, err := http.Get("http://" + addr + "/v2/" + path) + resp, err := http.Get(catalogue.Routed(a.Reference, addr)) if err != nil { t.Fatal(err) } @@ -101,7 +152,7 @@ func TestADoneBuildsArtifactsResolveInARealArtifactStore(t *testing.T) { t.Errorf("the done key is not deliverable, missing %v", missing) } for _, a := range done.Artifacts { - if !strings.HasSuffix(a.Reference, "/blobs/"+a.Digest) { + if !strings.HasSuffix(a.Reference, "@"+a.Digest) { t.Errorf("%s: the reference %q does not end in its digest %s", a.Name, a.Reference, a.Digest) } sum := sha256.Sum256(resolve(t, addr, a)) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index f02dcc07..5d1823f6 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -15,6 +15,7 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" sdkstep "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/forgekey" @@ -100,9 +101,42 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { return nil, err } res.JudgedBy = RunnerInstalled + if level == LevelUnit { + if err := compareGenerated(tree, res); err != nil { + return nil, err + } + } return res, nil } +// compareGenerated regenerates the generated Go files of each contracts module in the tree (the tree's own manifest and +// each modules//module.json; CAT-BG5) and fails the level on any out of step, Why naming each path. +func compareGenerated(tree string, res *TestResult) error { + manifests, _ := filepath.Glob(filepath.Join(tree, "modules", "*", "module.json")) + if _, err := os.Stat(filepath.Join(tree, "module.json")); err == nil { + manifests = append([]string{filepath.Join(tree, "module.json")}, manifests...) + } + for _, mf := range manifests { + dir := filepath.Dir(mf) + paths, err := buildstep.OutOfStep(dir, buildstep.GeneratedRootGo) + if err != nil { + return err + } + rel, _ := filepath.Rel(tree, dir) + for _, p := range paths { + res.OutOfStep = append(res.OutOfStep, filepath.ToSlash(filepath.Join(rel, p))) + } + } + if len(res.OutOfStep) > 0 { + why := "generated files out of step with the seat generator (regenerate them): " + strings.Join(res.OutOfStep, ", ") + if res.Why != "" { + why = res.Why + "; " + why + } + res.Passed, res.Why = false, why + } + return nil +} + // testAsCandidate runs a level as the candidate's program (ADR 0535 §1): the integration level is refused first as the // installed program would refuse it (no postgres), then the whole level runs in the candidate's one shut-off process. func (m *Module) testAsCandidate(ctx context.Context, workspace, level string, packages []string, pattern string, names []string) (*TestResult, error) { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go index aa6d1d1a..b0206d38 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go @@ -178,6 +178,40 @@ func TestUnitLevelOfAFixtureWithNoTestsSaysSo(t *testing.T) { } } +// CAT-BG5: a contracts module (a package artifact naming generated-from) whose generated Go files are out of step with +// the seat generator fails the unit level, each path named in its out-of-step and its why; its TypeScript is not this +// module's to compare. +func TestUnitLevelOfAContractsModuleOutOfStepFailsNamingEachPath(t *testing.T) { + m, states, ws, _ := workspaceFor(t, "empty") + tree := filepath.Join(m.Work, ws, "tree") + for p, text := range map[string]string{ + "module.json": `{"module":"c","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"],"generated-from":"."}]}}`, + "seats/index.json": `{"format":1,"seats":["one-store"]}`, + "seats/one-store/seat.json": `{"format":1,"seat":"one-store","scope":"mesh","grain":"seat-list","decision":"ADR 1","verbs":[{"name":"query","input":{"type":"object"}}]}`, + "go/go.mod": "module example.org/c/go\n", + "go/stray.go": "package stray\n", + "ts/package.json": `{"name":"@example/c"}`, + "ts/stray.ts": "// not this module's\n", + } { + full := filepath.Join(tree, filepath.FromSlash(p)) + _ = os.MkdirAll(filepath.Dir(full), 0o755) + if err := os.WriteFile(full, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + res, st := runUnit(t, m, states, ws) + stray := false + for _, p := range res.OutOfStep { + stray = stray || p == "go/stray.go" + if !strings.HasPrefix(p, "go/") { + t.Errorf("%s is not under the Go root", p) + } + } + if st.State != delivery.StepPassed || res.Passed || !stray || !strings.Contains(res.Why, "go/stray.go") { + t.Errorf("result = %+v (%s %s)", res, st.State, st.Reason) + } +} + // A package that does not build fails the level with the compiler's words. func TestUnitLevelOfAPackageThatDoesNotBuildFailsWithTheCompilersLine(t *testing.T) { m, states, ws, _ := workspaceFor(t, "broken") diff --git a/modules/mesh-build-go/files/registry b/modules/mesh-build-go/files/registry deleted file mode 100644 index 1bcadab2..00000000 --- a/modules/mesh-build-go/files/registry +++ /dev/null @@ -1 +0,0 @@ -${seat:mesh-artifact-store:reach} diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 2b01790e..58153282 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -9,9 +9,18 @@ require ( github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 github.com/therootcompany/xz v1.0.1 + golang.org/x/mod v0.42.0 ) require ( + git.novox.be/novox/mesh-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/gopackageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/npmpackageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/ociartifactregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index 52c1ca63..32a87eb9 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -178,3 +178,5 @@ google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index cf557092..f05faf62 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -19,6 +19,9 @@ "serves": ["current", "pause", "resume"] } ], + "consumes": [ + "mesh-delivery-builder.version.numbered" + ], "reads": [ "mesh-delivery-checker.kept" ], @@ -68,9 +71,18 @@ "requires": [ "forge-account", "go-toolchain", - "artifact-store", + "go-package-registry", "node-tools" ], + "binds": { + "go-package-registry": "${dir:state}/go-package-registry.json" + }, + "invokes": [ + "seat:oci-artifact-registry.publish", + "seat:oci-artifact-registry.publish-part", + "seat:go-package-registry.publish", + "seat:go-package-registry.publish-part" + ], "own-secrets": { "broker": "${dir:state}/broker" }, @@ -158,14 +170,6 @@ "mode": "0644", "template": "files/gitconfig" }, - { - "id": "registry", - "type": "file", - "path": "${dir:state}/registry", - "owner": "mesh-build-go", - "mode": "0644", - "template": "files/registry" - }, { "id": "git", "type": "package", @@ -204,6 +208,15 @@ "name": "files", "kind": "archive", "from": "files" + }, + { + "name": "buildstep", + "kind": "package", + "language": "go", + "paths": [ + "buildstep" + ], + "series": "0.1" } ] } diff --git a/modules/mesh-build-image/.gitignore b/modules/mesh-build-image/.gitignore index 00eb03a8..6a944359 100644 --- a/modules/mesh-build-image/.gitignore +++ b/modules/mesh-build-image/.gitignore @@ -1 +1,3 @@ -mesh-build-image +# the binary, built at the module or in its command directory; never the command directory itself +/mesh-build-image +/cmd/mesh-build-image/mesh-build-image diff --git a/modules/mesh-build-image/README.md b/modules/mesh-build-image/README.md index 86ebe743..e0f7dd92 100644 --- a/modules/mesh-build-image/README.md +++ b/modules/mesh-build-image/README.md @@ -34,14 +34,23 @@ The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can re - `build_image_build` builds one image artifact: `buildah bud` with the Dockerfile `from` names (in the module's directory or a subdirectory of it) as file and the module's own directory as context, so a COPY names paths from the module's root (issue 1703), `--target`, and the images `build.on` pins as `--build-arg`. Answers the image's digest. With - `publish: true` (a merged trunk commit only) the module's own process then pushes it to the registry under - `/:` (the first 12 hex digits of the digest unless the call gives one), waiting out - a registry held still, and the key carries the digest the registry serves and the reference pinned by it. Without - `publish` (a pull request) nothing is pushed. -- `build_image_copy` copies an upstream artifact (`module`, `artifact`, `from`) registry to registry into the artifact store - over the registry API, the whole index and every blob, by the digest the manifest pins (ADR 0498 §4), and verifies the - store serves that digest. Answers the digest and the reference `//@sha256:`. A - `from` without a digest fails `not pinned:`. It needs no workspace and no buildah. + `publish: true` (a merged trunk commit only) the module's own process then writes it as an OCI layout in the + workspace and publishes it through the seat `container-image-registry` (`publish`, `publish-part`, which the manifest + `invokes`; design 51 *Publishing*) as `/` at the ask's `version` (and a `tag` only when the call gives + one): its config, layers and manifest by digest, of which only the parts the holder lacks travel. The key carries the + layout's manifest digest and the seat's reference `container-image-registry//@sha256:`; a + publish the holder does not take fails `not stored:`. Without `publish` (a pull request) nothing is published. + An image the builder had not numbered yet is published by its digest alone; the module consumes the builder's + `version.numbered` and publishes that held digest under its version, sending nothing (`latepublish.go`, ADR 0617 §1); + a holder that no longer holds the digest fails the event, which is offered again. +- `build_image_copy` copies an upstream artifact (`module`, `artifact`, `from`) into the image seat by its `mirror`, by the + digest the manifest pins (ADR 0498 §4): the holder copies it and says `published` or `refused` on the seat's events, + which the manifest `consumes`; the step ends on that outcome (or fails `not stored:` after 30 minutes without one) and + answers the digest and the seat's reference. A `from` without a digest fails `not pinned:` before the seat is asked. It + needs no workspace, no buildah, and reads no outside registry itself. The copy's wait is kept in the state + `mirror-waits` under its digest before the seat is asked: the seat's events reach one instance of the module, which + writes the outcome into that key, and the waiting copy reads it there, woken by its watch of the state. A copy removes + its wait when it ends; one a stopped process left goes after two days. `build_image_mirror_waits` reads that state. - It refuses, naming why, an image whose build context is a whole repository (`context`) or whose `build.on` names another module's artifact (`unsupportedImage` in `build.go`): those forms belong to the images build-agent builds until it is removed at R4g. @@ -53,4 +62,5 @@ The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can re (issue 1839). A copy writes nothing to the disk and is not refused. - `build_image_step`, `build_image_steps`, `build_image_status` and `build_image_health` read. -The registry's address is placed into `/registry` as `${seat:mesh-artifact-store:reach}` (host:port): the provisioner's client fills a moved module's placement, and the artifact store has not moved, so a `${bound:artifact-store:...}` is refused at composition (issue 1703). +The module holds no registry address and no registry credential: every write goes through the image seat's verbs, and +the holder checks each publish with the builder. diff --git a/modules/mesh-build-image/cmd/mesh-build-image/announce.go b/modules/mesh-build-image/cmd/mesh-build-image/announce.go index ee48ed0a..0ffa6fff 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/announce.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/announce.go @@ -14,6 +14,7 @@ var Kept = []announce.State{ {Name: "keys", Readable: true}, {Name: "steps", Readable: true}, {Name: "pauses", Readable: true}, + {Name: MirrorWaitsState}, } // keepStates announces Kept as this module's (stdio.Keeps): main calls it before it serves. diff --git a/modules/mesh-build-image/cmd/mesh-build-image/build.go b/modules/mesh-build-image/cmd/mesh-build-image/build.go index fba7ceae..307446b4 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/build.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/build.go @@ -3,8 +3,8 @@ package main // The build step (novox/hq ADR 0487 §1 §2 §5 §6; issue 1703c): one `image` artifact of a module of a fetched workspace, // built from the Dockerfile in the module's own directory with buildah, rootless: as the module's own account, in a // user namespace, with its image store outside the home, no mount of the host, no registry credential and an -// environment built from scratch. The step runs the Dockerfile and nothing else; the push of a trunk build is made -// after the build has ended (push.go). +// environment built from scratch. The step runs the Dockerfile and nothing else; the publish of a trunk build through +// the image seat is made after the build has ended (publish.go). import ( "context" @@ -21,7 +21,7 @@ import ( ) // BuildResult is what a passed build step says: the image's manifest digest as buildah holds it, and, for a trunk -// build that published, the digest the registry serves and the reference pinned by it. +// build that published, the digest it was published by and the seat's reference pinned by it. type BuildResult struct { Artifact string `json:"artifact"` Digest string `json:"digest"` @@ -36,16 +36,19 @@ func (m *Module) buildTool() StepTool { Name: BuildTool, Description: "Build one image artifact of a module of a fetched workspace from the Dockerfile in the module's own directory, " + "rootless with buildah. Answers a step id at once; the step ends on its key with the image's digest. With publish " + - "true (a merged trunk commit) the image is then pushed to the registry from this module's own process and the key " + - "carries the digest the registry serves and the reference pinned by it; without it (a pull request) nothing is pushed.", + "true (a merged trunk commit) the image is then published through the seat container-image-registry from this " + + "module's own process, only the layers the holder lacks, and the key carries the digest it was published by and the " + + "seat's reference, container-image-registry//@sha256:; without it (a pull request) nothing is published.", Input: map[string]any{ "type": "object", "properties": map[string]any{ "workspace": map[string]any{"type": "string", "description": "the workspace id build_image_fetch answered"}, "module": map[string]any{"type": "string", "description": "the module's directory in the workspace, such as modules/n8n"}, "artifact": map[string]any{"type": "string", "description": "the image artifact's name in the module's manifest"}, - "publish": map[string]any{"type": "boolean", "description": "push the image to the registry after the build: only for a merged trunk commit"}, - "tag": map[string]any{"type": "string", "description": "the registry tag of a published image; the first 12 hex digits of the digest when left out"}, + "publish": map[string]any{"type": "boolean", "description": "publish the image through the image seat after the build: only for a merged trunk commit"}, + "seat": map[string]any{"type": "string", "description": "the seat it is published through, container-image-registry (the ask's own)"}, + "version": map[string]any{"type": "string", "description": "the version the builder gave it, named in the seat's publish"}, + "tag": map[string]any{"type": "string", "description": "a tag the published version carries, by hand; none when left out"}, }, "required": []string{"workspace", "module", "artifact"}, }, @@ -111,6 +114,10 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { name, _ := args["artifact"].(string) publish, _ := args["publish"].(bool) tag, _ := args["tag"].(string) + version, _ := args["version"].(string) + if seat, _ := args["seat"].(string); seat != "" && seat != buildstep.ImageSeat { + return nil, &buildstep.Refusal{Text: fmt.Sprintf("an image is published through %s, not %s", buildstep.ImageSeat, seat)} + } if !workspaceOK.MatchString(workspace) { return nil, fmt.Errorf("%q is not a workspace id", workspace) } @@ -177,10 +184,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { if !publish { return res, nil } - if tag == "" { - tag = strings.TrimPrefix(digest, "sha256:")[:12] - } - return m.push(ctx, workspace, manifest.Module, local, res, tag) + return m.publish(ctx, workspace, manifest.Module, local, res, version, tag) } // dockerfileOf is the Dockerfile an artifact's `from` names and the build context: the module's own directory, the diff --git a/modules/mesh-build-image/cmd/mesh-build-image/bus_integration_test.go b/modules/mesh-build-image/cmd/mesh-build-image/bus_integration_test.go index bc99e983..4444d927 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/bus_integration_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/bus_integration_test.go @@ -151,7 +151,7 @@ func TestTheStepsAnswerThroughARealBusWithTheCapturedShapes(t *testing.T) { } compareCaptured(t, name, normalized(t, st, "", "", f.address())) } - if len(f.registry) != 1 || f.pushes != 1 { - t.Errorf("the registry holds %v after %d pushes, want only the trunk build's one", f.registry, f.pushes) + if len(f.published) != 1 || f.pushes != 1 { + t.Errorf("the image seat was asked %v after %d layouts, want only the trunk build's one", f.published, f.pushes) } } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go b/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go index 7acb4454..d22fb01b 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go @@ -1,25 +1,45 @@ package main import ( + "encoding/json" "os" "path/filepath" + "slices" "strings" "testing" ) -// Issue 1703 (the live failure of the first assignment): the module is moved, so the provisioner's client on the node -// fills its templates, and it fills only config, secrets, collections, placement and what a moved provider told it. The -// registry's address therefore cannot be a ${bound:artifact-store:...} (the artifact store has not moved): it is the -// placement ${seat:mesh-artifact-store:reach}, the form the container runtime's daemon.json uses. The module's own part -// of that is what this holds: its registry file is that placement and none of its files or its manifest binds the store. -// That a node composes it is the node-engine's composition test (.work/sdk1/proofs-for-node-engine.md). -func TestTheRegistryIsThePlacementAndNoFileBindsTheStore(t *testing.T) { - raw, err := os.ReadFile(filepath.Join(moduleDir, "files", "registry")) - if err != nil { +// CAT-BI1 (design 51 *Publishing*): the module publishes and copies through the image seat's verbs, which its manifest +// invokes, hears the seat's outcomes through its consumes, and holds no registry: no registry file, no artifact-store +// requirement, and none of its files names the store's placement or binds it (issue 1703: the module is moved, and only +// what the mesh fills reaches its files). +func TestTheModuleInvokesTheImageSeatAndHoldsNoRegistry(t *testing.T) { + var m struct { + Requires []string `json:"requires"` + Invokes []string `json:"invokes"` + Consumes []string `json:"consumes"` + } + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil || json.Unmarshal(raw, &m) != nil { t.Fatal(err) } - if got := strings.TrimSpace(string(raw)); got != "${seat:mesh-artifact-store:reach}" { - t.Errorf("the registry file is %q, want the placement ${seat:mesh-artifact-store:reach}", got) + if want := []string{"seat:container-image-registry.publish", "seat:container-image-registry.publish-part", + "seat:container-image-registry.mirror"}; !slices.Equal(m.Invokes, want) { + t.Errorf("invokes = %v, want %v", m.Invokes, want) + } + // The builder's version.numbered for the late publish (ADR 0617 §1), and the seat's outcomes a copy waits on. + consumes := []string{NumberedPattern} + for _, e := range OutcomeEvents { + consumes = append(consumes, "container-image-registry."+e) + } + if !slices.Equal(m.Consumes, consumes) { + t.Errorf("consumes = %v, want %v", m.Consumes, consumes) + } + if slices.Contains(m.Requires, "artifact-store") { + t.Errorf("requires = %v: the module holds no store", m.Requires) + } + if _, err := os.Stat(filepath.Join(moduleDir, "files", "registry")); !os.IsNotExist(err) { + t.Errorf("files/registry is still there: %v", err) } files := []string{filepath.Join(moduleDir, "module.json")} entries, err := os.ReadDir(filepath.Join(moduleDir, "files")) @@ -34,8 +54,8 @@ func TestTheRegistryIsThePlacementAndNoFileBindsTheStore(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.Contains(string(body), "${bound:artifact-store") { - t.Errorf("%s binds the artifact store, which has not moved: a moved module's client cannot fill it", f) + if strings.Contains(string(body), "${bound:artifact-store") || strings.Contains(string(body), "mesh-artifact-store") { + t.Errorf("%s names the artifact store", f) } } } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/copy.go b/modules/mesh-build-image/cmd/mesh-build-image/copy.go index 7a7405f5..29866320 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/copy.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/copy.go @@ -1,34 +1,41 @@ package main -// The copy step (novox/hq ADR 0498 §4; issue 1587): an upstream artifact is copied registry to registry into the -// artifact store, whole (an index with every manifest it lists, every blob), by the digest its manifest pins. The copy -// runs no code of a change, so it needs neither buildah nor an isolation; it is made over the registry API, which -// keeps every digest as it is (a pull and a push through buildah may rewrite a manifest). An upstream without a -// digest is not copied: `not pinned:`. A copy that does not end with the pinned digest in the store is `not stored:`. +// The copy step (novox/hq ADR 0498 §4; design 51: an upstream image is copied by `mirror` with its pinned digest; +// CAT-BI1): an upstream artifact is copied into the image seat by the seat's verb mirror, which the manifest +// `invokes`, by the digest its manifest pins. The holder copies it, every platform and only what it lacks, and says +// published or refused on the seat's events; the step waits on that outcome, heard through the manifest's `consumes`, +// not on the call, so a large base never outlasts a call's bound; the wait is kept in the state mirror-waits, so an +// outcome another instance heard is matched there. The copy runs no code of a change and reads no +// outside registry itself. An upstream without a digest is not copied: `not pinned:`. A copy the holder does not end +// with the pinned digest stored is `not stored:`. import ( - "bytes" "context" - "crypto/sha256" - "encoding/hex" "encoding/json" + "errors" "fmt" - "io" - "net/http" - "net/url" "regexp" + "sort" + "strconv" "strings" + "sync" "time" - "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" ) // CopyTool is the copy step's tool. const CopyTool = "build_image_copy" -// CopyResult is what a passed copy step says: the pinned digest, now in the store, and the reference to it in the -// form an image's is (//@sha256:). +// MirrorWait is how long a copy waits for the holder's outcome before it fails `not stored:`. +const MirrorWait = 30 * time.Minute + +// CopyResult is what a passed copy step says: the pinned digest, now held by the image seat, and the seat's reference +// to it (container-image-registry/@sha256:). type CopyResult struct { Artifact string `json:"artifact"` Digest string `json:"digest"` @@ -37,19 +44,17 @@ type CopyResult struct { } var ( - nameOK = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*(/[a-z0-9][a-z0-9._-]*)*$`) - pinnedOK = regexp.MustCompile(`@(sha256:[0-9a-f]{64})$`) - manifestKind = strings.Join([]string{ - "application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json", - "application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json"}, ", ") + nameOK = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*(/[a-z0-9][a-z0-9._-]*)*$`) + pinnedOK = regexp.MustCompile(`@(sha256:[0-9a-f]{64})$`) ) func (m *Module) copyTool() StepTool { return StepTool{ Name: CopyTool, - Description: "Copy an upstream image, registry to registry, into the artifact store by the digest the manifest pins " + - "(novox/hq ADR 0498 §4). Answers a step id at once; the step ends on its key with the digest in the store and the " + - "reference to it. A reference without a digest fails `not pinned:`. Needs no workspace and runs no code of a change.", + Description: "Copy an upstream image into the seat container-image-registry by the digest the manifest pins, through " + + "the seat's mirror (novox/hq ADR 0498 §4, design 51). Answers a step id at once; the step ends on its key with the " + + "digest held and the seat's reference to it, when the holder says published. A reference without a digest fails " + + "`not pinned:`, a refusal `not stored:`. Needs no workspace and runs no code of a change.", Input: map[string]any{ "type": "object", "properties": map[string]any{ @@ -64,38 +69,13 @@ func (m *Module) copyTool() StepTool { } } -// upstreamRef is an image reference split the way a container runtime does: no host means the public hub. -type upstreamRef struct { - base, repository, digest string -} - -func parseUpstream(from string) (upstreamRef, error) { +// pinned is the digest a reference pins. +func pinned(from string) (string, error) { pin := pinnedOK.FindStringSubmatch(from) - if pin == nil { - return upstreamRef{}, fmt.Errorf("%q names no digest", from) + if pin == nil || strings.TrimSuffix(from, "@"+pin[1]) == "" { + return "", fmt.Errorf("%q names no digest", from) } - name := strings.TrimSuffix(from, "@"+pin[1]) - if colon := strings.LastIndex(name, ":"); colon > strings.LastIndex(name, "/") { - name = name[:colon] // the tag is only what it was called; the digest names the image - } - if name == "" { - return upstreamRef{}, fmt.Errorf("%q is not an image reference", from) - } - host, repository := "docker.io", name - if slash := strings.Index(name, "/"); slash >= 0 && (strings.ContainsAny(name[:slash], ".:") || name[:slash] == "localhost") { - host, repository = name[:slash], name[slash+1:] - } - if host == "docker.io" { - host = "registry-1.docker.io" - if !strings.Contains(repository, "/") { - repository = "library/" + repository - } - } - scheme := "https://" - if strings.HasPrefix(host, "localhost") || strings.HasPrefix(host, "127.") { - scheme = "http://" - } - return upstreamRef{base: scheme + host, repository: repository, digest: pin[1]}, nil + return pin[1], nil } func (m *Module) copy(ctx context.Context, args map[string]any) (any, error) { @@ -105,260 +85,289 @@ func (m *Module) copy(ctx context.Context, args map[string]any) (any, error) { if !nameOK.MatchString(module) || !nameOK.MatchString(name) { return nil, fmt.Errorf("%q / %q: a module and an artifact are named in lower case letters, digits, dots, dashes and underscores", module, name) } - src, err := parseUpstream(from) + digest, err := pinned(from) if err != nil { return nil, fmt.Errorf("not pinned: %s: %v; an upstream artifact is copied by the digest its manifest pins", name, err) } - addr, err := m.registry() + if m.Outcomes == nil || m.Outcomes.Waits == nil { + return nil, fmt.Errorf("%s%s: this process hears no outcome of the image seat", delivery.NotStored, name) + } + key := MirrorWaitKey(digest) + now := time.Now + if m.Now != nil { + now = m.Now + } + asked := CopyWait{Digest: digest, Module: module, Artifact: name, From: from, Node: m.Node, + Asked: now().UTC(), Token: strconv.FormatInt(now().UnixNano(), 36) + "-" + m.Node} + // Waited for before the ask, in memory and in the state mirror-waits, so an outcome said at once, or heard by another + // instance of the module (the seat's events reach one of them), is not missed. + heard, stop := m.Outcomes.Expect(digest) + defer stop() + if _, err := m.Outcomes.Waits.Put(key, asked); err != nil { + return nil, fmt.Errorf("%s%s: keeping the copy's wait in %s: %v", delivery.NotStored, name, MirrorWaitsState, err) + } + defer m.Outcomes.drop(key, asked.Token) + said := buildstep.NarrateSeat(ctx, buildstep.ImageSeat, "mirror of "+from) + raw, err := m.call()("seat:"+buildstep.ImageSeat+"."+reg.VerbMirror, reg.MirrorInput{Name: module + "/" + name, Source: from, Digest: digest}) if err != nil { - return nil, fmt.Errorf("not stored: %s: %w", name, err) + said.Refused(err) + return nil, fmt.Errorf("%s%s: %s's mirror: %v", delivery.NotStored, name, buildstep.ImageSeat, err) } - repository := module + "/" + name - c := &copier{m: m, src: src, dstBase: "http://" + addr, dstRepo: repository, done: map[string]bool{}} - if err := m.waitForRegistry(ctx, addr, "copying "+from, func() error { return c.copyManifest(ctx, src.digest) }); err != nil { - return nil, fmt.Errorf("not stored: %s: %w", name, err) + var answer reg.MirrorAnswer + _ = json.Unmarshal(raw, &answer) + m.say("asked %s to copy %s (%s); waiting for its outcome", buildstep.ImageSeat, from, answer.Reference) + wait := m.MirrorWait + if wait == 0 { + wait = MirrorWait } - if err := c.verify(ctx); err != nil { - return nil, fmt.Errorf("not stored: %s: %w", name, err) - } - return CopyResult{Artifact: name, Digest: src.digest, Reference: addr + "/" + repository + "@" + src.digest, Pushed: true}, nil -} - -// copier copies one image from its upstream registry into the store's repository. -type copier struct { - m *Module - src upstreamRef - dstBase string - dstRepo string - token string - done map[string]bool -} - -func (c *copier) client() *http.Client { - if c.m.HTTP != nil { - return c.m.HTTP - } - return &http.Client{Timeout: 30 * time.Minute} -} - -// get reads from the upstream registry, answering one bearer challenge with an anonymous token. -func (c *copier) get(ctx context.Context, path, accept string) (*http.Response, error) { - for attempt := 0; attempt < 2; attempt++ { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.src.base+"/v2/"+c.src.repository+path, nil) - if err != nil { - return nil, err - } - if accept != "" { - req.Header.Set("Accept", accept) - } - if c.token != "" { - req.Header.Set("Authorization", "Bearer "+c.token) - } - resp, err := c.client().Do(req) - if err != nil { - return nil, err - } - if resp.StatusCode == http.StatusUnauthorized && attempt == 0 { - challenge := resp.Header.Get("Www-Authenticate") - resp.Body.Close() - if c.token, err = c.fetchToken(ctx, challenge); err != nil { - return nil, err + bound := time.After(wait) + for { + // The state first: an outcome written there while this copy did not hear it is this copy's. + if o, ok := m.Outcomes.kept(key); ok { + res, err := copyEnd(name, o, answer.Reference) + if err != nil { + said.Refused(errors.New(o.Reason)) + } else { + said.Stored(res.(CopyResult).Reference) } + return res, err + } + select { + case <-heard: + case <-said.Waiting(): + said.StillWaiting() + case <-bound: + err := fmt.Errorf("%s said no outcome of the copy of %s in %s", buildstep.ImageSeat, from, wait) + said.Refused(err) + return nil, fmt.Errorf("%s%s: %w", delivery.NotStored, name, err) + case <-ctx.Done(): + said.Refused(ctx.Err()) + return nil, fmt.Errorf("%s%s: stopped while waiting for the copy of %s: %w", delivery.NotStored, name, from, ctx.Err()) + } + } +} + +// copyEnd is a copy's end on the outcome kept for it. +func copyEnd(name string, o MirrorOutcome, asked string) (any, error) { + if o.Event == reg.EventRefused { + return nil, fmt.Errorf("%s%s: %s refused the copy: %s", delivery.NotStored, name, buildstep.ImageSeat, o.Reason) + } + ref := o.Reference + if ref == "" { + ref = asked + } + return CopyResult{Artifact: name, Digest: o.Digest, Reference: ref, Pushed: true}, nil +} + +// The state mirror-waits (CAT-BI1, the lead's ruling on A3r): each upstream copy waiting on the image seat's outcome, +// keyed by its digest, with the outcome once an instance of the module heard it. The seat's published and refused reach +// one instance of the module, not necessarily the one whose copy waits, and none while no instance listens; the +// instance that hears it writes it to the copy's key, and the waiting copy reads it there, woken by its watch of the +// state. A copy removes its key when it ends; one a stopped process left goes with the state's time to live. +const MirrorWaitsState = "mirror-waits" + +// MirrorWaitKey is a copy's key in mirror-waits: its digest, with the colon a key cannot hold as a dash. +func MirrorWaitKey(digest string) string { return strings.Replace(digest, ":", "-", 1) } + +// CopyWait is one value of mirror-waits: the copy asked, and its outcome once heard. +type CopyWait struct { + Digest string `json:"digest"` + Module string `json:"module"` + Artifact string `json:"artifact"` + From string `json:"from"` + Node string `json:"node"` + Asked time.Time `json:"asked"` + Token string `json:"token"` + Outcome *MirrorOutcome `json:"outcome,omitempty"` +} + +// MirrorOutcome is the image seat's published or refused of a copy, as kept. +type MirrorOutcome struct { + reg.Outcome + Event string `json:"event"` + Heard time.Time `json:"heard"` + By string `json:"by,omitempty"` +} + +// WaitState is the state mirror-waits: a key's value (nil for none), a write, a removal. +type WaitState interface { + Get(key string) (json.RawMessage, error) + Put(key string, value any) (uint64, error) + Delete(key string) error +} + +// stateWaits is mirror-waits as the process keeps it on the bus. +type stateWaits struct{ s stdio.KeptState } + +func (w stateWaits) Get(key string) (json.RawMessage, error) { + e, err := w.s.Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil +} +func (w stateWaits) Put(key string, value any) (uint64, error) { return w.s.Put(key, value) } +func (w stateWaits) Delete(key string) error { return w.s.Delete(key) } + +// Outcomes hands the image seat's published and refused of an upstream copy to the copy waiting on its digest: written +// into the copy's key in Waits, and a wake for the copies of this process waiting on it. +type Outcomes struct { + // Waits is the state mirror-waits; Node names this instance in an outcome it writes; Now is the clock. + Waits WaitState + Node string + Now func() time.Time + + mu sync.Mutex + waiting map[string][]chan struct{} +} + +// Expect wakes on every change of digest's wait; stop ends it. +func (o *Outcomes) Expect(digest string) (<-chan struct{}, func()) { + ch := make(chan struct{}, 1) + o.mu.Lock() + if o.waiting == nil { + o.waiting = map[string][]chan struct{}{} + } + o.waiting[digest] = append(o.waiting[digest], ch) + o.mu.Unlock() + return ch, func() { + o.mu.Lock() + defer o.mu.Unlock() + left := o.waiting[digest][:0] + for _, c := range o.waiting[digest] { + if c != ch { + left = append(left, c) + } + } + if len(left) == 0 { + delete(o.waiting, digest) + } else { + o.waiting[digest] = left + } + } +} + +func (o *Outcomes) wake(digest string) { + o.mu.Lock() + defer o.mu.Unlock() + for _, ch := range o.waiting[digest] { + select { + case ch <- struct{}{}: + default: + } + } +} + +// read is the wait kept at key, nil for none or one this module cannot read. +func (o *Outcomes) read(key string) (*CopyWait, error) { + raw, err := o.Waits.Get(key) + if err != nil || len(raw) == 0 || string(raw) == "null" { + return nil, err + } + var w CopyWait + if json.Unmarshal(raw, &w) != nil || w.Digest == "" { + return nil, nil + } + return &w, nil +} + +// kept is the outcome kept for key's copy, when one was heard. +func (o *Outcomes) kept(key string) (MirrorOutcome, bool) { + w, err := o.read(key) + if err != nil || w == nil || w.Outcome == nil { + return MirrorOutcome{}, false + } + return *w.Outcome, true +} + +// drop removes key's wait when it is still the copy's own (token): a later copy of the same digest keeps its own. +func (o *Outcomes) drop(key, token string) { + if w, err := o.read(key); err == nil && w != nil && w.Token == token { + _ = o.Waits.Delete(key) + } +} + +// Heard takes one event of the seat (published or refused) in: an upstream copy's outcome is written into its wait in +// mirror-waits and wakes the copies of this process waiting on its digest; any other, and one no copy waits on, is +// passed. A wait the state cannot take is an error, so the event is offered again. +func (o *Outcomes) Heard(event string, body json.RawMessage) error { + var got reg.Outcome + if err := json.Unmarshal(body, &got); err != nil { + return fmt.Errorf("the image seat's %s is not an outcome: %w", event, err) + } + if got.Kind != "upstream" || got.Digest == "" || o.Waits == nil { + return nil + } + key := MirrorWaitKey(got.Digest) + w, err := o.read(key) + if err != nil { + return fmt.Errorf("reading the copy's wait %s: %w", key, err) + } + if w == nil { + return nil + } + if w.Outcome == nil { + now := time.Now + if o.Now != nil { + now = o.Now + } + w.Outcome = &MirrorOutcome{Outcome: got, Event: event, Heard: now().UTC(), By: o.Node} + if _, err := o.Waits.Put(key, w); err != nil { + return fmt.Errorf("keeping the outcome of %s in %s: %w", key, MirrorWaitsState, err) + } + } + o.wake(got.Digest) + return nil +} + +// Changed takes one change of mirror-waits in, as the module's watch of the state hands it: a wait whose outcome +// another instance wrote wakes this process's copies of its digest. +func (o *Outcomes) Changed(change stdio.StateChange) error { + var w CopyWait + if len(change.Value) == 0 || json.Unmarshal(change.Value, &w) != nil || w.Outcome == nil { + return nil + } + o.wake(w.Digest) + return nil +} + +// MirrorWaitsTool reads mirror-waits. +const MirrorWaitsTool = "build_image_mirror_waits" + +// Waiting reads every copy waiting in mirror-waits, oldest first, with its outcome once heard. +func (m *Module) Waiting() ([]CopyWait, error) { + if m.ReadWaits.Keys == nil || m.ReadWaits.Get == nil { + return nil, fmt.Errorf("the state %s cannot be read here", MirrorWaitsState) + } + keys, err := m.ReadWaits.Keys() + if err != nil { + return nil, err + } + raws, err := readValues(keys, m.ReadWaits.Get) + if err != nil { + return nil, err + } + out := []CopyWait{} + for _, raw := range raws { + var w CopyWait + if len(raw) == 0 || json.Unmarshal(raw, &w) != nil || w.Digest == "" { continue } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("upstream answered %s for %s%s", resp.Status, c.src.repository, path) - } - return resp, nil + out = append(out, w) } - return nil, fmt.Errorf("upstream refused %s%s after a token", c.src.repository, path) + sort.Slice(out, func(i, j int) bool { return out[i].Asked.Before(out[j].Asked) }) + return out, nil } -var challengeParam = regexp.MustCompile(`(\w+)="([^"]*)"`) - -func (c *copier) fetchToken(ctx context.Context, challenge string) (string, error) { - if !strings.HasPrefix(strings.ToLower(challenge), "bearer ") { - return "", fmt.Errorf("upstream asks for credentials (%q) and this copy holds none", challenge) - } - p := map[string]string{} - for _, kv := range challengeParam.FindAllStringSubmatch(challenge, -1) { - p[kv[1]] = kv[2] - } - realm := p["realm"] - if realm == "" { - return "", fmt.Errorf("upstream's challenge %q names no realm", challenge) - } - q := url.Values{"service": {p["service"]}, "scope": {"repository:" + c.src.repository + ":pull"}} - req, err := http.NewRequestWithContext(ctx, http.MethodGet, realm+"?"+q.Encode(), nil) - if err != nil { - return "", err - } - resp, err := c.client().Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - var tok struct { - Token string `json:"token"` - Access string `json:"access_token"` - } - if resp.StatusCode != http.StatusOK || json.NewDecoder(resp.Body).Decode(&tok) != nil || (tok.Token == "" && tok.Access == "") { - return "", fmt.Errorf("upstream's token service answered %s", resp.Status) - } - if tok.Token != "" { - return tok.Token, nil - } - return tok.Access, nil -} - -// copyManifest copies the manifest of digest and everything it names, children before the manifest itself. -func (c *copier) copyManifest(ctx context.Context, digest string) error { - if c.done[digest] { - return nil - } - resp, err := c.get(ctx, "/manifests/"+digest, manifestKind) - if err != nil { - return err - } - body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) - resp.Body.Close() - if err != nil { - return err - } - sum := sha256.Sum256(body) - if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { - return fmt.Errorf("upstream served %s for the manifest pinned as %s", got, digest) - } - var doc struct { - MediaType string `json:"mediaType"` - Config struct{ Digest string } `json:"config"` - Layers []struct{ Digest string } `json:"layers"` - Manifests []struct{ Digest string } `json:"manifests"` - } - if err := json.Unmarshal(body, &doc); err != nil { - return fmt.Errorf("the manifest %s is not JSON: %w", digest, err) - } - for _, child := range doc.Manifests { - if err := c.copyManifest(ctx, child.Digest); err != nil { - return err - } - } - blobs := []string{} - if doc.Config.Digest != "" { - blobs = append(blobs, doc.Config.Digest) - } - for _, l := range doc.Layers { - blobs = append(blobs, l.Digest) - } - for _, b := range blobs { - if err := c.copyBlob(ctx, b); err != nil { - return err - } - } - mediaType := resp.Header.Get("Content-Type") - if mediaType == "" { - mediaType = doc.MediaType - } - if err := c.putManifest(ctx, digest, mediaType, body); err != nil { - return err - } - c.done[digest] = true - return nil -} - -func (c *copier) copyBlob(ctx context.Context, digest string) error { - if !digestOK.MatchString(digest) { - return fmt.Errorf("%q is not a blob digest", digest) - } - if c.done[digest] { - return nil - } - head, err := http.NewRequestWithContext(ctx, http.MethodHead, c.dstBase+"/v2/"+c.dstRepo+"/blobs/"+digest, nil) - if err != nil { - return err - } - if r, err := c.client().Do(head); err != nil { - return err - } else { - r.Body.Close() - if r.StatusCode == http.StatusOK { - c.done[digest] = true - return nil - } - } - post, err := http.NewRequestWithContext(ctx, http.MethodPost, c.dstBase+"/v2/"+c.dstRepo+"/blobs/uploads/", nil) - if err != nil { - return err - } - pr, err := c.client().Do(post) - if err != nil { - return err - } - pr.Body.Close() - loc, err := pr.Location() - if pr.StatusCode != http.StatusAccepted || err != nil { - return fmt.Errorf("the store answered %s to a blob upload of %s", pr.Status, digest) - } - q := loc.Query() - q.Set("digest", digest) - loc.RawQuery = q.Encode() - blob, err := c.get(ctx, "/blobs/"+digest, "") - if err != nil { - return err - } - defer blob.Body.Close() - put, err := http.NewRequestWithContext(ctx, http.MethodPut, loc.String(), blob.Body) - if err != nil { - return err - } - put.ContentLength = blob.ContentLength - put.Header.Set("Content-Type", "application/octet-stream") - resp, err := c.client().Do(put) - if err != nil { - return err - } - resp.Body.Close() - if resp.StatusCode != http.StatusCreated { - return fmt.Errorf("the store answered %s to the blob %s", resp.Status, digest) - } - c.done[digest] = true - return nil -} - -func (c *copier) putManifest(ctx context.Context, digest, mediaType string, body []byte) error { - req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.dstBase+"/v2/"+c.dstRepo+"/manifests/"+digest, bytes.NewReader(body)) - if err != nil { - return err - } - req.Header.Set("Content-Type", mediaType) - resp, err := c.client().Do(req) - if err != nil { - return err - } - resp.Body.Close() - if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { - return fmt.Errorf("the store answered %s to the manifest %s", resp.Status, digest) - } - return nil -} - -// verify reads back what the store serves for the pinned digest: only the store can say. -func (c *copier) verify(ctx context.Context) error { - req, err := http.NewRequestWithContext(ctx, http.MethodHead, c.dstBase+"/v2/"+c.dstRepo+"/manifests/"+c.src.digest, nil) - if err != nil { - return err - } - req.Header.Set("Accept", manifestKind) - resp, err := c.client().Do(req) - if err != nil { - return err - } - resp.Body.Close() - if got := resp.Header.Get("Docker-Content-Digest"); resp.StatusCode != http.StatusOK || got != c.src.digest { - return fmt.Errorf("the store answered %s with digest %q for the pinned %s", resp.Status, got, c.src.digest) - } - return nil +func (m *Module) mirrorWaitsTool() stdio.Tool { + return stdio.Tool{Name: MirrorWaitsTool, + Description: "The upstream copies waiting on the image seat's outcome of their mirror (the module's state " + MirrorWaitsState + + "), oldest first: the digest, the module and artifact, the reference, the node whose copy asked and when, and the " + + "seat's published or refused once an instance heard it. A copy removes its wait when it ends. Read-only.", + Run: func(map[string]any) (any, error) { + w, err := m.Waiting() + return map[string]any{"waits": w}, err + }} } // Plain is a long step that needs nothing of the machine: no guard. The copy runs no buildah. diff --git a/modules/mesh-build-image/cmd/mesh-build-image/copy_test.go b/modules/mesh-build-image/cmd/mesh-build-image/copy_test.go index d546dde0..a6de6df3 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/copy_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/copy_test.go @@ -1,116 +1,15 @@ package main import ( - "crypto/sha256" - "encoding/hex" "encoding/json" - "io" - "net/http" - "net/http/httptest" "strings" - "sync" "testing" + "time" + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" "git.novox.be/novox/mesh-sdk/go/delivery" ) -func digestOf(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) } - -// memRegistry is a registry in memory: blobs and manifests by digest, per repository. -type memRegistry struct { - mu sync.Mutex - blobs map[string][]byte // repo|digest - manifests map[string][]byte // repo|digest - types map[string]string - puts []string - srv *httptest.Server -} - -func newMem(t *testing.T) *memRegistry { - r := &memRegistry{blobs: map[string][]byte{}, manifests: map[string][]byte{}, types: map[string]string{}} - r.srv = httptest.NewServer(http.HandlerFunc(r.serve)) - t.Cleanup(r.srv.Close) - return r -} - -func (r *memRegistry) addr() string { return r.srv.Listener.Addr().String() } - -func (r *memRegistry) serve(w http.ResponseWriter, req *http.Request) { - r.mu.Lock() - defer r.mu.Unlock() - path := strings.TrimPrefix(req.URL.Path, "/v2/") - if i := strings.Index(path, "/blobs/uploads/"); i >= 0 { - if req.Method == http.MethodPost { - w.Header().Set("Location", "/v2/"+path[:i]+"/blobs/uploads/u1") - w.WriteHeader(http.StatusAccepted) - return - } - body, _ := io.ReadAll(req.Body) - d := req.URL.Query().Get("digest") - if digestOf(body) != d { - w.WriteHeader(http.StatusBadRequest) - return - } - r.blobs[path[:i]+"|"+d] = body - r.puts = append(r.puts, "blob "+d) - w.WriteHeader(http.StatusCreated) - return - } - for _, kind := range []string{"blobs", "manifests"} { - if i := strings.Index(path, "/"+kind+"/"); i >= 0 { - key := path[:i] + "|" + path[i+len(kind)+2:] - store := r.blobs - if kind == "manifests" { - store = r.manifests - } - if req.Method == http.MethodPut { - body, _ := io.ReadAll(req.Body) - if digestOf(body) != path[i+len(kind)+2:] { - w.WriteHeader(http.StatusBadRequest) - return - } - r.manifests[key] = body - r.types[key] = req.Header.Get("Content-Type") - r.puts = append(r.puts, "manifest "+path[i+len(kind)+2:]) - w.WriteHeader(http.StatusCreated) - return - } - body, ok := store[key] - if !ok { - http.NotFound(w, req) - return - } - w.Header().Set("Docker-Content-Digest", path[i+len(kind)+2:]) - w.Header().Set("Content-Type", r.types[key]) - if req.Method == http.MethodHead { - return - } - w.Write(body) - return - } - } - http.NotFound(w, req) -} - -// upstreamImage puts a two-platform index (two manifests, each with a config and a layer) into r and answers the index digest. -func upstreamImage(r *memRegistry, repo string) string { - var children []map[string]string - for _, tag := range []string{"amd64", "arm64"} { - cfg, layer := []byte(`{"arch":"`+tag+`"}`), []byte("layer-"+tag) - r.blobs[repo+"|"+digestOf(cfg)], r.blobs[repo+"|"+digestOf(layer)] = cfg, layer - mf, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", - "config": map[string]string{"digest": digestOf(cfg)}, "layers": []map[string]string{{"digest": digestOf(layer)}}}) - r.manifests[repo+"|"+digestOf(mf)] = mf - r.types[repo+"|"+digestOf(mf)] = "application/vnd.oci.image.manifest.v1+json" - children = append(children, map[string]string{"digest": digestOf(mf)}) - } - idx, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json", "manifests": children}) - d := digestOf(idx) - r.manifests[repo+"|"+d] = idx - r.types[repo+"|"+d] = "application/vnd.oci.image.index.v1+json" - return d -} - func runCopy(t *testing.T, m *Module, states *keptStates, args map[string]any) (CopyResult, delivery.Step) { t.Helper() if _, err := tool(t, m, CopyTool)(args); err != nil { @@ -126,93 +25,73 @@ func runCopy(t *testing.T, m *Module, states *keptStates, args map[string]any) ( return res, st } -// ADR 0498 §4 / AC: the copy leaves the pinned digest in the store and answers it, with the whole index (every -// manifest it lists, every blob), and runs no buildah. -func TestTheCopyLeavesThePinnedDigestInTheStoreAndAnswersIt(t *testing.T) { - up, store := newMem(t), newMem(t) - digest := upstreamImage(up, "library/nats") - f := newFake(t) - states := newStates() - m := moduleWith(t, f, states) - m.HTTP = nil - reg := store.addr() - writeFile(t, m.RegistryFile, reg+"\n") - from := up.addr() + "/library/nats:2.11.17-alpine@" + digest - res, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": from}) - if st.State != delivery.StepPassed { - t.Fatalf("step = %+v", st) - } - if res.Digest != digest || res.Reference != reg+"/nats/server@"+digest || !res.Pushed || res.Artifact != "server" { - t.Errorf("result = %+v", res) - } - if _, ok := store.manifests["nats/server|"+digest]; !ok { - t.Fatal("the index is not in the store") - } - if len(store.manifests) != 3 || len(store.blobs) != 4 { - t.Errorf("store holds %d manifests and %d blobs, want 3 and 4", len(store.manifests), len(store.blobs)) - } - if len(f.lines()) != 0 { - t.Errorf("the copy ran commands: %v", f.lines()) - } - // A second copy writes nothing new but the manifests. - store.puts = nil - if _, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": from}); st.State != delivery.StepPassed { - t.Fatalf("again: %+v", st) - } - for _, p := range store.puts { - if strings.HasPrefix(p, "blob ") { - t.Errorf("a blob the store holds was sent again: %s", p) - } - } -} - -// ADR 0498 §4 / AC: an upstream without a digest fails `not pinned:`, and nothing is read from the upstream. -func TestAnUpstreamWithoutADigestFailsNotPinned(t *testing.T) { - up, store := newMem(t), newMem(t) - upstreamImage(up, "library/nats") - f := newFake(t) - states := newStates() - m := moduleWith(t, f, states) - writeFile(t, m.RegistryFile, store.addr()+"\n") - _, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": up.addr() + "/library/nats:2.11.17-alpine"}) - if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "not pinned: server") { - t.Fatalf("step = %+v", st) - } - if len(store.puts) != 0 { - t.Errorf("the store was written: %v", store.puts) - } -} - -// A manifest the upstream serves under another digest than the pin is not copied; the failure is `not stored:`. -func TestAnUpstreamServingAnotherManifestThanPinnedIsNotStored(t *testing.T) { - up, store := newMem(t), newMem(t) - d := upstreamImage(up, "library/nats") - up.manifests["library/nats|"+d] = []byte(`{"tampered":true}`) - f := newFake(t) - states := newStates() - m := moduleWith(t, f, states) - writeFile(t, m.RegistryFile, store.addr()+"\n") - _, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": up.addr() + "/library/nats@" + d}) - if st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, "not stored: server:") { - t.Fatalf("step = %+v", st) - } - if len(store.manifests) != 0 { - t.Errorf("the store holds %v", store.manifests) - } -} - -func TestAShortNameIsQualifiedAsTheHubsAndAHostIsKept(t *testing.T) { +// ADR 0498 §4, design 51 (CAT-BI1): an upstream copy is the image seat's mirror by the pinned digest; the step ends on +// the holder's outcome on the seat's events, answering its reference, and runs no command. A refusal, no outcome in the +// bound and no pin each fail the step, the last before the seat is asked. +func TestTheCopyIsTheImageSeatsMirrorAndEndsOnItsOutcome(t *testing.T) { d := "sha256:" + strings.Repeat("a", 64) - for from, want := range map[string]upstreamRef{ - "alpine@" + d: {"https://registry-1.docker.io", "library/alpine", d}, - "nats:2.11.17-alpine@" + d: {"https://registry-1.docker.io", "library/nats", d}, - "grafana/grafana:11@" + d: {"https://registry-1.docker.io", "grafana/grafana", d}, - "ghcr.io/org/app:1@" + d: {"https://ghcr.io", "org/app", d}, - "localhost:5000/x/y@" + d: {"http://localhost:5000", "x/y", d}, + from := "nats:2.11.17-alpine@" + d + for name, c := range map[string]struct { + from string + mirrored func(reg.MirrorInput) (string, reg.Outcome) + reason string + // elsewhere, watched, atOnce: who hears the outcome and when (the fake's fields of the same names). + elsewhere, watched, atOnce bool + }{ + "published": {from: from}, + "heard by another instance, handed over by the watch of mirror-waits": {from: from, elsewhere: true, watched: true}, + "heard by another instance before this copy read its wait": {from: from, elsewhere: true, atOnce: true}, + "heard by another instance that refused it": {from: from, elsewhere: true, atOnce: true, mirrored: func(in reg.MirrorInput) (string, reg.Outcome) { + return reg.EventRefused, reg.Outcome{Kind: "upstream", Digest: in.Digest, Reason: "the upstream answered 404"} + }, reason: "refused the copy: the upstream answered 404"}, + "refused": {from: from, mirrored: func(in reg.MirrorInput) (string, reg.Outcome) { + return reg.EventRefused, reg.Outcome{Kind: "upstream", Digest: in.Digest, Reason: "the upstream answered 404"} + }, reason: "not stored: server: container-image-registry refused the copy: the upstream answered 404"}, + "another digest's outcome is not this copy's": {from: from, mirrored: func(in reg.MirrorInput) (string, reg.Outcome) { + return reg.EventPublished, reg.Outcome{Kind: "upstream", Digest: "sha256:" + strings.Repeat("b", 64)} + }, reason: "said no outcome of the copy"}, + "a published image is no copy's outcome": {from: from, mirrored: func(in reg.MirrorInput) (string, reg.Outcome) { + return reg.EventPublished, reg.Outcome{Kind: "image", Digest: in.Digest} + }, reason: "said no outcome of the copy"}, + "no pin": {from: "nats:2.11.17-alpine", reason: "not pinned: server"}, } { - got, err := parseUpstream(from) - if err != nil || got != want { - t.Errorf("%s = %+v, %v; want %+v", from, got, err, want) - } + t.Run(name, func(t *testing.T) { + f := newFake(t) + f.mirrored, f.elsewhere, f.watched, f.atOnce = c.mirrored, c.elsewhere, c.watched, c.atOnce + states := newStates() + m := moduleWith(t, f, states) + m.MirrorWait = 200 * time.Millisecond + waits := m.Outcomes.Waits.(*memWaits) + m.ReadWaits = StateRead{Keys: waits.Keys, Get: waits.Get} + var asked []CopyWait + f.atAsk = func() { asked, _ = m.Waiting() } + res, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": c.from}) + if c.reason != "" { + if st.State != delivery.StepFailed || !strings.Contains(st.Reason, c.reason) { + t.Fatalf("step = %+v, want failed with %q", st, c.reason) + } + if c.reason == "not pinned: server" && len(f.mirrors) != 0 { + t.Errorf("an unpinned copy asked the seat: %v", f.mirrors) + } + return + } + if st.State != delivery.StepPassed || res.Digest != d || !res.Pushed || + res.Reference != reg.Seat+"/upstream/nats/server@"+d { + t.Fatalf("step = %+v result = %+v", st, res) + } + if len(f.mirrors) != 1 || f.mirrors[0] != (reg.MirrorInput{Name: "nats/server", Source: from, Digest: d}) { + t.Errorf("mirrors = %+v", f.mirrors) + } + if len(f.lines()) != 0 { + t.Errorf("the copy ran commands: %v", f.lines()) + } + // The read tool of mirror-waits answers the copy's wait while it waits. + if len(asked) != 1 || asked[0].Digest != d || asked[0].From != from || asked[0].Node != "ace" || asked[0].Outcome != nil { + t.Errorf("%s at the ask = %+v", MirrorWaitsTool, asked) + } + if keys, _ := waits.Keys(); len(keys) != 0 { + t.Errorf("an ended copy left its wait in mirror-waits: %v", keys) + } + }) } } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/forge_test.go b/modules/mesh-build-image/cmd/mesh-build-image/forge_test.go index 71c952ef..bfaf3fbb 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/forge_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/forge_test.go @@ -144,7 +144,7 @@ func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) { // template in the files/ archive (novox/hq ADR 0349 §2). func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) { m := readForgeManifest(t) - if want := []any{"forge-account", "rootless-image-build", "artifact-store", "node-tools"}; !reflect.DeepEqual(m.Requires, want) { + if want := []any{"forge-account", "rootless-image-build", "node-tools"}; !reflect.DeepEqual(m.Requires, want) { t.Errorf("requires = %v, want %v", m.Requires, want) } keeps := false diff --git a/modules/mesh-build-image/cmd/mesh-build-image/helpers_test.go b/modules/mesh-build-image/cmd/mesh-build-image/helpers_test.go index b8f1357c..a7571b1f 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/helpers_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/helpers_test.go @@ -2,9 +2,10 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" - "net/http" - "net/http/httptest" + "fmt" "os" "path/filepath" "strings" @@ -12,6 +13,9 @@ import ( "testing" "time" + stdio "git.novox.be/novox/mesh-sdk/go" + + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/forgekey" ) @@ -64,46 +68,186 @@ type cmdRecord struct { line string } -// fake stands in for git and buildah at the process boundary and for the registry over HTTP. +// fake stands in for git and buildah at the process boundary and for the image seat's holder at the seat's verbs. type fake struct { mu sync.Mutex calls []cmdRecord - digest string // what `buildah images` prints - pushes int - refuse int // the first n pushes answer "connection refused" + digest string // what `buildah images` prints + pushes int // layouts written failBud forgekey.Ran // what a failing buildah bud answers; a zero Status is a passing bud - // digestFile: "" writes the digest, "none" writes no file, "empty" writes an empty one. + // digestFile: "" writes the layout's digest, "none" writes no file, "other" writes another digest. digestFile string - // registry: repository:tag -> digest - registry map[string]string - server *httptest.Server + // pushed is the manifest digest of the last layout written; parts its parts' digests, the manifest last. + pushed string + parts []string + // The holder: held are the parts it holds already; refuse fails every publish; published the publishes it was + // asked, sent the parts it was sent whole; mirrors the mirrors asked, answered by mirrored (published when nil). + held map[string]bool + refuse error + published []reg.PublishInput + sent []string + mirrors []reg.MirrorInput + mirrored func(reg.MirrorInput) (string, reg.Outcome) + outcomes *Outcomes + // elsewhere: the outcome is heard by another instance of the module over the same mirror-waits, not by this + // process; watched: this process's watch of mirror-waits hands that instance's write over; atOnce: the outcome is + // said before the mirror answers. + elsewhere, watched, atOnce bool + // atAsk runs when the mirror is asked. + atAsk func() +} + +// memWaits is the state mirror-waits in memory; watch, when set, is handed every write as the state's watch hands it. +type memWaits struct { + mu sync.Mutex + m map[string]json.RawMessage + watch func(stdio.StateChange) error +} + +func (w *memWaits) Get(key string) (json.RawMessage, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.m[key], nil +} + +func (w *memWaits) Put(key string, value any) (uint64, error) { + b, err := json.Marshal(value) + if err != nil { + return 0, err + } + w.mu.Lock() + if w.m == nil { + w.m = map[string]json.RawMessage{} + } + w.m[key] = b + watch := w.watch + w.mu.Unlock() + if watch != nil { + _ = watch(stdio.StateChange{State: MirrorWaitsState, Key: key, Op: "put", Value: b}) + } + return 1, nil +} + +func (w *memWaits) Delete(key string) error { + w.mu.Lock() + defer w.mu.Unlock() + delete(w.m, key) + return nil +} + +func (w *memWaits) Keys() ([]string, error) { + w.mu.Lock() + defer w.mu.Unlock() + keys := []string{} + for k := range w.m { + keys = append(keys, k) + } + return keys, nil } const imageDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111" func newFake(t *testing.T) *fake { - f := &fake{digest: imageDigest, registry: map[string]string{}} - f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - f.mu.Lock() - defer f.mu.Unlock() - key := strings.TrimPrefix(r.URL.Path, "/v2/") - i := strings.Index(key, "/manifests/") - if r.Method != http.MethodHead || i < 0 { - http.NotFound(w, r) - return - } - d, ok := f.registry[key[:i]+":"+key[i+len("/manifests/"):]] - if !ok { - http.NotFound(w, r) - return - } - w.Header().Set("Docker-Content-Digest", d) - })) - t.Cleanup(f.server.Close) - return f + return &fake{digest: imageDigest, held: map[string]bool{}} } -func (f *fake) address() string { return f.server.Listener.Addr().String() } +// address is the registry part of the seat's reference: the seat's name. +func (f *fake) address() string { return reg.Seat } + +func sha(b []byte) string { + s := sha256.Sum256(b) + return "sha256:" + hex.EncodeToString(s[:]) +} + +// writeLayout writes an OCI layout of one config, one layer and their manifest into dir, as buildah push oci: does. +func (f *fake) writeLayout(dir, digestFile string) { + blobs := filepath.Join(dir, "blobs", "sha256") + _ = os.MkdirAll(blobs, 0o755) + put := func(b []byte) map[string]any { + d := sha(b) + _ = os.WriteFile(filepath.Join(blobs, strings.TrimPrefix(d, "sha256:")), b, 0o644) + return map[string]any{"digest": d, "size": len(b)} + } + config := put([]byte(`{"architecture":"amd64","os":"linux"}`)) + layer := put([]byte("the layer's bytes")) + config["mediaType"], layer["mediaType"] = "application/vnd.oci.image.config.v1+json", "application/vnd.oci.image.layer.v1.tar+gzip" + raw, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": config, "layers": []any{layer}}) + manifest := put(raw) + manifest["mediaType"] = "application/vnd.oci.image.manifest.v1+json" + index, _ := json.Marshal(map[string]any{"schemaVersion": 2, "manifests": []any{manifest}}) + _ = os.WriteFile(filepath.Join(dir, "index.json"), index, 0o644) + f.pushed = manifest["digest"].(string) + f.parts = []string{config["digest"].(string), layer["digest"].(string), f.pushed} + switch f.digestFile { + case "": + _ = os.WriteFile(digestFile, []byte(f.pushed), 0o644) + case "other": + _ = os.WriteFile(digestFile, []byte(imageDigest), 0o644) + } +} + +// call is the holder's side of the seat's verbs. +func (f *fake) call(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + f.mu.Lock() + defer f.mu.Unlock() + switch key { + case "seat:" + reg.Seat + "." + reg.VerbPublish: + var in reg.PublishInput + _ = json.Unmarshal(raw, &in) + f.published = append(f.published, in) + if f.refuse != nil { + return nil, f.refuse + } + a := reg.PublishAnswer{Transfer: in.Transfer} + for _, p := range in.Parts { + if !f.held[p.Digest] { + a.Lacks = append(a.Lacks, p.Digest) + } + } + a.Held = len(a.Lacks) == 0 + return json.Marshal(a) + case "seat:" + reg.Seat + "." + reg.VerbPublishPart: + var in reg.PublishPartInput + _ = json.Unmarshal(raw, &in) + if in.Last { + f.sent = append(f.sent, in.Part) + } + return json.Marshal(reg.PublishPartAnswer{Transfer: in.Transfer, Part: in.Part, Complete: in.Last}) + case "seat:" + reg.Seat + "." + reg.VerbMirror: + var in reg.MirrorInput + _ = json.Unmarshal(raw, &in) + f.mirrors = append(f.mirrors, in) + if f.atAsk != nil { + f.atAsk() + } + event, o := reg.EventPublished, reg.Outcome{Kind: "upstream", Name: "upstream/" + in.Name, Digest: in.Digest, + Holder: "distribution", Reference: reg.Seat + "/upstream/" + in.Name + "@" + in.Digest} + if f.mirrored != nil { + event, o = f.mirrored(in) + } + if event != "" && f.outcomes != nil { + hearer := f.outcomes + if f.elsewhere { + waits := f.outcomes.Waits.(*memWaits) + if f.watched { + waits.watch = f.outcomes.Changed + } + hearer = &Outcomes{Waits: waits, Node: "g14"} + } + b, _ := json.Marshal(o) + if f.atOnce { + _ = hearer.Heard(event, b) + } else { + // Said after the answer, on the seat's events, as the holder's watch says it. + go func() { _ = hearer.Heard(event, b) }() + } + } + return json.Marshal(reg.MirrorAnswer{Digest: in.Digest, Reference: reg.Seat + "/upstream/" + in.Name + "@" + in.Digest}) + } + return nil, fmt.Errorf("no verb %s", key) +} func (f *fake) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { line := name + " " + strings.Join(args, " ") @@ -124,28 +268,17 @@ func (f *fake) run(ctx context.Context, env []string, name string, args ...strin case name == "buildah" && strings.Contains(line, " push "): f.mu.Lock() defer f.mu.Unlock() - if f.refuse > 0 { - f.refuse-- - return forgekey.Ran{Status: 125, Stderr: "Error: dial tcp 127.0.0.1:5000: connect: connection refused\n"} - } f.pushes++ - var digestFile, dest string + var digestFile, dir string for i, a := range args { if a == "--digestfile" { digestFile = args[i+1] } - if strings.HasPrefix(a, "docker://") { - dest = strings.TrimPrefix(a, "docker://") + if strings.HasPrefix(a, "oci:") { + dir = strings.TrimPrefix(a, "oci:") } } - dest = strings.TrimPrefix(dest, f.address()+"/") - f.registry[dest] = f.digest - switch f.digestFile { - case "": - os.WriteFile(digestFile, []byte(f.digest), 0o644) - case "empty": - os.WriteFile(digestFile, nil, 0o644) - } + f.writeLayout(dir, digestFile) } return forgekey.Ran{} } @@ -186,14 +319,13 @@ func moduleWith(t *testing.T, f *fake, states *keptStates) *Module { } m := &Module{ Node: "ace", Account: "mesh-build-image", UID: "1234", Home: t.TempDir(), Work: t.TempDir(), Store: t.TempDir(), - Steps: states.putStep, Ctx: context.Background(), Run: f.run, HTTP: f.server.Client(), - SubUIDFile: write("subuid", "someone:100000:65536\nmesh-build-image:624288:65536\n"), - SubGIDFile: write("subgid", "mesh-build-image:624288:65536\n"), - RegistryFile: write("registry", f.address()+"\n"), - Tun: write("tun", ""), - RegistryWait: 2 * time.Second, - RegistryPause: 5 * time.Millisecond, + Steps: states.putStep, Ctx: context.Background(), Run: f.run, Call: f.call, Outcomes: &Outcomes{Waits: &memWaits{}}, + SubUIDFile: write("subuid", "someone:100000:65536\nmesh-build-image:624288:65536\n"), + SubGIDFile: write("subgid", "mesh-build-image:624288:65536\n"), + Tun: write("tun", ""), + MirrorWait: 5 * time.Second, } + f.outcomes = m.Outcomes return m } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/latepublish.go b/modules/mesh-build-image/cmd/mesh-build-image/latepublish.go new file mode 100644 index 00000000..2e5a03c8 --- /dev/null +++ b/modules/mesh-build-image/cmd/mesh-build-image/latepublish.go @@ -0,0 +1,23 @@ +package main + +// The late publish (novox/hq ADR 0617 §1, CAT-BI1): an image whose floor the builder did not know yet is built +// unnumbered and published by its digest; once the builder numbers it, its version.numbered publishes the held digest +// under that version through the image seat, sending nothing (buildstep/latepublish.go, the build modules' one). + +import ( + "context" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" +) + +// NumberedPattern is the event type this module consumes for the late publish, as its manifest's `consumes` names it. +const NumberedPattern = buildstep.NumberedPattern + +// LatePublish publishes a numbered image's held digest under its version on the image seat; a bundle or an archive is +// mesh-build-go's. +func (m *Module) LatePublish(ctx context.Context, body []byte) (bool, error) { + return buildstep.LatePublish(ctx, m.call(), body, buildstep.ImageSeat, KindImage) +} + +// hearNumbered hears the builder's version.numbered for the late publish. +func (m *Module) hearNumbered(ctx context.Context) { buildstep.HearNumbered(ctx, m.LatePublish) } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/main.go b/modules/mesh-build-image/cmd/mesh-build-image/main.go index 32f75e72..220eb6e9 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/main.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/main.go @@ -9,6 +9,7 @@ package main import ( "context" "fmt" + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" "os" "os/user" "strings" @@ -44,7 +45,9 @@ func main() { if u, err := user.Current(); err == nil { m.Home, m.Account, m.UID = u.HomeDir, u.Username, u.Uid } - m.RegistryFile = m.Home + "/registry" + waits := stdio.State(MirrorWaitsState) + m.Outcomes = &Outcomes{Waits: stateWaits{waits}, Node: m.Node} + m.ReadWaits = readOf(waits) m.Settings = m.Home + "/settings.json" m.Run = StepRunner(m.Home) // Outside a launch by the runtime — a build, a check — there is no node and no state to write. @@ -66,6 +69,14 @@ func main() { // started before booted. go m.endLeftRunning(ctx, m.stepScan(), booted, logf) } + if m.Node != "" { + // The image seat's outcomes, which a copy waits on (the manifest's consumes), and the copies' waits in + // mirror-waits, where another instance writes an outcome it heard. + go hearOutcomes(ctx, m.Outcomes) + // The builder's version.numbered, for the late publish of an image built unnumbered (ADR 0617 §1). + go m.hearNumbered(ctx) + go waits.Follow("", m.Outcomes.Changed) + } if m.Node != "" { // The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again. go func() { _ = puller.Pull(ctx) }() @@ -75,3 +86,28 @@ func main() { os.Exit(1) } } + +// OutcomeEvents are the image seat's events a copy's end is said by, as the manifest consumes them. +var OutcomeEvents = []string{reg.EventPublished, reg.EventRefused} + +// hearOutcomes subscribes to the image seat's published and refused, asking again until the runtime takes each. +func hearOutcomes(ctx context.Context, o *Outcomes) { + for _, event := range OutcomeEvents { + pattern := reg.Seat + "." + event + go func() { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.Subscribe(pattern, func(e stdio.Envelope) error { return o.Heard(event, e.Body) }) + if err == nil { + logf("listening for %s", pattern) + return + } + logf("not hearing %s yet (%v); asking again in %s", pattern, err, wait) + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + } + }() + } +} diff --git a/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go b/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go index f826d2b9..2727e2b6 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go @@ -100,20 +100,21 @@ func TestWorkAndStoreDirectoriesLieOutsideTheHome(t *testing.T) { } } -// The states are the two the module keeps, the one seat it claims is its build seat (ADR 0525 §2), the health is its own tool, which the manifest also lists, +// The states are the four the module keeps (mirror-waits a copy's wait for its outcome, CAT-BI1), the one seat it claims is its build seat (ADR 0525 §2), the health is its own tool, which the manifest also lists, // and the registry's address is bound into a file of the home that the program reads. func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *testing.T) { m := readManifest(t) states := []string{} for _, s := range m.State { states = append(states, s.Name) - // A step's key is per step and dropped after its age, so the bucket cannot fill; a pause has no age. - if (s.Name == "steps") != (s.TTLSeconds > 0) { - t.Errorf("state %s has ttl-seconds %d: only steps has an age", s.Name, s.TTLSeconds) + // A step's key is per step and dropped after its age, so the bucket cannot fill, and so is a copy's wait, which a + // stopped process leaves; a pause has no age. + if (s.Name == "steps" || s.Name == MirrorWaitsState) != (s.TTLSeconds > 0) { + t.Errorf("state %s has ttl-seconds %d: only steps and %s have an age", s.Name, s.TTLSeconds, MirrorWaitsState) } } slices.Sort(states) - if want := []string{"keys", "pauses", "steps"}; !slices.Equal(states, want) { + if want := []string{"keys", MirrorWaitsState, "pauses", "steps"}; !slices.Equal(states, want) { t.Errorf("state = %v, want %v", states, want) } if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "pause", "resume"}) { @@ -123,14 +124,6 @@ func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *tes if !strings.Contains(string(raw), `"tool": "`+HealthTool+`"`) { t.Errorf("runtime-health does not name %s", HealthTool) } - reg := resource(t, m, "registry") - if reg["path"] != "${dir:state}/registry" { - t.Errorf("registry file = %v, want ${dir:state}/registry (the program reads /registry)", reg) - } - tpl, _ := os.ReadFile(filepath.Join(moduleDir, "files", "registry")) - if strings.TrimSpace(string(tpl)) != "${seat:mesh-artifact-store:reach}" { - t.Errorf("registry template = %q", tpl) - } raw, _ = os.ReadFile(filepath.Join(moduleDir, "module.json")) var doc struct { Config map[string]struct { diff --git a/modules/mesh-build-image/cmd/mesh-build-image/module.go b/modules/mesh-build-image/cmd/mesh-build-image/module.go index 17e8494e..515efdc2 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/module.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/module.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "net/http" "sync" "time" @@ -42,17 +41,18 @@ type Module struct { SubUIDFile, SubGIDFile string // Tun is the tun device buildah's network for a RUN line (pasta) opens; TunDevice when empty. Tun string - // RegistryFile holds the registry's address, host:port, which the manifest binds from the artifact-store provision. - RegistryFile string + // Call asks a seat's verb (the image seat's publish, publish-part and mirror); the runtime's stdio.Ask when nil. + Call buildstep.SeatCall + // Outcomes are the image seat's published and refused as this process hears them, which a copy waits on, kept in the + // state mirror-waits; ReadWaits reads that state for its read tool. + Outcomes *Outcomes + ReadWaits StateRead // RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of // the SDK's fetch and merge. A test points them at a local repository (buildstep.AllowFileRemote). RemoteBase string FetchOptions []buildstep.Option - // HTTP talks to the registry; a client of its own when nil. RegistryWait and RegistryPause bound the wait for a - // registry held still (RegistryWait and RegistryFirstPause when zero). - HTTP *http.Client - RegistryWait time.Duration - RegistryPause time.Duration + // MirrorWait bounds a copy's wait for its outcome on the image seat's events (MirrorWait when zero). + MirrorWait time.Duration // Now is the clock; time.Now when nil. Now func() time.Time // Floor refuses every fetch and build while the work or the image store's filesystem has less free than it (issue diff --git a/modules/mesh-build-image/cmd/mesh-build-image/publish.go b/modules/mesh-build-image/cmd/mesh-build-image/publish.go new file mode 100644 index 00000000..6890142b --- /dev/null +++ b/modules/mesh-build-image/cmd/mesh-build-image/publish.go @@ -0,0 +1,151 @@ +package main + +// The publish of a trunk build (novox/hq design 51 *Publishing*, ADR 0611; CAT-BI1): after the build has ended, from +// the module's own process and not from a Dockerfile step, the image is written as an OCI layout in the step's +// workspace and published through the seat container-image-registry's verbs publish and publish-part, which the +// manifest `invokes`, with the build modules' one sender (buildstep.PublishFiles): its config, its layers and its +// manifest, by their digests, of which only the parts the holder lacks travel. This module holds no registry address and +// no credential; the holder checks the build with the builder and says published or refused on the seat's events. The +// step answers the seat's reference, container-image-registry//@sha256:. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" +) + +// KindImage is the kind an image is published as on its seat. +const KindImage = "image" + +// call is the seat's request: Call when a test set it, else the runtime's request over the bus (stdio.Ask). +func (m *Module) call() buildstep.SeatCall { + if m.Call != nil { + return m.Call + } + return stdio.Ask +} + +// layout is an OCI image layout one build wrote: its one manifest's digest and the files of its parts. +type layout struct { + Digest string + Files []buildstep.File +} + +type descriptor struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +// readLayout reads the layout in dir: index.json names exactly one manifest, which names its config and layers; every +// part is a blob of the layout, of the size its descriptor says. The manifest comes last, so a holder stores it after +// what it names. +func readLayout(dir string) (layout, error) { + var index struct { + Manifests []descriptor `json:"manifests"` + } + raw, err := os.ReadFile(filepath.Join(dir, "index.json")) + if err != nil { + return layout{}, fmt.Errorf("the image's layout has no index: %w", err) + } + if err := json.Unmarshal(raw, &index); err != nil || len(index.Manifests) != 1 { + return layout{}, fmt.Errorf("the image's layout names %d manifests, not one (%v)", len(index.Manifests), err) + } + top := index.Manifests[0] + blob := func(d descriptor) (buildstep.File, error) { + if !digestOK.MatchString(d.Digest) { + return buildstep.File{}, fmt.Errorf("%q is not a digest", d.Digest) + } + p := filepath.Join(dir, "blobs", "sha256", strings.TrimPrefix(d.Digest, "sha256:")) + info, err := os.Stat(p) + if err != nil { + return buildstep.File{}, fmt.Errorf("the layout lacks the blob %s: %w", d.Digest, err) + } + if info.Size() != d.Size { + return buildstep.File{}, fmt.Errorf("the blob %s holds %d bytes, not the %d its descriptor says", d.Digest, info.Size(), d.Size) + } + return buildstep.File{Digest: d.Digest, Size: d.Size, Path: p}, nil + } + manifest, err := blob(top) + if err != nil { + return layout{}, err + } + var body struct { + Config descriptor `json:"config"` + Layers []descriptor `json:"layers"` + } + raw, err = os.ReadFile(manifest.Path) + if err != nil || json.Unmarshal(raw, &body) != nil || body.Config.Digest == "" { + return layout{}, fmt.Errorf("the manifest %s names no config (%v)", top.Digest, err) + } + out := layout{Digest: top.Digest} + seen := map[string]bool{} + for _, d := range append([]descriptor{body.Config}, body.Layers...) { + if seen[d.Digest] { + continue + } + seen[d.Digest] = true + f, err := blob(d) + if err != nil { + return layout{}, err + } + out.Files = append(out.Files, f) + } + out.Files = append(out.Files, manifest) + return out, nil +} + +// publish writes local as an OCI layout and publishes it on the image seat as /, at the version the +// builder gave it (and the tag the call names): the step answers the layout's manifest digest, checked against the +// digest buildah wrote, and the seat's reference; a publish the holder does not take fails `not stored:`. +func (m *Module) publish(ctx context.Context, workspace, module, local string, res BuildResult, version, tag string) (any, error) { + dir := filepath.Join(m.Work, workspace, "layout."+res.Artifact) + digestFile := filepath.Join(m.Work, workspace, "pushed."+res.Artifact) + _ = os.RemoveAll(dir) + defer os.RemoveAll(dir) + argv := append(m.storeFlags(), "push", "--digestfile", digestFile, local, "oci:"+dir) + if r := m.Run(ctx, m.env(), "buildah", argv...); r.Status != 0 || r.Err != "" { + m.sayFailure("buildah push of "+res.Artifact+" to its layout", r) + return nil, fmt.Errorf("%s%s: writing its layout: %s", delivery.NotStored, res.Artifact, cause(r)) + } + img, err := readLayout(dir) + if err != nil { + return nil, fmt.Errorf("%s%s: %w", delivery.NotStored, res.Artifact, err) + } + // What buildah wrote is checked against the layout: a digest never goes out unchecked. + raw, err := os.ReadFile(digestFile) + if pushed := strings.TrimSpace(string(raw)); err != nil || pushed != img.Digest { + return nil, fmt.Errorf("%s%s: buildah wrote the digest %q (%v), but its layout's manifest is %s", delivery.NotStored, res.Artifact, pushed, err, img.Digest) + } + name := module + "/" + res.Artifact + q := KindImage + if version != "" { + q = "v" + version + } + in := reg.PublishInput{Name: name, Kind: KindImage, Version: version, Tag: tag, Digest: img.Digest, + Transfer: buildstep.TransferName(name, q, img.Digest)} + sent, err := buildstep.PublishFiles(ctx, m.call(), buildstep.ImageSeat, in, img.Files) + if err != nil { + return nil, fmt.Errorf("%s%s: %v", delivery.NotStored, res.Artifact, err) + } + m.say("published %s on %s: %d of its %d parts sent, the rest held", name, buildstep.ImageSeat, len(sent.Parts), len(img.Files)) + res.Digest, res.Pushed = img.Digest, true + res.Reference = catalogue.SeatReference(buildstep.ImageSeat, module, res.Artifact, img.Digest) + return res, nil +} + +func (m *Module) say(format string, a ...any) { + if m.Say != nil { + m.Say(format, a...) + } +} diff --git a/modules/mesh-build-image/cmd/mesh-build-image/publish_test.go b/modules/mesh-build-image/cmd/mesh-build-image/publish_test.go new file mode 100644 index 00000000..abe0987d --- /dev/null +++ b/modules/mesh-build-image/cmd/mesh-build-image/publish_test.go @@ -0,0 +1,171 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + reg "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// 1703d/AC (ADR 0487 §5): a candidate's build leaves the image seat unchanged: nothing is written, nothing is asked. +func TestACandidatesBuildLeavesTheRegistryUnchanged(t *testing.T) { + m, f, states := setup(t, same) + res, st := runBuild(t, m, states, nil) // no publish + if st.State != delivery.StepPassed || res.Pushed { + t.Fatalf("step = %+v result = %+v", st, res) + } + if len(f.called("push")) != 0 || len(f.published) != 0 { + t.Errorf("a candidate's build reached the seat: published %v, calls %v", f.published, f.lines()) + } +} + +// CAT-BI1 (design 51 *Publishing*): a trunk build writes its image as an OCI layout after the build has ended and +// publishes it once through container-image-registry, as / at the ask's version, its config, layers +// and manifest by digest, sending only the parts the holder lacks; the key carries the layout's manifest digest and the +// seat's reference. A holder refusing, a digest file that disagrees and a seat other than the image seat each fail. +func TestATrunkBuildPublishesThroughTheImageSeatOnlyWhatTheHolderLacks(t *testing.T) { + for name, c := range map[string]struct { + extra map[string]any + heldLayer bool + refuse error + digestFile string + reason string + }{ + "every part sent": {extra: map[string]any{"version": "1.4.0"}}, + "the held layer is not sent": {extra: map[string]any{"version": "1.4.0"}, heldLayer: true}, + "a tag of the call's own": {extra: map[string]any{"tag": "fp-abc123"}}, + "the holder refuses": {refuse: errors.New("refused: the builder did not number it"), reason: "not stored: server: "}, + "buildah wrote another digest": {digestFile: "other", reason: "not stored: server: buildah wrote the digest"}, + "buildah wrote no digest": {digestFile: "none", reason: "not stored: server: buildah wrote the digest \"\""}, + "another seat": {extra: map[string]any{"seat": "oci-artifact-registry"}, reason: "is published through container-image-registry"}, + } { + t.Run(name, func(t *testing.T) { + m, f, states := setup(t, same) + f.refuse, f.digestFile = c.refuse, c.digestFile + args := map[string]any{"publish": true} + for k, v := range c.extra { + args[k] = v + } + if c.heldLayer { + // The layout's layer is the fake's one fixed layer: held under any name. + f.held[sha([]byte("the layer's bytes"))] = true + } + res, st := runBuild(t, m, states, args) + if c.reason != "" { + if (st.State != delivery.StepFailed && st.State != delivery.StepRefused) || !strings.Contains(st.Reason, c.reason) { + t.Fatalf("step = %+v, want failed with %q", st, c.reason) + } + return + } + if st.State != delivery.StepPassed { + t.Fatalf("step = %+v", st) + } + if !res.Pushed || res.Digest != f.pushed || res.Reference != reg.Seat+"/fixture-image/server@"+f.pushed { + t.Errorf("result = %+v, want the layout's %s", res, f.pushed) + } + if len(f.published) != 1 { + t.Fatalf("published = %+v", f.published) + } + p := f.published[0] + version, _ := c.extra["version"].(string) + tag, _ := c.extra["tag"].(string) + if p.Name != "fixture-image/server" || p.Kind != "image" || p.Digest != f.pushed || p.Version != version || p.Tag != tag || p.Transfer == "" { + t.Errorf("publish = %+v", p) + } + var parts []string + for _, x := range p.Parts { + parts = append(parts, x.Digest) + } + if !slices.Equal(parts, f.parts) { + t.Errorf("parts = %v, want config, layer, manifest %v", parts, f.parts) + } + want := f.parts + if c.heldLayer { + want = []string{f.parts[0], f.parts[2]} + } + if !slices.Equal(f.sent, want) { + t.Errorf("sent = %v, want %v", f.sent, want) + } + push := f.called("push") + if len(push) != 1 || !strings.Contains(push[0].line, " oci:"+filepath.Join(m.Work, "ws1", "layout.server")) || strings.Contains(push[0].line, "docker://") { + t.Fatalf("push = %v", push) + } + for _, bad := range []string{"--creds", "--authfile", "--sign", "--tls-verify"} { + if strings.Contains(push[0].line, bad) { + t.Errorf("the push carries %s", bad) + } + } + if _, err := os.Stat(filepath.Join(m.Work, "ws1", "layout.server")); !os.IsNotExist(err) { + t.Errorf("the layout outlived the step: %v", err) + } + // The layout is written after the build has ended: bud, then images, then push. + order := []string{} + for _, l := range f.lines() { + for _, v := range []string{" bud ", " images ", " push ", " rmi "} { + if strings.Contains(l, "buildah") && strings.Contains(l, v) { + order = append(order, strings.TrimSpace(v)) + } + } + } + if strings.Join(order, ",") != "bud,images,push,rmi" { + t.Errorf("order = %v", order) + } + }) + } + m2, f2, states2 := setup(t, same) + if _, st := runBuild(t, m2, states2, map[string]any{"publish": true, "tag": "a/b:c"}); st.State != delivery.StepFailed || len(f2.called("bud")) != 0 { + t.Errorf("a tag that is not one: step = %+v", st) + } +} + +// ADR 0617 §1: the builder's version.numbered for an image publishes the held digest under its version through the image +// seat, sending nothing; a digest the holder lacks fails the event (offered again); what is not this module's passes. +func TestTheNumberedEventPublishesAHeldImageUnderItsVersion(t *testing.T) { + digest := "sha256:" + strings.Repeat("ab", 32) + numbered := func(kind, seat, digest string) []byte { + raw, err := delivery.EncodeVersionNumbered(delivery.VersionNumbered{Name: "fixture-image/server", Kind: kind, Version: "1.4.0", + Module: "fixture-image", Fingerprint: "fp", Commit: "c0", Digest: digest, Seat: seat}) + if err != nil { + t.Fatal(err) + } + return raw + } + for name, c := range map[string]struct { + body []byte + did, fail bool + }{ + "a held image is numbered": {body: numbered("image", reg.Seat, digest), did: true}, + "a digest the holder lacks fails": {body: numbered("image", reg.Seat, "sha256:"+strings.Repeat("cd", 32)), fail: true}, + "a bundle is mesh-build-go's": {body: numbered("bundle", "oci-artifact-registry", digest)}, + "an event naming no seat is a bundle's": {body: numbered("image", "", digest)}, + "no digest, nothing to publish": {body: numbered("image", reg.Seat, "")}, + } { + t.Run(name, func(t *testing.T) { + m, f, _ := setup(t, same) + f.held[digest] = true + did, err := m.LatePublish(t.Context(), c.body) + if did != c.did || (err != nil) != c.fail { + t.Fatalf("did %v, err %v; want did %v, fail %v", did, err, c.did, c.fail) + } + if len(f.sent) != 0 { + t.Errorf("a late publish sent parts: %v", f.sent) + } + if !c.did { + return + } + if len(f.published) != 1 { + t.Fatalf("published %+v", f.published) + } + p := f.published[0] + if p.Name != "fixture-image/server" || p.Kind != KindImage || p.Version != "1.4.0" || p.Digest != digest || + p.Fingerprint != "fp" || p.Commit != "c0" || len(p.Parts) != 1 || p.Parts[0].Digest != digest { + t.Errorf("published %+v", p) + } + }) + } +} diff --git a/modules/mesh-build-image/cmd/mesh-build-image/push.go b/modules/mesh-build-image/cmd/mesh-build-image/push.go deleted file mode 100644 index 75c22c50..00000000 --- a/modules/mesh-build-image/cmd/mesh-build-image/push.go +++ /dev/null @@ -1,161 +0,0 @@ -package main - -// The trunk push (novox/hq ADR 0487 §5; issue 1703d): after the build has ended, from the module's own process and not -// from a Dockerfile step, the image is pushed to the registry under a tag of its own and the digest is read back from -// the registry, which is the only one that can say what it serves for the reference (issue 1742). The push and the -// read wait out a registry held still by the store's nightly collection (issue 457). - -import ( - "context" - "errors" - "fmt" - "net/http" - "os" - "path/filepath" - "strings" - "syscall" - "time" - - "git.novox.be/novox/mesh-sdk/go/forgekey" -) - -// RegistryWait is how long a call waits for a registry that refuses connections, longer than the nightly collection -// holds it still (about a minute and a half); RegistryFirstPause is the first pause, doubling to RegistryMostPause. -const ( - RegistryWait = 5 * time.Minute - RegistryFirstPause = time.Second - RegistryMostPause = 10 * time.Second -) - -// registry is the registry's address, host:port, as the manifest bound it into RegistryFile. -func (m *Module) registry() (string, error) { - raw, err := os.ReadFile(m.RegistryFile) - if err != nil { - return "", fmt.Errorf("the registry's address cannot be read: %w", err) - } - addr := strings.TrimSpace(string(raw)) - if addr == "" || strings.Contains(addr, "${") || strings.HasPrefix(addr, ":") || strings.HasSuffix(addr, ":") { - return "", fmt.Errorf("the registry's address in %s is %q, not host:port", m.RegistryFile, addr) - } - return addr, nil -} - -func (m *Module) push(ctx context.Context, workspace, module, local string, res BuildResult, tag string) (any, error) { - addr, err := m.registry() - if err != nil { - return nil, err - } - repository := module + "/" + res.Artifact - dest := addr + "/" + repository + ":" + tag - digestFile := filepath.Join(m.Work, workspace, "pushed."+tag) - argv := append(m.storeFlags(), "push", "--tls-verify=false", "--digestfile", digestFile, local, "docker://"+dest) - var last forgekey.Ran - if err := m.waitForRegistry(ctx, addr, "buildah push "+dest, func() error { - last = m.Run(ctx, m.env(), "buildah", argv...) - if last.Status != 0 || last.Err != "" { - return errors.New(cause(last)) - } - return nil - }); err != nil { - m.sayFailure("buildah push "+dest, last) - return nil, fmt.Errorf("pushing %s: %w", dest, err) - } - var served string - if err := m.waitForRegistry(ctx, addr, "reading "+dest, func() error { - var err error - served, err = m.registryDigest(ctx, addr, repository, tag) - return err - }); err != nil { - return nil, fmt.Errorf("pushed %s and cannot read back what the registry serves for it: %w", dest, err) - } - // A push that succeeded wrote the digest it made: a file missing, unreadable or empty is a fault, and the registry's - // digest is never answered unchecked. - raw, err := os.ReadFile(digestFile) - if err != nil { - return nil, fmt.Errorf("pushed %s, but the digest buildah pushed cannot be read from %s, so the registry's %s cannot be checked: %w", dest, digestFile, served, err) - } - pushed := strings.TrimSpace(string(raw)) - if !digestOK.MatchString(pushed) { - return nil, fmt.Errorf("pushed %s, but the digest file %s holds %q, not a digest, so the registry's %s cannot be checked", dest, digestFile, pushed, served) - } - if pushed != served { - return nil, fmt.Errorf("pushed %s as %s, but the registry serves %s for it", dest, pushed, served) - } - res.Digest, res.Pushed = served, true - res.Reference = addr + "/" + repository + "@" + served - return res, nil -} - -// registryDigest is the digest the registry answers for repository:tag (Docker-Content-Digest of a HEAD). -func (m *Module) registryDigest(ctx context.Context, addr, repository, tag string) (string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodHead, "http://"+addr+"/v2/"+repository+"/manifests/"+tag, nil) - if err != nil { - return "", err - } - req.Header.Set("Accept", strings.Join([]string{ - "application/vnd.oci.image.manifest.v1+json", "application/vnd.oci.image.index.v1+json", - "application/vnd.docker.distribution.manifest.v2+json", "application/vnd.docker.distribution.manifest.list.v2+json"}, ", ")) - client := m.HTTP - if client == nil { - client = &http.Client{Timeout: time.Minute} - } - resp, err := client.Do(req) - if err != nil { - return "", err - } - resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("the registry answered %s for %s:%s", resp.Status, repository, tag) - } - d := resp.Header.Get("Docker-Content-Digest") - if !digestOK.MatchString(d) { - return "", fmt.Errorf("the registry named no digest for %s:%s (Docker-Content-Digest %q)", repository, tag, d) - } - return d, nil -} - -// refused is whether an error is a connection refused: from a dial here, or as buildah said it in its output. -func refused(err error) bool { - return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused")) -} - -// waitForRegistry runs try, and while it fails because the registry at addr refuses connections, tries again with a -// growing pause until RegistryWait has passed (novox/hq issue 457). A refusal is the one error waited for: nothing -// was sent, so trying again does nothing twice. what names the call, for what the module says. -func (m *Module) waitForRegistry(ctx context.Context, addr, what string, try func() error) error { - err := try() - if !refused(err) { - return err - } - limit, pause := m.RegistryWait, m.RegistryPause - if limit == 0 { - limit = RegistryWait - } - if pause == 0 { - pause = RegistryFirstPause - } - started := time.Now() - m.say("%s: refused; waiting for the registry at %s for up to %s — the store's nightly collection holds it still for about a minute and a half (issue 457)", what, addr, limit) - for { - left := limit - time.Since(started) - if left <= 0 { - return fmt.Errorf("the registry at %s refused every connection for %s, longer than its nightly collection holds it still, so it is down, not paused: %w", addr, time.Since(started).Round(time.Millisecond), err) - } - select { - case <-ctx.Done(): - return fmt.Errorf("stopped while waiting for the registry at %s: %w (last: %v)", addr, ctx.Err(), err) - case <-time.After(min(pause, left)): - } - pause = min(pause*2, RegistryMostPause) - if err = try(); !refused(err) { - m.say("%s: the registry at %s answers again after %s", what, addr, time.Since(started).Round(time.Millisecond)) - return err - } - } -} - -func (m *Module) say(format string, a ...any) { - if m.Say != nil { - m.Say(format, a...) - } -} diff --git a/modules/mesh-build-image/cmd/mesh-build-image/push_test.go b/modules/mesh-build-image/cmd/mesh-build-image/push_test.go deleted file mode 100644 index fa882ec2..00000000 --- a/modules/mesh-build-image/cmd/mesh-build-image/push_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package main - -import ( - "context" - "os" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/forgekey" -) - -// 1703d/AC (ADR 0487 §5): a candidate's build leaves the registry unchanged: nothing is pushed, nothing is asked of it. -func TestACandidatesBuildLeavesTheRegistryUnchanged(t *testing.T) { - m, f, states := setup(t, same) - res, st := runBuild(t, m, states, nil) // no publish - if st.State != delivery.StepPassed || res.Pushed { - t.Fatalf("step = %+v result = %+v", st, res) - } - if len(f.called("push")) != 0 || len(f.registry) != 0 || f.pushes != 0 { - t.Errorf("a candidate's build reached the registry: pushes %d, registry %v, calls %v", f.pushes, f.registry, f.lines()) - } -} - -// 1703d/AC: a trunk build adds exactly one digest, under / and a tag of its own (the digest's first 12 -// hex digits), after the build has ended, with TLS verification off (plain HTTP) and no credential; the key carries the -// digest the registry serves and the reference pinned by it. -func TestATrunkBuildPushesOnceAndAnswersTheRegistrysDigest(t *testing.T) { - m, f, states := setup(t, same) - res, st := runBuild(t, m, states, map[string]any{"publish": true}) - if st.State != delivery.StepPassed { - t.Fatalf("step = %+v", st) - } - tag := "111111111111" - if len(f.registry) != 1 || f.registry["fixture-image/server:"+tag] != imageDigest || f.pushes != 1 { - t.Fatalf("registry = %v after %d pushes, want exactly fixture-image/server:%s", f.registry, f.pushes, tag) - } - if !res.Pushed || res.Digest != imageDigest || res.Reference != f.address()+"/fixture-image/server@"+imageDigest { - t.Errorf("result = %+v", res) - } - push := f.called("push") - if len(push) != 1 || !strings.Contains(push[0].line, "--tls-verify=false") || !strings.Contains(push[0].line, "docker://"+f.address()+"/fixture-image/server:"+tag) { - t.Fatalf("push = %v", push) - } - for _, bad := range []string{"--creds", "--authfile", "--sign"} { - if strings.Contains(push[0].line, bad) { - t.Errorf("the push carries %s", bad) - } - } - // The push comes after the build has ended: bud, then images, then push. - order := []string{} - for _, l := range f.lines() { - for _, v := range []string{" bud ", " images ", " push ", " rmi "} { - if strings.Contains(l, "buildah") && strings.Contains(l, v) { - order = append(order, strings.TrimSpace(v)) - } - } - } - if strings.Join(order, ",") != "bud,images,push,rmi" { - t.Errorf("order = %v", order) - } -} - -// A tag the call gives is the one used; one that is not a tag is refused. -func TestATagOfTheCallsOwn(t *testing.T) { - m, f, states := setup(t, same) - if _, st := runBuild(t, m, states, map[string]any{"publish": true, "tag": "fp-abc123"}); st.State != delivery.StepPassed { - t.Fatalf("step = %+v", st) - } - if _, ok := f.registry["fixture-image/server:fp-abc123"]; !ok || len(f.registry) != 1 { - t.Errorf("registry = %v", f.registry) - } - m2, f2, states2 := setup(t, same) - if _, st := runBuild(t, m2, states2, map[string]any{"publish": true, "tag": "a/b:c"}); st.State != delivery.StepFailed || len(f2.called("bud")) != 0 { - t.Errorf("step = %+v", st) - } -} - -// issue 457: a registry held still (connection refused) is waited for, and the push then goes through once. -func TestAPushWaitsForARegistryHeldStill(t *testing.T) { - m, f, states := setup(t, same) - f.refuse = 2 - res, st := runBuild(t, m, states, map[string]any{"publish": true}) - if st.State != delivery.StepPassed || !res.Pushed { - t.Fatalf("step = %+v result = %+v", st, res) - } - if f.pushes != 1 || len(f.registry) != 1 || len(f.called("push")) != 3 { - t.Errorf("pushes %d, registry %v, push calls %d; want two refused and one made", f.pushes, f.registry, len(f.called("push"))) - } -} - -// A registry that stays down past the wait fails the step saying how long it was refused; nothing is claimed pushed. -func TestARegistryDownPastTheWaitFailsTheStep(t *testing.T) { - m, f, states := setup(t, same) - f.refuse = 1 << 20 - m.RegistryWait = 50 * time.Millisecond - _, st := runBuild(t, m, states, map[string]any{"publish": true}) - if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "refused every connection") { - t.Errorf("step = %+v", st) - } - if len(f.registry) != 0 { - t.Errorf("registry = %v", f.registry) - } -} - -// The digest the registry serves is the answer: when it differs from the one the push reported, the step fails. -func TestAnAnswerThatDisagreesWithThePushFails(t *testing.T) { - m, f, states := setup(t, same) - run := m.Run - m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { - r := run(ctx, env, name, args...) - if name == "buildah" && strings.Contains(strings.Join(args, " "), " push ") { - f.mu.Lock() - for k := range f.registry { - f.registry[k] = "sha256:2222222222222222222222222222222222222222222222222222222222222222" - } - f.mu.Unlock() - } - return r - } - _, st := runBuild(t, m, states, map[string]any{"publish": true}) - if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "the registry serves") { - t.Errorf("step = %+v", st) - } -} - -func TestTheRegistrysAddressMustBeBound(t *testing.T) { - for _, bad := range []string{"", "${bound:artifact-store:at}:5000", ":5000"} { - m, _, _ := setup(t, same) - writeFile(t, m.RegistryFile, bad) - if _, err := m.registry(); err == nil { - t.Errorf("%q was taken as the registry's address", bad) - } - } -} - -func writeFile(t *testing.T, path, body string) { - t.Helper() - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatal(err) - } -} - -// A push that succeeded wrote its digest file: one missing, unreadable or empty is a fault, and the registry's digest is -// never answered unchecked (review round 1). -func TestAPushWithoutAReadableDigestFileFailsTheStep(t *testing.T) { - for _, kind := range []string{"none", "empty"} { - m, f, states := setup(t, same) - f.digestFile = kind - _, st := runBuild(t, m, states, map[string]any{"publish": true}) - if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "digest") { - t.Errorf("%s: step = %+v, want failed naming the digest file", kind, st) - } - } -} diff --git a/modules/mesh-build-image/cmd/mesh-build-image/tools.go b/modules/mesh-build-image/cmd/mesh-build-image/tools.go index 811a2a9f..4ded89ef 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/tools.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/tools.go @@ -45,6 +45,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool { tools = append(tools, t) } tools = append(tools, m.ReadTools()...) + tools = append(tools, m.mirrorWaitsTool()) tools = append(tools, buildstep.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held)) tools = append(tools, m.releaseTool()) return append(tools, diff --git a/modules/mesh-build-image/files/registry b/modules/mesh-build-image/files/registry deleted file mode 100644 index 1bcadab2..00000000 --- a/modules/mesh-build-image/files/registry +++ /dev/null @@ -1 +0,0 @@ -${seat:mesh-artifact-store:reach} diff --git a/modules/mesh-build-image/go.mod b/modules/mesh-build-image/go.mod index 38b96cbe..f1b8ad11 100644 --- a/modules/mesh-build-image/go.mod +++ b/modules/mesh-build-image/go.mod @@ -10,6 +10,14 @@ require ( ) require ( + git.novox.be/novox/mesh-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/gopackageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/npmpackageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/ociartifactregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.0 // indirect diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index 54bc2fb0..ebe66b7e 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -9,7 +9,8 @@ ], "state": [ "keys", - {"name": "steps", "ttl-seconds": 172800}, "pauses" + {"name": "steps", "ttl-seconds": 172800}, "pauses", + {"name": "mirror-waits", "ttl-seconds": 172800} ], "claims": [ { @@ -42,6 +43,7 @@ "build_image_fetch", "build_image_build", "build_image_copy", + "build_image_mirror_waits", "build_image_release", "build_image_step", "build_image_steps", @@ -52,9 +54,18 @@ "requires": [ "forge-account", "rootless-image-build", - "artifact-store", "node-tools" ], + "invokes": [ + "seat:container-image-registry.publish", + "seat:container-image-registry.publish-part", + "seat:container-image-registry.mirror" + ], + "consumes": [ + "mesh-delivery-builder.version.numbered", + "container-image-registry.published", + "container-image-registry.refused" + ], "own-secrets": { "broker": "${dir:state}/broker" }, @@ -142,14 +153,6 @@ "mode": "0644", "template": "files/gitconfig" }, - { - "id": "registry", - "type": "file", - "path": "${dir:state}/registry", - "owner": "mesh-build-image", - "mode": "0644", - "template": "files/registry" - }, { "id": "git", "type": "package", diff --git a/modules/mesh-build-node/.gitignore b/modules/mesh-build-node/.gitignore index f7604aaa..ce26533a 100644 --- a/modules/mesh-build-node/.gitignore +++ b/modules/mesh-build-node/.gitignore @@ -1 +1,3 @@ -mesh-build-node +# the binary, built at the module or in its command directory; never the command directory itself +/mesh-build-node +/cmd/mesh-build-node/mesh-build-node diff --git a/modules/mesh-build-node/README.md b/modules/mesh-build-node/README.md index 9f1b1a54..c0307486 100644 --- a/modules/mesh-build-node/README.md +++ b/modules/mesh-build-node/README.md @@ -24,9 +24,9 @@ the module runs `npm ci --ignore-scripts` there once (a directory without its `. **How the registry is reached** (the choice of issue 1488): the lock holds @novox/mesh-sdk at the forge's public address, and npm 12 installs a locked tarball only from the registry it is configured with. The module therefore -requires `npm-package-registry` (read access, provided by gitea) and, like the lab module, takes its binding and sealed -secret as files in its home. The install gets a private npm config file with the `@novox` scope at the binding's -`public-root` (else scheme, address, port and npm path) and the bound account's credential, and removes it afterwards; +requires `npm-package-registry` (provided by gitea, form none: reads are anonymous) and, like the lab module, takes its +binding as a file in its home. The install gets a private npm config file with the `@novox` scope at the binding's +`public-root` (else scheme, address, port and npm path), with no credential, and removes it afterwards; every other package resolves from the public registry. This is the way today's build agent names it. The runtime tree for bundles with externals (the pruned sdk and nats today in `/app/runtime`) is not installed @@ -83,14 +83,16 @@ toolchain install: a long step answers at once with its id and ends on `.< toolchain's packages (not typescript or esbuild) copied to `stage-/node_modules`: the build agent's depth (`/app/modules/module`, `/app/node_modules`), so esbuild's `../../node_modules/...` path comments, and the bundle's digest, equal its. Copies, not links, because the mesh makes no symlink. A bundle that keeps an external gets them in its own `node_modules`. - Once the build ended, this process pushes the packed archive to the artifact store (novox/hq ADR 0498 §1 §2, mesh-sdk's - `buildstep.PushBlob`, the address in `${dir:state}/registry`) under `/` by its digest, a candidate's as a - trunk build's, and answers `reference: artifact-store:////blobs/`; a push that fails fails the - step `not stored: : `. The packed bytes never stay on the node. tsc and esbuild, like the + Once the build ended, this process publishes the packed archive through the seat `oci-artifact-registry` (design 51 + *Publishing*; buildstep's one sender, `buildstep.PublishOne`, the manifest `invokes` publish and publish-part) under + `/` by its digest, a candidate's as a trunk build's, and answers the seat's reference + `oci-artifact-registry//@`; a publish the holder refuses, or a seat nothing holds, fails the + step `not stored: : `. The packed bytes never stay on the node. tsc and esbuild, like the type-check and the test script, run shut off in bubblewrap (ADR 0448): no network, none of the account, the toolchain read-only, the npm cache beneath a throwaway layer unless the step is told `publish` (a trunk build). A `package` artifact (ADR 0498 §5) is built by its own `build` script and packed with `npm pack --ignore-scripts`, both - shut off; its digest is the packed file's. Only with `publish` is it published, by this process with the registry's - write credential (never in a workspace or a step's mounts): a version the registry lacks is published, the same - packed file already there is done without a publish, another file under that version fails `version taken: - @`; the reference `@` is answered only then. A candidate answers its digest alone. + shut off; its digest is the packed file's. A trunk build the builder gave a version packs it under that version (its + package.json's `version` set for the pack, then put back) and publishes it through `npm-package-registry`; the holder + asks the builder, answers held for the same file and refuses another under that version. The reference + `@` is answered only then. A candidate, or a trunk build not numbered, answers its digest alone. + The module holds no credential for the registry: reads are anonymous (ADR 0346) and writes go through the seat. diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks.go b/modules/mesh-build-node/cmd/mesh-build-node/asks.go index c723ecaf..dad04a1d 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks.go @@ -2,6 +2,7 @@ package main import ( "context" + "path/filepath" "time" "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" @@ -29,6 +30,7 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps { } return buildstep.AskSteps{ Fetch: byName["build_node_fetch"], Build: byName["build_node_build"], Test: byName["build_node_test"], + Compare: buildstep.CompareStep(buildstep.GeneratedRootTS, func(ws string) string { return filepath.Join(m.Work, ws, "tree") }), Toolchain: func(ctx context.Context) (delivery.ToolchainUsed, error) { installed, err := m.Probe(ctx) if err != nil { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/build.go b/modules/mesh-build-node/cmd/mesh-build-node/build.go index 4627c2c7..e6a4750d 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/build.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/build.go @@ -19,6 +19,7 @@ import ( "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" sdkstep "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -88,7 +89,9 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { return nil, err } if a.Kind == "package" { - return m.buildPackage(ctx, workspace, module, root, dir, a, publish) + version, _ := args["version"].(string) + seat, _ := args["seat"].(string) + return m.buildPackage(ctx, workspace, module, root, dir, a, publish, version, seat) } if why, err := lockProblem(dir, module); err != nil || why != "" { if err == nil { @@ -139,9 +142,9 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { if err != nil { return nil, fmt.Errorf("packing %s: %w", a.Name, err) } - // Pushed from this process once the build ended, a candidate's as a trunk build's (ADR 0498 §1 §2): the step answers - // the reference, or fails not stored. The packed bytes are never written to the node (the lead's decision L9). - if err := m.push(ctx, a.module+"/"+a.Name, digest, archive); err != nil { + // Published from this process once the build ended, a candidate's as a trunk build's (ADR 0498 §1 §2; CAT-BN3): the + // step answers the seat's reference, or fails not stored. The packed bytes are never written to the node (L9). + if err := m.publishOn(ctx, buildstep.ArtifactSeat, a.module+"/"+a.Name, "bundle", "", digest, archive); err != nil { return nil, fmt.Errorf("%s%s: %v", delivery.NotStored, a.Name, err) } files := []string{} @@ -154,7 +157,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { }) sort.Strings(files) return BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: digest, Bytes: len(archive), - Reference: buildstep.BlobReference(a.module, a.Name, digest), Files: files}, nil + Reference: catalogue.SeatReference(buildstep.ArtifactSeat, a.module, a.Name, digest), Files: files}, nil } // bundle makes every entrypoint and every launcher ONE file, with esbuild (builder.go:1971-2039): an entrypoint is diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check.go b/modules/mesh-build-node/cmd/mesh-build-node/check.go index 651688ac..f632563d 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/check.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/check.go @@ -59,6 +59,9 @@ type TestResult struct { // By is the build module and the SDK it was built with (sdkstep.By): the checker names it when a failure hides // its cause. By string `json:"by,omitempty"` + // OutOfStep are the generated TypeScript files of a contracts module out of step with the seat generator (CAT-BN2, + // buildstep.OutOfStep): the unit level fails on any, Why naming each. Absent when none is. + OutOfStep []string `json:"out-of-step,omitempty"` } // Failure is the SDK's failure contract (sdkstep.Failure, issue 1901): a failing test of the test script, read from @@ -89,6 +92,29 @@ func (m *Module) testTool() StepTool { } func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { + res, err := m.testLevel(ctx, args) + if err != nil || res.Level != "unit" { + return res, err + } + // CAT-BN2: a contracts module's generated TypeScript is regenerated and compared; a file out of step fails the level. + _, dir, err := m.moduleDirOf(res.Workspace, res.Module) + if err != nil { + return nil, err + } + if res.OutOfStep, err = buildstep.OutOfStep(dir, buildstep.GeneratedRootTS); err != nil { + return nil, err + } + if len(res.OutOfStep) > 0 { + why := "generated files out of step with the seat generator (regenerate them): " + strings.Join(res.OutOfStep, ", ") + if res.Why != "" && !res.NoTests { + why = res.Why + "; " + why + } + res.Passed, res.NoTests, res.Why = false, false, why + } + return res, nil +} + +func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResult, error) { workspace, _ := args["workspace"].(string) module, _ := args["module"].(string) level, _ := args["level"].(string) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check_test.go b/modules/mesh-build-node/cmd/mesh-build-node/check_test.go index f1329d34..54b2e533 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/check_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/check_test.go @@ -79,6 +79,29 @@ func TestAPackageWithNeitherScriptNorTypeScriptHasNoTests(t *testing.T) { if len(f.calls) != 0 { t.Errorf("%d commands ran for a package with no tests", len(f.calls)) } + // CAT-BN2: a contracts module (a package artifact naming generated-from) with no tests of its own still fails its + // unit level on a generated TypeScript file out of step with the seat generator, each named; its Go files are + // mesh-build-go's to compare. + m, _, work = newMachine(t) + workspaceWith(t, work, map[string]string{ + "module.json": `{"module":"c","build":{"artifacts":[{"name":"ts","kind":"package","language":"typescript","paths":["ts"],"generated-from":"."}]}}`, + "seats/index.json": `{"format":1,"seats":["one-store"]}`, + "seats/one-store/seat.json": `{"format":1,"seat":"one-store","scope":"mesh","grain":"seat-list","decision":"ADR 1","verbs":[{"name":"query","input":{"type":"object"}}]}`, + "go/go.mod": "module example.org/c/go\n", "go/stray.go": "package stray\n", + "ts/package.json": `{"name":"@example/c"}`, "ts/stray.ts.txt": "not written by the generator\n", + }) + out, err := m.test(context.Background(), testArgs("unit")) + r := out.(*TestResult) + stray := false + for _, p := range r.OutOfStep { + stray = stray || p == "ts/stray.ts.txt" + if !strings.HasPrefix(p, "ts/") { + t.Errorf("%s is not under the TypeScript root", p) + } + } + if err != nil || r.Passed || r.NoTests || !stray || !strings.Contains(r.Why, "ts/stray.ts.txt") { + t.Errorf("a contracts module out of step = %+v, %v", r, err) + } } // 1491/AC4: with test:integration present the integration level runs it, and reports the setup (the type-check) and diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go index ee023a45..c8d58beb 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go @@ -192,7 +192,7 @@ func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *test files["modules/m/package-lock.json"] = `{}` m, f := fetchMachine(t, files, map[string]string{"@novox/other/index.js": "o", "lodash/index.js": "l"}) m.Toolchain.Registry = func() (Registry, error) { - return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "s3cret-pw!"}, nil + return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil } var npmrc string inner := f.effect @@ -209,20 +209,17 @@ func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *test if err != nil { t.Fatal(err) } - if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/") || !strings.Contains(npmrc, ":_auth=") { + // Reads are anonymous (ADR 0346, CAT-BN3): the config names the scope's registry and no credential. + if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/") || strings.Contains(npmrc, "_auth") { t.Errorf("npm ci was given the config %q", npmrc) } - // The password holds characters a workspace's random name never does, so a name like "mvpwby…" is no false match. - if strings.Contains(strings.Join(npmCalls(f)[0].args, " "), "s3cret-pw!") { - t.Errorf("the credential is on npm's command line") - } r := out.(FetchResult) if _, err := os.Stat(filepath.Join(m.Work, r.Workspace, "tree", "modules", "m", "node_modules", "@novox", "other", "index.js")); err != nil { t.Errorf("@novox/other was not installed: %v", err) } for _, e := range mustReadDir(t, filepath.Join(m.Work, r.Workspace)) { if strings.HasPrefix(e, "npm-") { - t.Errorf("the scratch install (and its credential file) is left: %s", e) + t.Errorf("the scratch install is left: %s", e) } } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/isolate.go b/modules/mesh-build-node/cmd/mesh-build-node/isolate.go index f4fceff4..b00afcc3 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/isolate.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/isolate.go @@ -2,7 +2,7 @@ package main // The run steps shut off (novox/hq ADR 0448; issue 1309, lead decision L2 of R4e): the type-check, the package's test // script, tsc and esbuild run the change's code, so each runs through mesh-sdk's buildstep.Isolated, in bubblewrap: no -// network, none of the account (its home, its ssh key, its registry secret, its bus login, the runtime directory), the +// network, none of the account (its home, its ssh key, its bus login, the runtime directory), the // workspace read-write, the toolchain read-only, the shared npm cache read-only beneath a throwaway writable layer, or // read-write for a trunk build alone (publish). The fetch step (git, npm ci of the module's own packages) holds the key // and the network and runs none of the change's code, so it runs as before. diff --git a/modules/mesh-build-node/cmd/mesh-build-node/main.go b/modules/mesh-build-node/cmd/mesh-build-node/main.go index 8f2e338d..60dd9c8e 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/main.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/main.go @@ -36,7 +36,7 @@ func main() { } paths := Paths{Home: home, Cache: CacheDir} registry := func() (Registry, error) { - return ReadRegistry(filepath.Join(home, "package-registry.json"), filepath.Join(home, "package-registry.secret")) + return ReadRegistry(filepath.Join(home, "package-registry.json")) } m := &Module{ Node: strings.TrimSpace(os.Getenv("MESH_NODE")), @@ -55,7 +55,6 @@ func main() { Ctx: ctx, Say: logf, Bubblewrap: ProbeBubblewrap, - RegistryFile: filepath.Join(home, "registry"), } m.Floor = floorOf(m.Node) m.Settings = filepath.Join(home, "settings.json") diff --git a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go index 3da4725c..5987c4ab 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go @@ -213,37 +213,45 @@ func TestTheModuleDeclaresBubblewrapAndTheCapabilityItNeeds(t *testing.T) { } } -// 1587-S5 (ADR 0498 §2): the module requires the artifact store and is told its address in a file of its home, which no -// run step is given (a run step has no network either, ADR 0448). -func TestTheModuleRequiresTheArtifactStoreAndIsToldItsAddress(t *testing.T) { +// CAT-BN3 (design 51 *Publishing*, ADR 0346): the module publishes only through the registry seats' verbs, which it +// invokes; it requires no store, places no store's address, and holds no credential for the package registry: reads +// are anonymous (the provision's form none), so it contributes nothing to it and is sealed no secret. +func TestTheModuleInvokesTheRegistrySeatsAndHoldsNoWriteCredential(t *testing.T) { m := readManifest(t) - if !slices.Contains(m.Requires, "artifact-store") { - t.Errorf("requires %v lack artifact-store", m.Requires) + for _, seat := range []string{"oci-artifact-registry", "npm-package-registry"} { + for _, verb := range []string{"publish", "publish-part"} { + if !slices.Contains(m.Invokes, "seat:"+seat+"."+verb) { + t.Errorf("invokes %v lack the seat %s's %s", m.Invokes, seat, verb) + } + } } - // Decoding leaves the template (files/registry) unread in the resource, so its text is read here. - r := resource(t, m, "registry") - content, err := os.ReadFile(filepath.Join(moduleDir, "files", "registry")) - if err != nil || r["type"] != "file" || r["path"] != "${dir:state}/registry" || r["template"] != "files/registry" || - !strings.Contains(string(content), "${seat:mesh-artifact-store:reach}") { - t.Errorf("the registry resource is %v (template %q, %v)", r, content, err) + if slices.Contains(m.Requires, "artifact-store") { + t.Errorf("requires %v still name artifact-store", m.Requires) + } + for _, r := range m.Resources { + if r["id"] == "registry" { + t.Errorf("the store's address is still placed: %v", r) + } + } + if _, err := os.Stat(filepath.Join(moduleDir, "files", "registry")); err == nil { + t.Error("files/registry is still shipped") } -} - -// 1587-S17 (ADR 0498 §5): the module publishes a trunk package itself, so its contribution to the package registry asks -// for write access; the credential stays with the module's own process (a run step has neither the home nor a network). -func TestTheModuleContributesWriteAccessToThePackageRegistry(t *testing.T) { raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) if err != nil { t.Fatal(err) } var mj struct { - Contributes map[string]map[string]any `json:"contributes"` + Contributes map[string]any `json:"contributes"` + Secrets map[string]string `json:"secrets"` } if err := json.Unmarshal(raw, &mj); err != nil { t.Fatal(err) } - if got := mj.Contributes["npm-package-registry"]["access"]; got != "write" { - t.Errorf("npm-package-registry access = %v, want write", got) + if _, ok := mj.Contributes["npm-package-registry"]; ok { + t.Errorf("contributes %v still ask the package registry for access", mj.Contributes) + } + if _, ok := mj.Secrets["npm-package-registry"]; ok { + t.Errorf("secrets %v still seal the package registry's credential", mj.Secrets) } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/module.go b/modules/mesh-build-node/cmd/mesh-build-node/module.go index 1932be8c..92075c57 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/module.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/module.go @@ -57,10 +57,9 @@ type Module struct { Say func(format string, a ...any) // HTTP reads and publishes to the package registry; a client of its own when nil. A test serves its own. HTTP *http.Client - // RegistryFile holds the artifact store's address (host:port), placed by the manifest; Push puts a blob there - // (buildstep.PushBlob when nil; a test's fake store). - RegistryFile string - Push func(ctx context.Context, registry, repository, digest string, body []byte) error + // Call asks a registry seat's verb over the bus, the publish of a done build (publish.go): stdio.Ask when nil; a + // test's fake holder. + Call buildstep.SeatCall // Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is // unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing. Bubblewrap func(ctx context.Context) (string, error) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/package.go b/modules/mesh-build-node/cmd/mesh-build-node/package.go index 851637b5..d022b301 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/package.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/package.go @@ -98,9 +98,8 @@ func (m *Module) installDev(ctx context.Context, root, dir, rel string) error { return writeShims(filepath.Join(dir, "node_modules"), shimDir(root, rel)) } -// npmrcIn writes the npm configuration for the mesh's registry, with its credential, for one command of this process; -// the caller removes it. It lies in this process's temporary directory, never in a workspace: a run step is given the -// workspace and a /tmp of its own, so no step's mounts hold the credential (ADR 0498 §5, ADR 0500 §4). +// npmrcIn writes the npm configuration for the mesh's registry, read anonymously, for one command of this process; the +// caller removes it. It lies in this process's temporary directory, never in a workspace (ADR 0498 §5, ADR 0500 §4). func (m *Module) npmrcIn(root string) (string, error) { if m.Toolchain == nil || m.Toolchain.Registry == nil { return "", fmt.Errorf("the module has no package registry") @@ -189,7 +188,14 @@ func writeShims(nm, into string) error { // buildAndPack builds the package in dir with its own build script, if it has one, and packs it with npm pack, both // shut off (write: a trunk build's, which alone writes the shared cache). The packed file is read and removed: it never // stays on the node. -func (m *Module) buildAndPack(ctx context.Context, root, dir, rel string, write bool) (packed, error) { +func (m *Module) buildAndPack(ctx context.Context, root, dir, rel string, write bool, version string) (packed, error) { + if version != "" { + restore, err := setPackageVersion(dir, version) + if err != nil { + return packed{}, err + } + defer restore() + } p, err := readPackageJSON(dir) if err != nil { return packed{}, err @@ -281,7 +287,7 @@ func (m *Module) packGroup(ctx context.Context, root string, run forgekey.Runner if err := m.installDev(ctx, root, dir, "package"); err != nil { return packed{}, err } - return m.buildAndPack(ctx, root, dir, "package", false) + return m.buildAndPack(ctx, root, dir, "package", false, "") } // sdkLocked is what a module's lock names of the SDK: its version, its integrity and where it was resolved from; ok is @@ -412,7 +418,6 @@ func (m *Module) registryVersion(ctx context.Context, name, version string) (tar if err != nil { return "", "", false, err } - reg.authorize(req) resp, err := m.httpClient().Do(req) if err != nil { return "", "", false, fmt.Errorf("asking the registry for %s: %w", name, err) @@ -442,15 +447,10 @@ func (m *Module) registryPackage(ctx context.Context, name, version string) ([]b if err != nil || !held { return nil, held, err } - reg, err := m.Toolchain.Registry() - if err != nil { - return nil, false, err - } req, err := http.NewRequestWithContext(ctx, http.MethodGet, tarball, nil) if err != nil { return nil, false, err } - reg.authorize(req) resp, err := m.httpClient().Do(req) if err != nil { return nil, false, fmt.Errorf("downloading %s@%s: %w", name, version, err) @@ -521,63 +521,60 @@ func fetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, return buildstep.FetchCommit(ctx, run, dir, remote, commit, ref, m.FetchOptions...) } -// buildPackage builds and packs a package artifact (ADR 0498 §5): its digest is the packed file's. Only a trunk build -// (publish) publishes it, from this module's own process with the registry's write credential, which no run step is -// given: a version the registry lacks is published; the same packed file already there is done without a publish; -// another packed file under that version fails `version taken:`. The reference is @, answered only when -// the registry holds this build's file; a candidate answers its digest alone. -func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir string, a artifact, publish bool) (any, error) { +// buildPackage builds and packs a package artifact (ADR 0498 §5): its digest is the packed file's. A trunk build +// (publish) the builder gave a version packs it under that version and publishes it through npm-package-registry +// (design 51 *Publishing*, CAT-BN3), from this module's own process, which holds no write credential: the holder asks +// the builder, answers held for the same file and refuses another under that version. The reference is +// @, answered only when the holder took it; a candidate, or a trunk build not numbered, answers its +// digest alone. +func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir string, a artifact, publish bool, version, seat string) (any, error) { rel := module if rel == "" { rel = "." } - p, err := m.buildAndPack(ctx, root, dir, rel, publish) + if !publish { + version = "" + } + if seat == "" { + seat = buildstep.NpmSeat + } + if version != "" && seat != buildstep.NpmSeat { + return nil, fmt.Errorf("the package %s is published through %s, not %s: this module publishes npm packages", a.Name, buildstep.NpmSeat, seat) + } + p, err := m.buildAndPack(ctx, root, dir, rel, publish, version) if err != nil { return nil, err } res := BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: p.Digest, Bytes: len(p.tgz)} - if publish { - if res.Reference, err = m.publish(ctx, root, p); err != nil { - return nil, err - } + if version == "" { + return res, nil } + if err := m.publishOn(ctx, seat, p.Name, "package", p.Version, p.Digest, p.tgz); err != nil { + return nil, fmt.Errorf("%s%s: %v", delivery.NotStored, a.Name, err) + } + res.Reference = p.Name + "@" + p.Version return res, nil } -// publish puts a packed package into the registry unless it holds that version, and answers @. -func (m *Module) publish(ctx context.Context, root string, p packed) (string, error) { - ref := p.Name + "@" + p.Version - _, integrity, held, err := m.registryVersion(ctx, p.Name, p.Version) +// setPackageVersion writes version as dir's package.json `version` (the builder's, design 51 *The versions*), so npm +// packs the package under it; restore puts the file back as it was. +func setPackageVersion(dir, version string) (restore func(), err error) { + path := filepath.Join(dir, "package.json") + raw, err := os.ReadFile(path) if err != nil { - return "", err + return nil, fmt.Errorf("the package's package.json: %w", err) } - if held { - if integrity == p.Integrity { - return ref, nil // this very file, published before: done without a publish - } - return "", fmt.Errorf("%s%s: the registry holds it as %s, and this build packs %s", delivery.VersionTaken, ref, integrity, p.Integrity) + var doc map[string]json.RawMessage + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("the package's package.json does not parse: %w", err) } - file, err := os.CreateTemp(root, "publish-*.tgz") + doc["version"], _ = json.Marshal(version) + out, err := json.MarshalIndent(doc, "", " ") if err != nil { - return "", err + return nil, err } - defer os.Remove(file.Name()) // the packed file leaves the node once its publish answered - if _, err := file.Write(p.tgz); err != nil { - file.Close() - return "", err + if err := os.WriteFile(path, append(out, '\n'), 0o644); err != nil { + return nil, err } - if err := file.Close(); err != nil { - return "", err - } - npmrc, err := m.npmrcIn(root) - if err != nil { - return "", err - } - defer os.Remove(npmrc) - env := append(m.baseEnv(), "npm_config_cache="+m.npmCache()) - r := m.Run(ctx, env, "npm", "publish", file.Name(), "--ignore-scripts", "--userconfig", npmrc) - if why := ranFailed(r); why != "" { - return "", fmt.Errorf("publishing %s: %s", ref, why) - } - return ref, nil + return func() { _ = os.WriteFile(path, raw, 0o644) }, nil } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/publish.go b/modules/mesh-build-node/cmd/mesh-build-node/publish.go new file mode 100644 index 00000000..bb3b1cec --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/publish.go @@ -0,0 +1,29 @@ +package main + +// The publish of a done build's file (novox/hq design 51 *Publishing*, ADR 0611; CAT-BN3): from this module's own +// process, after the build step ended, through a registry seat's verbs publish and publish-part, which the manifest +// `invokes`, with the build modules' one sender (buildstep.PublishOne): a bundle's archive on oci-artifact-registry, a +// trunk package on npm-package-registry under the version the builder gave it. No build step reaches a store, and this +// module holds no write credential: the holder checks the build with the builder and says published or refused. + +import ( + "context" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" +) + +// publishOn publishes body, of digest, as name (at version, when it has one) of kind on seat: through Call when a +// test set it, else the runtime's request over the bus (stdio.Ask). +func (m *Module) publishOn(ctx context.Context, seat, name, kind, version, digest string, body []byte) error { + call := m.Call + if call == nil { + call = stdio.Ask + } + q := kind + if version != "" { + q = "v" + version + } + return buildstep.PublishOne(ctx, call, seat, name, kind, version, digest, buildstep.TransferName(name, q, digest), body) +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/publish_integration_test.go b/modules/mesh-build-node/cmd/mesh-build-node/publish_integration_test.go index c8420b88..6f7d5710 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/publish_integration_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/publish_integration_test.go @@ -4,82 +4,18 @@ package main import ( "context" - "encoding/base64" - "encoding/json" - "io" - "net/http" - "net/http/httptest" + "os" "os/exec" "path/filepath" "strings" - "sync" "testing" "git.novox.be/novox/mesh-sdk/go/forgekey" ) -// npmRegistry is a registry of the test's own speaking npm's publish protocol: a PUT of the package's document with the -// packed file attached, then GET of the document and the file; it asks for the account's credential. -type npmRegistry struct { - mu sync.Mutex - versions map[string]map[string]any - files map[string][]byte - publishes int -} - -func (r *npmRegistry) serve(t *testing.T) *httptest.Server { - var srv *httptest.Server - srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, q *http.Request) { - r.mu.Lock() - defer r.mu.Unlock() - if u, p, ok := q.BasicAuth(); !ok || u != "builder" || p != "the write credential" { - w.WriteHeader(http.StatusUnauthorized) - return - } - doc := strings.TrimPrefix(q.URL.EscapedPath(), "/npm/") - switch { - case q.Method == http.MethodPut && (doc == "@novox%2fmesh-sdk" || doc == "@novox%2Fmesh-sdk"): - var body struct { - Versions map[string]map[string]any `json:"versions"` - Attachments map[string]struct { - Data string `json:"data"` - } `json:"_attachments"` - } - raw, _ := io.ReadAll(q.Body) - if err := json.Unmarshal(raw, &body); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - for v, meta := range body.Versions { - for name, a := range body.Attachments { - data, _ := base64.StdEncoding.DecodeString(a.Data) - r.files[name] = data - meta["dist"] = map[string]any{"tarball": srv.URL + "/files/" + name, "integrity": integrityOf(data)} - } - r.versions[v] = meta - } - r.publishes++ - w.WriteHeader(http.StatusCreated) - w.Write([]byte(`{"ok":true}`)) - case q.Method == http.MethodGet && (doc == "@novox%2fmesh-sdk" || doc == "@novox%2Fmesh-sdk"): - if len(r.versions) == 0 { - http.NotFound(w, q) - return - } - json.NewEncoder(w).Encode(map[string]any{"name": "@novox/mesh-sdk", "versions": r.versions}) - case strings.HasPrefix(q.URL.Path, "/files/"): - w.Write(r.files[strings.TrimPrefix(q.URL.Path, "/files/")]) - default: - http.NotFound(w, q) - } - })) - t.Cleanup(srv.Close) - return srv -} - -// 1587-S20 (ADR 0498 §5): the real npm publishes a trunk build's package to a registry speaking npm's protocol, with -// the module's credential, once: a second trunk build of the same package finds its file there and publishes nothing; -// a candidate publishes nothing at all. +// 1587-S20 (ADR 0498 §5), CAT-BN3 (design 51 *Publishing*): the real npm packs a trunk build's package under the +// version the builder gave it, and the module publishes that packed file through npm-package-registry's verbs, with no +// credential and no npm publish; the workspace's package.json is put back. A candidate publishes nothing at all. func TestARealTrunkPackageIsPublishedOnce(t *testing.T) { needBubblewrap(t) for _, tool := range []string{"node", "npm"} { @@ -89,34 +25,39 @@ func TestARealTrunkPackageIsPublishedOnce(t *testing.T) { } m, _, work := newMachine(t) m.Run = forgekey.ExecRunner - reg := &npmRegistry{versions: map[string]map[string]any{}, files: map[string][]byte{}} - srv := reg.serve(t) - m.Toolchain.Registry = func() (Registry, error) { - return Registry{URL: srv.URL + "/npm/", Username: "builder", Password: "the write credential"}, nil - } - m.HTTP = srv.Client() + store := withStore(t, m) files := map[string]string{} for n, c := range sdkFixture { files[n] = c } workspaceWith(t, work, files) dir := filepath.Join(work, "w1", "tree") + before, _ := os.ReadFile(filepath.Join(dir, "package.json")) if err := m.installDev(context.Background(), filepath.Join(work, "w1"), dir, "."); err != nil { t.Fatal(err) } args := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib"} - if out, err := m.build(context.Background(), args); err != nil || out.(BuildResult).Reference != "" || reg.publishes != 0 { - t.Fatalf("a candidate: %+v %v, %d publishes", out, err, reg.publishes) + if out, err := m.build(context.Background(), args); err != nil || out.(BuildResult).Reference != "" || len(store.published) != 0 { + t.Fatalf("a candidate: %+v %v, %d published", out, err, len(store.published)) } - args["publish"] = true + args["publish"], args["version"], args["seat"] = true, "9.9.10", "npm-package-registry" out, err := m.build(context.Background(), args) - if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@9.9.9" || reg.publishes != 1 { - t.Fatalf("a trunk build: %+v %v, %d publishes", out, err, reg.publishes) + if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@9.9.10" || len(store.published) != 1 { + t.Fatalf("a trunk build: %+v %v, %d published", out, err, len(store.published)) } - if got := digestOf(reg.files["@novox/mesh-sdk-9.9.9.tgz"]); got != out.(BuildResult).Digest { - t.Errorf("the registry holds %s, the build packed %s (files %d)", got, out.(BuildResult).Digest, len(reg.files)) + p := store.published[0] + tgz := store.blobs[p.Name+"@"+p.Digest] + if p.Name != "@novox/mesh-sdk" || p.Version != "9.9.10" || p.Kind != "package" || digestOf(tgz) != out.(BuildResult).Digest { + t.Errorf("published %+v (%d bytes), the build packed %s", p, len(tgz), out.(BuildResult).Digest) } - if again, err := m.build(context.Background(), args); err != nil || again.(BuildResult).Reference != "@novox/mesh-sdk@9.9.9" || reg.publishes != 1 { - t.Errorf("the same package again: %+v %v, %d publishes", again, err, reg.publishes) + into := t.TempDir() + if err := unpackPackage(tgz, into); err != nil { + t.Fatal(err) + } + if raw, _ := os.ReadFile(filepath.Join(into, "package.json")); !strings.Contains(string(raw), `"version": "9.9.10"`) { + t.Errorf("the packed package.json is %s, want the builder's version", raw) + } + if after, _ := os.ReadFile(filepath.Join(dir, "package.json")); string(after) != string(before) { + t.Errorf("the workspace's package.json was not put back: %s", after) } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/publish_test.go b/modules/mesh-build-node/cmd/mesh-build-node/publish_test.go index e1ab72cb..54a0b70a 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/publish_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/publish_test.go @@ -2,9 +2,9 @@ package main import ( "context" + "errors" "os" "path/filepath" - "slices" "strings" "testing" @@ -69,26 +69,37 @@ func TestACandidatesPackageIsBuiltAndPackedAndNeverPublished(t *testing.T) { } } -// ADR 0498 §5: a trunk build (publish) publishes a version the registry lacks and answers @; the same -// packed file already there is done without a publish; another file under that version fails `version taken:`. The -// packed file is gone from the node afterwards. +// ADR 0498 §5, CAT-BN3 (design 51 *Publishing*): a trunk build the builder gave a version packs the package under that +// version and publishes it through npm-package-registry, never by npm publish nor with a credential, and answers +// @; its package.json is put back. A trunk build not numbered publishes nothing; a refusal fails it +// not stored. The packed file is gone from the node afterwards. func TestATrunkPackageIsPublishedOnceAndAVersionTakenIsRefused(t *testing.T) { - tgz := tgzOf(t, sdkPackage, "0.1.15") - args := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib", "publish": true} + tgz := tgzOf(t, sdkPackage, "0.1.16") + args := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib", "publish": true, "version": "0.1.16", "seat": "npm-package-registry"} m, f, work := packageMachine(t, tgz) - fakeRegistry(t, m, nil) - out, err := m.build(context.Background(), args) - if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@0.1.15" || len(publishes(f)) != 1 { - t.Fatalf("an absent version: %+v %v, %d publishes", out, err, len(publishes(f))) - } - if p := publishes(f)[0]; !slices.Contains(p.args, "--ignore-scripts") || !strings.HasSuffix(p.args[1], ".tgz") { - t.Errorf("published with %v", p.args) - } - // ADR 0500 §4: the registry's credential is never in a workspace, which a run step is given. - for i, a := range publishes(f)[0].args { - if a == "--userconfig" && strings.HasPrefix(publishes(f)[0].args[i+1], work) { - t.Errorf("the credential was written into the workspace: %s", publishes(f)[0].args[i+1]) + var packedAs string + inner := f.effect + f.effect = func(c call) { + if c.name == "npm" && len(c.args) > 0 && c.args[0] == "pack" { + raw, _ := os.ReadFile(filepath.Join(work, "w1", "tree", "package.json")) + packedAs = string(raw) } + inner(c) + } + store := withStore(t, m) + out, err := m.build(context.Background(), args) + if err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@0.1.16" || len(publishes(f)) != 0 { + t.Fatalf("a numbered trunk package: %+v %v, %d npm publishes", out, err, len(publishes(f))) + } + if !strings.Contains(packedAs, `"version": "0.1.16"`) { + t.Errorf("packed with package.json %s, want the builder's version", packedAs) + } + if raw, _ := os.ReadFile(filepath.Join(work, "w1", "tree", "package.json")); !strings.Contains(string(raw), `"version":"0.1.15"`) { + t.Errorf("package.json was not put back: %s", raw) + } + if len(store.published) != 1 || store.published[0].Name != "@novox/mesh-sdk" || store.published[0].Version != "0.1.16" || + store.published[0].Kind != "package" || store.seats[0] != "npm-package-registry" || store.published[0].Digest != digestOf(tgz) { + t.Errorf("published %+v through %v", store.published, store.seats) } var left []string filepath.WalkDir(work, func(p string, d os.DirEntry, err error) error { @@ -102,15 +113,15 @@ func TestATrunkPackageIsPublishedOnceAndAVersionTakenIsRefused(t *testing.T) { } m, f, _ = packageMachine(t, tgz) - fakeRegistry(t, m, map[string][]byte{"0.1.15": tgz}) - if out, err := m.build(context.Background(), args); err != nil || out.(BuildResult).Reference != "@novox/mesh-sdk@0.1.15" || len(publishes(f)) != 0 { - t.Errorf("the same file published before: %+v %v, %d publishes", out, err, len(publishes(f))) + store = withStore(t, m) + unnumbered := map[string]any{"workspace": "w1", "module": ".", "artifact": "lib", "publish": true} + if out, err := m.build(context.Background(), unnumbered); err != nil || out.(BuildResult).Reference != "" || len(store.published) != 0 || len(publishes(f)) != 0 { + t.Errorf("a trunk package not numbered: %+v %v, %d published", out, err, len(store.published)) } - m, f, _ = packageMachine(t, tgz) - fakeRegistry(t, m, map[string][]byte{"0.1.15": tgzOf(t, sdkPackage, "another")}) - _, err = m.build(context.Background(), args) - if err == nil || !strings.HasPrefix(err.Error(), delivery.VersionTaken+"@novox/mesh-sdk@0.1.15") || len(publishes(f)) != 0 { - t.Errorf("another file under the version: %v, %d publishes", err, len(publishes(f))) + m, _, _ = packageMachine(t, tgz) + withStore(t, m).fail = errors.New("refused: the builder did not number it") + if _, err = m.build(context.Background(), args); err == nil || !strings.HasPrefix(err.Error(), delivery.NotStored+"lib: ") { + t.Errorf("a refused publish: %v", err) } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/push.go b/modules/mesh-build-node/cmd/mesh-build-node/push.go deleted file mode 100644 index fb468a03..00000000 --- a/modules/mesh-build-node/cmd/mesh-build-node/push.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -// The push of a done build's archive (novox/hq ADR 0498 §1 §2; issue 1587): from this module's own process, after the -// build step ended, with mesh-sdk's one client (buildstep.PushBlob), to the artifact store whose address the manifest -// binds into RegistryFile. No build step reaches the store: a run step has no network (ADR 0448). - -import ( - "context" - "fmt" - "net/http" - "os" - "strings" - "time" - - "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" -) - -// registry is the artifact store's address, host:port, as the manifest bound it into RegistryFile. -func (m *Module) registry() (string, error) { - raw, err := os.ReadFile(m.RegistryFile) - if err != nil { - return "", fmt.Errorf("the artifact store's address cannot be read: %w", err) - } - addr := strings.TrimSpace(string(raw)) - if addr == "" || strings.Contains(addr, "${") || strings.HasPrefix(addr, ":") || strings.HasSuffix(addr, ":") { - return "", fmt.Errorf("the artifact store's address in %s is %q, not host:port", m.RegistryFile, addr) - } - return addr, nil -} - -// push puts body into the artifact store as a blob of repository (/) by its digest: Push when a test -// set it, else buildstep.PushBlob. -func (m *Module) push(ctx context.Context, repository, digest string, body []byte) error { - addr, err := m.registry() - if err != nil { - return err - } - if m.Push != nil { - return m.Push(ctx, addr, repository, digest, body) - } - return buildstep.PushBlob(ctx, &http.Client{Timeout: 10 * time.Minute}, addr, repository, digest, body) -} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go index 50cbe7e9..df343ce8 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/push_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/push_test.go @@ -3,7 +3,9 @@ package main import ( "context" "crypto/sha256" + "encoding/base64" "encoding/hex" + "encoding/json" "errors" "os" "path/filepath" @@ -11,54 +13,87 @@ import ( "sync" "testing" + registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" "git.novox.be/novox/mesh-sdk/go/delivery" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" ) -// pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back. +// pushed are the blobs the fake holder took, by digest, for the tests that read an archive back. var pushed sync.Map -// fakeStore is a store of the test's own: its address in RegistryFile, each blob kept by repository and digest; fail, -// when set, is the store's refusal. +// fakeStore is the registry seats' holder as m's publish meets it (publish.go; CAT-BN3): publish opens a transfer and +// answers the part it lacks, publish-part takes the chunks in sequence, and the last one keeps the whole part when it +// hashes to its digest. Each file is kept by name and digest; fail, when set, is the holder's refusal at the last chunk. type fakeStore struct { - mu sync.Mutex - blobs map[string][]byte // "@" - fail error + mu sync.Mutex + blobs map[string][]byte // "@" + fail error + open map[string]registry.PublishInput + chunks map[string][]byte + published []registry.PublishInput + seats []string } func withStore(t *testing.T, m *Module) *fakeStore { t.Helper() - s := &fakeStore{blobs: map[string][]byte{}} - m.RegistryFile = filepath.Join(t.TempDir(), "registry") - if err := os.WriteFile(m.RegistryFile, []byte("store.test:5000\n"), 0o644); err != nil { - t.Fatal(err) - } - m.Push = func(ctx context.Context, registry, repository, digest string, body []byte) error { + s := &fakeStore{blobs: map[string][]byte{}, open: map[string]registry.PublishInput{}, chunks: map[string][]byte{}} + m.Call = func(key string, body any) (json.RawMessage, error) { s.mu.Lock() defer s.mu.Unlock() - if s.fail != nil { - return s.fail + raw, _ := json.Marshal(body) + seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".") + if seat != buildstep.ArtifactSeat && seat != buildstep.NpmSeat { + return nil, errors.New("nats: no responders available for request") } - if sum := sha256.Sum256(body); "sha256:"+hex.EncodeToString(sum[:]) != digest || registry != "store.test:5000" { - t.Errorf("pushed %s to %s with bytes of another digest", digest, registry) + switch verb { + case "publish": + var in registry.PublishInput + _ = json.Unmarshal(raw, &in) + if (seat == buildstep.ArtifactSeat) != (in.Kind == "bundle") || in.Transfer == "" || strings.ContainsAny(in.Transfer, "/. *>") { + t.Errorf("publish on %s of kind %q, transfer %q", seat, in.Kind, in.Transfer) + } + s.open[in.Transfer] = in + s.seats = append(s.seats, seat) + return json.Marshal(registry.PublishAnswer{Transfer: in.Transfer, Lacks: []string{in.Digest}}) + case "publish-part": + var c registry.PublishPartInput + _ = json.Unmarshal(raw, &c) + data, _ := base64.StdEncoding.DecodeString(c.Data) + s.chunks[c.Transfer] = append(s.chunks[c.Transfer], data...) + if !c.Last { + return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part}) + } + in, whole := s.open[c.Transfer], s.chunks[c.Transfer] + delete(s.chunks, c.Transfer) + if s.fail != nil { + return nil, s.fail + } + if sum := sha256.Sum256(whole); "sha256:"+hex.EncodeToString(sum[:]) != in.Digest { + t.Errorf("published %s with bytes of another digest", in.Digest) + } + s.blobs[in.Name+"@"+in.Digest] = whole + s.published = append(s.published, in) + pushed.Store(in.Digest, whole) + return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part, Complete: true}) } - s.blobs[repository+"@"+digest] = append([]byte(nil), body...) - pushed.Store(digest, s.blobs[repository+"@"+digest]) - return nil + return nil, errors.New("nats: no responders available for request") } return s } -// archiveOf is the archive a build pushed, by its digest. +// archiveOf is the archive a build published, by its digest. func archiveOf(t *testing.T, r BuildResult) []byte { t.Helper() raw, ok := pushed.Load(r.Digest) if !ok { - t.Fatalf("nothing was pushed under %s", r.Digest) + t.Fatalf("nothing was published under %s", r.Digest) } return raw.([]byte) } -// 1587-S6 (ADR 0498 §1): a done bundle answers the reference of the archive it pushed, under the manifest's module. +// 1587-S6 (ADR 0498 §1; CAT-BN3): a done bundle is published through oci-artifact-registry and answers the seat's +// reference of its archive, under the manifest's module. func TestADoneBundleAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) { m, _, _ := builtMachine(t, ``) store := withStore(t, m) @@ -67,23 +102,33 @@ func TestADoneBundleAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) { t.Fatal(err) } r := out.(BuildResult) - if want := "artifact-store://m/tools/blobs/" + r.Digest; r.Reference != want { + if want := "oci-artifact-registry/m/tools@" + r.Digest; r.Reference != want { t.Errorf("reference %q, want %q", r.Reference, want) } - if store.blobs["m/tools@"+r.Digest] == nil { - t.Errorf("the store holds %d blobs, not m/tools@%s", len(store.blobs), r.Digest) + if store.blobs["m/tools@"+r.Digest] == nil || len(store.seats) != 1 || store.seats[0] != buildstep.ArtifactSeat { + t.Errorf("the holder holds %d files through %v, not m/tools@%s through %s", len(store.blobs), store.seats, r.Digest, buildstep.ArtifactSeat) } } -// 1587-S7 and S8 (ADR 0498 §2, the lead's decision L9): a push the store refuses fails the step `not stored: -// : `, and whatever the push answered, no packed file is left on the node. +// 1587-S7 and S8 (ADR 0498 §2, the lead's decision L9): a publish the holder refuses fails the step `not stored: +// : `, a seat nothing holds too, and whatever it answered, no packed file is left. func TestARefusedPushFailsNotStoredAndNoPackedFileStays(t *testing.T) { - for _, fail := range []error{nil, errors.New("store.test:5000 refused the blob: 507 Insufficient Storage")} { + for _, c := range []struct { + fail error + noSeat bool + words string + }{{nil, false, ""}, {errors.New("refused: the store did not take it: 507 Insufficient Storage"), false, "507"}, + {nil, true, "no responders"}} { m, _, work := builtMachine(t, ``) - withStore(t, m).fail = fail + withStore(t, m).fail = c.fail + if c.noSeat { + m.Call = func(string, any) (json.RawMessage, error) { + return nil, errors.New("nats: no responders available for request") + } + } _, err := m.build(context.Background(), buildArgs()) - if fail != nil && (err == nil || !strings.HasPrefix(err.Error(), delivery.NotStored+"tools: ") || !strings.Contains(err.Error(), "507")) { - t.Errorf("a refused push answered %v", err) + if c.words != "" && (err == nil || !strings.HasPrefix(err.Error(), delivery.NotStored+"tools: ") || !strings.Contains(err.Error(), c.words)) { + t.Errorf("a refused publish answered %v", err) } var archives []string filepath.WalkDir(work, func(p string, d os.DirEntry, err error) error { @@ -93,7 +138,7 @@ func TestARefusedPushFailsNotStoredAndNoPackedFileStays(t *testing.T) { return nil }) if len(archives) != 0 { - t.Errorf("push answered %v: packed files left on the node: %v", fail, archives) + t.Errorf("publish answered %v: packed files left on the node: %v", c.fail, archives) } } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/registry.go b/modules/mesh-build-node/cmd/mesh-build-node/registry.go index e7b8e859..fa5a0720 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/registry.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/registry.go @@ -3,16 +3,13 @@ package main // How the toolchain's install reaches the mesh's package registry (novox/hq ADR 0405 §5, ADR 0336; issue 1488). // The lock names @novox/mesh-sdk at the registry's public address, and npm installs a locked tarball only from the // registry it is configured with, so the install is told that registry for the @novox scope and nothing else: every -// other package resolves from the public one. This is how today's build agent names it (mesh-controller -// internal/builder/dependencies.go: the scope's registry from the npm-package-registry binding, the account and -// secret the mesh sealed), read here from the module's own binding and secret files (module.json `binds` and -// `secrets`), the same files the lab module reads. +// other package resolves from the public one. The address is read from the module's npm-package-registry binding +// (module.json `binds`). Reads are anonymous (ADR 0346; the provision's form none, CAT-BN3): this module holds no +// credential for the registry, and publishes only through the seat's verbs (publish.go). import ( - "encoding/base64" "encoding/json" "fmt" - "net/http" "os" "strings" ) @@ -22,19 +19,18 @@ const Scope = "@novox" // Registry is the mesh's package registry as the binding names it. type Registry struct { - URL, Username, Password string + URL string } -// ReadRegistry reads the npm-package-registry binding and its sealed secret. The registry is the binding's +// ReadRegistry reads the npm-package-registry binding. The registry is the binding's // `public-root` when it has one (the address the lock holds), and otherwise scheme://at:port plus `npm-path`. -func ReadRegistry(bindingPath, secretPath string) (Registry, error) { +func ReadRegistry(bindingPath string) (Registry, error) { raw, err := os.ReadFile(bindingPath) if err != nil { return Registry{}, fmt.Errorf("no package registry is bound to this module: %w", err) } var b struct { At string `json:"at"` - As string `json:"as"` Serves struct { Scheme string `json:"scheme"` Port int `json:"port"` @@ -59,31 +55,10 @@ func ReadRegistry(bindingPath, secretPath string) (Registry, error) { if !strings.HasSuffix(url, "/") { url += "/" } - secret, err := os.ReadFile(secretPath) - if err != nil { - return Registry{}, fmt.Errorf("the package registry's secret cannot be read: %w", err) - } - return Registry{URL: url, Username: b.As, Password: strings.TrimSpace(string(secret))}, nil + return Registry{URL: url}, nil } -// Npmrc is the npm configuration text for the install: the @novox scope at the registry, with the account's -// credential for it when there is one. +// Npmrc is the npm configuration text for the install: the @novox scope at the registry, read anonymously. func (r Registry) Npmrc() string { - var sb strings.Builder - fmt.Fprintf(&sb, "%s:registry=%s\n", Scope, r.URL) - if r.Username != "" && r.Password != "" { - hostPath := r.URL - if i := strings.Index(hostPath, "://"); i >= 0 { - hostPath = hostPath[i+1:] - } - fmt.Fprintf(&sb, "%s:_auth=%s\n", hostPath, base64.StdEncoding.EncodeToString([]byte(r.Username+":"+r.Password))) - } - return sb.String() -} - -// authorize gives a request to the registry the account's credential, when there is one. -func (r Registry) authorize(req *http.Request) { - if r.Username != "" && r.Password != "" { - req.SetBasicAuth(r.Username, r.Password) - } + return fmt.Sprintf("%s:registry=%s\n", Scope, r.URL) } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/store_integration_test.go b/modules/mesh-build-node/cmd/mesh-build-node/store_integration_test.go index 1ad3d262..11dd4e95 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/store_integration_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/store_integration_test.go @@ -3,14 +3,16 @@ package main import ( + "bytes" "context" "crypto/sha256" "encoding/hex" + "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" - "os" - "path/filepath" + "net/url" "strings" "testing" @@ -18,12 +20,15 @@ import ( "github.com/distribution/distribution/v3/registry/handlers" _ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" + registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) // realStore starts the artifact store's own program (the distribution registry, modules/distribution) inside this -// test, with its in-memory storage, on a loopback port, and points m at it with the real push client. It is imported -// by this integration test alone: the module's program does not carry it. +// test, with its in-memory storage, on a loopback port, and plays the artifact seat's holder storing into it: the fake +// holder (push_test.go) takes the publish, and each file it took is uploaded into the store as distribution's holder +// does. It is imported by this integration test alone: the module's program does not carry it. func realStore(t *testing.T, m *Module) string { t.Helper() config := &configuration.Configuration{Storage: configuration.Storage{"inmemory": configuration.Parameters{}}} @@ -33,14 +38,57 @@ func realStore(t *testing.T, m *Module) string { srv := httptest.NewServer(handlers.NewApp(context.Background(), config)) t.Cleanup(srv.Close) addr := strings.TrimPrefix(srv.URL, "http://") - m.RegistryFile = filepath.Join(t.TempDir(), "registry") - if err := os.WriteFile(m.RegistryFile, []byte(addr+"\n"), 0o644); err != nil { - t.Fatal(err) + s := withStore(t, m) + inner := m.Call + m.Call = func(key string, body any) (json.RawMessage, error) { + raw, err := inner(key, body) + var a registry.PublishPartAnswer + if err != nil || !strings.HasSuffix(key, ".publish-part") || json.Unmarshal(raw, &a) != nil || !a.Complete { + return raw, err + } + s.mu.Lock() + defer s.mu.Unlock() + for k, b := range s.blobs { + name, digest, _ := strings.Cut(k, "@") + if err := uploadBlob(srv.URL, name, digest, b); err != nil { + return nil, err + } + delete(s.blobs, k) + } + return raw, nil } - m.Push = nil return addr } +// uploadBlob is the store's monolithic blob upload, a POST opening it and a PUT with the digest, as the holder does. +func uploadBlob(base, name, digest string, body []byte) error { + resp, err := http.Post(base+"/v2/"+name+"/blobs/uploads/", "", nil) + if err != nil { + return err + } + resp.Body.Close() + loc, err := url.Parse(resp.Header.Get("Location")) + if err != nil || resp.StatusCode != http.StatusAccepted { + return fmt.Errorf("the store opened no upload of %s: %s %v", digest, resp.Status, err) + } + b, _ := url.Parse(base) + u := b.ResolveReference(loc) + q := u.Query() + q.Set("digest", digest) + u.RawQuery = q.Encode() + req, _ := http.NewRequest(http.MethodPut, u.String(), bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/octet-stream") + put, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + put.Body.Close() + if put.StatusCode != http.StatusCreated { + return fmt.Errorf("the store did not take %s: %s", digest, put.Status) + } + return nil +} + // 1587-S10 (ADR 0498's check): a done bundle names a reference whose tail is its digest, and that reference resolves in // a real artifact store to bytes of that digest. The build is the real tsc's and esbuild's, shut off. func TestADoneBundleResolvesInARealArtifactStore(t *testing.T) { @@ -57,11 +105,11 @@ func TestADoneBundleResolvesInARealArtifactStore(t *testing.T) { if _, err := delivery.EncodeBuilderBuild(done); err != nil { t.Errorf("the done key the builder would write is refused: %v", err) } - path, ok := strings.CutPrefix(r.Reference, "artifact-store://") - if !ok || !strings.HasSuffix(r.Reference, "/blobs/"+r.Digest) { - t.Fatalf("the reference %q", r.Reference) + if _, _, ok := catalogue.InRegistrySeat(r.Reference); !ok || !strings.HasSuffix(r.Reference, "@"+r.Digest) { + t.Fatalf("the reference %q is not in the form a registry seat answers", r.Reference) } - resp, err := http.Get("http://" + addr + "/v2/" + path) + // Read as a node-engine reads it: the seat's reference routed to the store's address. + resp, err := http.Get(catalogue.Routed(r.Reference, addr)) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/toolchain.go b/modules/mesh-build-node/cmd/mesh-build-node/toolchain.go index 76d68521..08091c42 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/toolchain.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/toolchain.go @@ -25,7 +25,7 @@ const installedMark = ".installed" // Paths are the module's places on its machine. type Paths struct { // Home is the account's home (the state directory): where the mesh writes toolchain/package.json and - // toolchain/package-lock.json (module.json resources) and the registry binding and secret. + // toolchain/package-lock.json (module.json resources) and the registry binding. Home string // Cache is the cache directory, outside the home (ADR 0466 §1). Cache string diff --git a/modules/mesh-build-node/cmd/mesh-build-node/toolchain_test.go b/modules/mesh-build-node/cmd/mesh-build-node/toolchain_test.go index ccf06d4c..8b208b60 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/toolchain_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/toolchain_test.go @@ -49,7 +49,7 @@ func toolchainIn(t *testing.T, fake *fakeNpm) Toolchain { os.WriteFile(filepath.Join(home, "toolchain", "package-lock.json"), []byte(`{"lock":1}`), 0o600) return Toolchain{Paths: Paths{Home: home, Cache: cache}, Run: fake.run, Registry: func() (Registry, error) { - return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "pw"}, nil + return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil }} } @@ -106,13 +106,10 @@ func TestTheInstallNamesTheRegistryInAConfigFileNotOnTheCommandLine(t *testing.T if err := tc.Ensure(context.Background(), "v1"); err != nil { t.Fatal(err) } - if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/\n") || - !strings.Contains(npmrc, "//git.novox.be/api/packages/novox/npm/:_auth=") { + // Reads are anonymous (ADR 0346, CAT-BN3): the scope's registry and no credential. + if npmrc != "@novox:registry=https://git.novox.be/api/packages/novox/npm/\n" { t.Errorf("npmrc = %q", npmrc) } - if strings.Contains(strings.Join(fake.calls[0].args, " "), "pw") { - t.Errorf("the credential is on the command line: %v", fake.calls[0].args) - } } // 1488/AC4: an install that fails leaves no directory taken for a toolchain, says why, and the next call tries again. @@ -158,18 +155,17 @@ func TestPrepareInstallsOnlyBehindAPassingGuard(t *testing.T) { func TestRegistryFromTheBinding(t *testing.T) { dir := t.TempDir() - b, s := filepath.Join(dir, "b.json"), filepath.Join(dir, "s") - os.WriteFile(s, []byte("secret\n"), 0o600) + b := filepath.Join(dir, "b.json") os.WriteFile(b, []byte(`{"provision":"npm-package-registry","at":"10.42.0.1","as":"mesh-build-node-ace","serves":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/"}}`), 0o600) - r, err := ReadRegistry(b, s) - if err != nil || r.URL != "http://10.42.0.1:3000/api/packages/novox/npm/" || r.Username != "mesh-build-node-ace" || r.Password != "secret" { + r, err := ReadRegistry(b) + if err != nil || r.URL != "http://10.42.0.1:3000/api/packages/novox/npm/" || r.Npmrc() != "@novox:registry=http://10.42.0.1:3000/api/packages/novox/npm/\n" { t.Fatalf("= %+v, %v", r, err) } os.WriteFile(b, []byte(`{"at":"h","as":"a","serves":{"port":1,"npm-path":"/p/","public-root":"https://git.novox.be/p"}}`), 0o600) - if r, _ := ReadRegistry(b, s); r.URL != "https://git.novox.be/p/" { + if r, _ := ReadRegistry(b); r.URL != "https://git.novox.be/p/" { t.Errorf("the public root is not preferred: %q", r.URL) } - if _, err := ReadRegistry(filepath.Join(dir, "none"), s); err == nil { + if _, err := ReadRegistry(filepath.Join(dir, "none")); err == nil { t.Errorf("no binding was accepted") } } diff --git a/modules/mesh-build-node/files/registry b/modules/mesh-build-node/files/registry deleted file mode 100644 index 1bcadab2..00000000 --- a/modules/mesh-build-node/files/registry +++ /dev/null @@ -1 +0,0 @@ -${seat:mesh-artifact-store:reach} diff --git a/modules/mesh-build-node/go.mod b/modules/mesh-build-node/go.mod index 07c3aef0..c3732c76 100644 --- a/modules/mesh-build-node/go.mod +++ b/modules/mesh-build-node/go.mod @@ -10,6 +10,14 @@ require ( ) require ( + git.novox.be/novox/mesh-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/gopackageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/npmpackageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/clients/ociartifactregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index 25f8611b..a15225b6 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -63,20 +63,17 @@ "forge-account", "npm-package-registry", "node-toolchain", - "artifact-store", "node-tools" ], - "contributes": { - "npm-package-registry": { - "access": "write" - } - }, "binds": { "npm-package-registry": "${dir:state}/package-registry.json" }, - "secrets": { - "npm-package-registry": "${dir:state}/package-registry.secret" - }, + "invokes": [ + "seat:oci-artifact-registry.publish", + "seat:oci-artifact-registry.publish-part", + "seat:npm-package-registry.publish", + "seat:npm-package-registry.publish-part" + ], "own-secrets": { "broker": "${dir:state}/broker" }, @@ -164,14 +161,6 @@ "mode": "0644", "template": "files/gitconfig" }, - { - "id": "registry", - "type": "file", - "path": "${dir:state}/registry", - "owner": "mesh-build-node", - "mode": "0644", - "template": "files/registry" - }, { "id": "toolchain-dir", "type": "archive",