From 904b8aedd43b27f9ec13c05adfbd7f4ec8f0e94d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:00:48 +0100 Subject: [PATCH] BG7, BN7: a fetch naming builds reads each package member from the artifact store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each member's build is read at /v2///blobs/ through the required provision artifact-store (bound to a file), checked against its digest and laid as the package it names: a Go module zip replaced in the temporary modfile, an npm tarball in node_modules. A mismatch or a store that does not answer fails the step naming the member. Builds: design 47 G §465, §373, §466 (lead answer A-151) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-build-go/buildstep/storedbuilds.go | 116 ++++++++++++++++++ .../mesh-build-go/cmd/mesh-build-go/builds.go | 110 +++++++++++++++++ .../cmd/mesh-build-go/builds_test.go | 75 +++++++++++ .../mesh-build-go/cmd/mesh-build-go/fetch.go | 17 ++- .../cmd/mesh-build-go/forge_test.go | 3 +- .../mesh-build-go/cmd/mesh-build-go/main.go | 1 + .../cmd/mesh-build-go/manifest_test.go | 9 +- .../mesh-build-go/cmd/mesh-build-go/module.go | 2 + modules/mesh-build-go/module.json | 2 + .../cmd/mesh-build-node/builds.go | 70 +++++++++++ .../cmd/mesh-build-node/builds_test.go | 57 +++++++++ .../cmd/mesh-build-node/fetch.go | 30 ++++- .../cmd/mesh-build-node/main.go | 1 + .../cmd/mesh-build-node/manifest_test.go | 5 +- .../cmd/mesh-build-node/module.go | 2 + modules/mesh-build-node/module.json | 2 + 16 files changed, 489 insertions(+), 13 deletions(-) create mode 100644 modules/mesh-build-go/buildstep/storedbuilds.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/builds.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/builds_test.go create mode 100644 modules/mesh-build-node/cmd/mesh-build-node/builds.go create mode 100644 modules/mesh-build-node/cmd/mesh-build-node/builds_test.go diff --git a/modules/mesh-build-go/buildstep/storedbuilds.go b/modules/mesh-build-go/buildstep/storedbuilds.go new file mode 100644 index 000000000..f10101295 --- /dev/null +++ b/modules/mesh-build-go/buildstep/storedbuilds.go @@ -0,0 +1,116 @@ +package buildstep + +// **A group run's package members, read from the artifact store** (design 47 G §465, §373): the ask's fetch names each +// member's build by its digest; the build module reads it from the artifact store, through the provision it requires +// (`artifact-store`, bound to a file holding `at` and `serves.port`), at +// `artifact-store:////blobs/`, that is GET `/v2///blobs/`, and checks +// the bytes against the digest. It only reads: nothing writes into the store but its holder (ADR 0614 §2). + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "regexp" + "strings" + + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" +) + +// MaxStoredBuild is the most bytes of one member's build the module reads. +const MaxStoredBuild = 512 << 20 + +var blobDigest = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + +// BuildsArg reads a fetch's `builds` argument (the part build-step's Fetch.builds): a list of {package, module, +// artifact, digest}, as JSON gives it or as the ask's own form. nil for none. +func BuildsArg(v any) ([]sdkstep.FetchBuild, error) { + switch v := v.(type) { + case nil: + return nil, nil + case []sdkstep.FetchBuild: + return append([]sdkstep.FetchBuild(nil), v...), check(v) + case []any: + raw, err := json.Marshal(v) + if err != nil { + return nil, err + } + var out []sdkstep.FetchBuild + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("builds is a list of {package, module, artifact, digest}: %w", err) + } + return out, check(out) + } + return nil, fmt.Errorf("builds is a list of {package, module, artifact, digest}") +} + +func check(bs []sdkstep.FetchBuild) error { + for _, b := range bs { + if b.Package == "" || b.Module == "" || b.Artifact == "" || !blobDigest.MatchString(b.Digest) { + return fmt.Errorf("builds: %+v names no package, module, artifact or sha256: digest", b) + } + } + return nil +} + +// StoreURL is the artifact store's base URL from its binding file (`at`, and `serves` scheme and port), as the Go +// seat's binding reads; an error naming the file when it is not bound. +func StoreURL(bindFile string) (string, error) { + raw, err := os.ReadFile(bindFile) + if err != nil { + return "", fmt.Errorf("the artifact store is not bound (%v)", err) + } + var b struct { + At string `json:"at"` + Serves struct { + Scheme string `json:"scheme"` + Port int `json:"port"` + } `json:"serves"` + } + if json.Unmarshal(raw, &b) != nil || b.At == "" { + return "", fmt.Errorf("the artifact store's binding %s names no address", bindFile) + } + scheme := b.Serves.Scheme + if scheme == "" { + scheme = "http" + } + if b.Serves.Port == 0 { + return fmt.Sprintf("%s://%s", scheme, b.At), nil + } + return fmt.Sprintf("%s://%s:%d", scheme, b.At, b.Serves.Port), nil +} + +// ReadStoredBuild reads one member's build from the store at base and checks its bytes against its digest; a failure +// names the member. +func ReadStoredBuild(ctx context.Context, client *http.Client, base string, b sdkstep.FetchBuild) ([]byte, error) { + member := fmt.Sprintf("the build of %s (%s %s at %s)", b.Package, b.Module, b.Artifact, b.Digest) + url := strings.TrimRight(base, "/") + "/v2/" + b.Module + "/" + b.Artifact + "/blobs/" + b.Digest + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("%s: %w", member, err) + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("%s: the artifact store did not answer: %w", member, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s: the artifact store answered %s", member, resp.Status) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, MaxStoredBuild+1)) + if err != nil { + return nil, fmt.Errorf("%s: reading it: %w", member, err) + } + if len(data) > MaxStoredBuild { + return nil, fmt.Errorf("%s: larger than %d bytes", member, MaxStoredBuild) + } + sum := sha256.Sum256(data) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != b.Digest { + return nil, fmt.Errorf("%s: the bytes read are %s, not the digest asked", member, got) + } + return data, nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/builds.go b/modules/mesh-build-go/cmd/mesh-build-go/builds.go new file mode 100644 index 000000000..23ff24ac5 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/builds.go @@ -0,0 +1,110 @@ +package main + +// **A group run's package members, laid from their builds** (design 47 G §465, §373): a fetch naming `builds` reads each +// member's build (a Go module zip, every file under @/, gopackage.go) from the artifact store by +// its digest, checks it, unpacks it beside the workspace and replaces the package by that directory in each module +// directory's temporary modfile: nothing is fetched by a version, and no version is reserved. A member that does not +// match its digest, or a store that does not answer, fails the step naming the member. + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" +) + +// layBuilds lays each of builds as the package it names in every module directory dirs (relative to tree) of the +// workspace root; a directory that vendors resolves nothing and is left. +func (m *Module) layBuilds(ctx context.Context, root string, dirs []string, builds []sdkstep.FetchBuild) error { + if len(builds) == 0 { + return nil + } + base, err := buildstep.StoreURL(m.ArtifactStoreFile) + if err != nil { + return fmt.Errorf("a group run's members are read from the artifact store: %w", err) + } + tree := filepath.Join(root, "tree") + for i, b := range builds { + data, err := buildstep.ReadStoredBuild(ctx, m.httpClient(), base, b) + if err != nil { + return err + } + where := filepath.Join(root, "builds", fmt.Sprintf("%d", i)) + if err := unzipBuild(data, where); err != nil { + return fmt.Errorf("the build of %s (%s %s at %s): %w", b.Package, b.Module, b.Artifact, b.Digest, err) + } + for _, d := range dirs { + dir := filepath.Join(tree, d) + if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil { + continue + } + if err := m.replaceInModfile(dir, b.Package, where); err != nil { + return err + } + } + if m.Say != nil { + m.Say("laid %s from its build %s (%s %s)", b.Package, b.Digest, b.Module, b.Artifact) + } + } + return nil +} + +// unzipBuild unpacks a Go module zip into dir, the first path element (@) taken off; a name that +// leaves dir is refused. +func unzipBuild(data []byte, dir string) error { + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return fmt.Errorf("it is not a module zip: %w", err) + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + var total int64 + for _, f := range zr.File { + if f.FileInfo().IsDir() { + continue + } + _, rest, ok := strings.Cut(f.Name, "@") + if !ok { + return fmt.Errorf("the entry %q is outside @/", f.Name) + } + _, rest, ok = strings.Cut(rest, "/") + clean := path.Clean("/" + rest) + if !ok || clean == "/" || strings.Contains(rest, "..") { + return fmt.Errorf("the entry %q is not a file of the module", f.Name) + } + target := filepath.Join(dir, filepath.FromSlash(clean)) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return err + } + rc, err := f.Open() + if err != nil { + return err + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + rc.Close() + return err + } + n, err := io.Copy(out, io.LimitReader(rc, buildstep.MaxStoredBuild)) + rc.Close() + if cerr := out.Close(); err == nil { + err = cerr + } + if err != nil { + return err + } + if total += n; total > buildstep.MaxStoredBuild { + return fmt.Errorf("it unpacks to more than %d bytes", buildstep.MaxStoredBuild) + } + } + return nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/builds_test.go b/modules/mesh-build-go/cmd/mesh-build-go/builds_test.go new file mode 100644 index 000000000..21656f5bd --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/builds_test.go @@ -0,0 +1,75 @@ +package main + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" +) + +// Design 47 G §465, §373 (BG7): a fetch naming builds reads each member from the artifact store by its digest, checks the +// bytes and lays them as the package named; a mismatch, or a store that does not answer, fails naming the member. +func TestAFetchLaysAGroupMembersBuildFromTheArtifactStore(t *testing.T) { + var zbuf bytes.Buffer + zw := zip.NewWriter(&zbuf) + w, _ := zw.Create("example.test/lib@v0.0.0/go.mod") + w.Write([]byte("module example.test/lib\n")) + w, _ = zw.Create("example.test/lib@v0.0.0/sub/a.go") + w.Write([]byte("package sub\n")) + zw.Close() + sum := sha256.Sum256(zbuf.Bytes()) + digest := "sha256:" + hex.EncodeToString(sum[:]) + var asked []string + store := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + asked = append(asked, r.URL.Path) + if r.URL.Path != "/v2/lib/pkg/blobs/"+digest { + http.NotFound(rw, r) + return + } + rw.Write(zbuf.Bytes()) + })) + defer store.Close() + + m := &Module{Work: t.TempDir(), Home: t.TempDir()} + root := filepath.Join(m.Work, "ws1") + if err := os.MkdirAll(filepath.Join(root, "tree"), 0o700); err != nil { + t.Fatal(err) + } + os.WriteFile(filepath.Join(root, "tree", "go.mod"), []byte("module example.test/app\n\nrequire example.test/lib v0.0.0-00010101000000-000000000000\n"), 0o600) + m.ArtifactStoreFile = filepath.Join(m.Home, "artifact-store.json") + os.WriteFile(m.ArtifactStoreFile, []byte(`{"at":"`+strings.TrimPrefix(store.URL, "http://")+`"}`), 0o600) + member := sdkstep.FetchBuild{Package: "example.test/lib", Module: "lib", Artifact: "pkg", Digest: digest} + + if err := m.layBuilds(context.Background(), root, []string{"."}, []sdkstep.FetchBuild{member}); err != nil { + t.Fatal(err) + } + if b, err := os.ReadFile(filepath.Join(root, "builds", "0", "sub", "a.go")); err != nil || string(b) != "package sub\n" { + t.Fatalf("the member was not unpacked: %q, %v", b, err) + } + mod, _ := os.ReadFile(filepath.Join(root, "pins", "root", "go.mod")) + if want := "replace example.test/lib => " + filepath.Join(root, "builds", "0"); !strings.Contains(string(mod), want) { + t.Errorf("the temporary modfile lacks %q:\n%s", want, mod) + } + // A member whose bytes are not its digest, and a store that answers nothing, fail naming the member. + bad := member + bad.Digest = "sha256:" + strings.Repeat("0", 64) + for _, c := range []sdkstep.FetchBuild{bad, {Package: "example.test/lib", Module: "lib", Artifact: "missing", Digest: digest}} { + err := m.layBuilds(context.Background(), root, []string{"."}, []sdkstep.FetchBuild{c}) + if err == nil || !strings.Contains(err.Error(), "example.test/lib") { + t.Errorf("member %+v: error %v", c, err) + } + } + store.Close() + if err := m.layBuilds(context.Background(), root, []string{"."}, []sdkstep.FetchBuild{member}); err == nil || !strings.Contains(err.Error(), "example.test/lib") { + t.Errorf("a store that does not answer: %v", err) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 8e76eb0f6..9802ff224 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -77,6 +77,11 @@ func (m *Module) fetchTool() StepTool { "dir": sdkstep.FetchRequestProperties()["dir"], "paths": sdkstep.FetchRequestProperties()["paths"], "packages": sdkstep.FetchRequestProperties()["packages"], + "builds": map[string]any{"type": "array", "description": "optional: a group run's package members, each by its build's digest, read from the artifact store and laid as the package it names (design 47 G §465)", + "items": map[string]any{"type": "object", "properties": map[string]any{ + "package": map[string]any{"type": "string"}, "module": map[string]any{"type": "string"}, + "artifact": map[string]any{"type": "string"}, "digest": map[string]any{"type": "string"}, + }, "required": []string{"package", "module", "artifact", "digest"}}}, "beside": map[string]any{"type": "array", "description": "optional sibling repositories the tests read, each placed as a plain directory (no symlink) under the workspace's beside/, which build_go_test gives the tests as MESH_CHECK_BESIDE", "items": map[string]any{"type": "object", "properties": map[string]any{ "repository": map[string]any{"type": "string", "description": "/ on the forge"}, @@ -104,6 +109,10 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err e if err != nil { return nil, err } + builds, err := buildstep.BuildsArg(args["builds"]) + if err != nil { + return nil, err + } m.CleanWorkspaces(m.now()) id := sdkstep.NewID() done := m.spaces().Hold(id) @@ -115,10 +124,10 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err e } } }() - return m.fetchInto(ctx, req, id, siblings) + return m.fetchInto(ctx, req, id, siblings, builds) } -func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id string, siblings []siblingArg) (any, error) { +func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id string, siblings []siblingArg, builds []sdkstep.FetchBuild) (any, error) { repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base var err error root := filepath.Join(m.Work, id) @@ -159,6 +168,10 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str if base != "" && res.Merge == "" { return res, nil // a conflict: the tree is at the trunk, nothing of the head is built or tested } + // A group run's package members are laid from their builds, never fetched by a version (builds.go, §465). + if err := m.layBuilds(ctx, root, dirs, builds); err != nil { + return nil, err + } // The directories download side by side, up to the bound (ADR 0536 §2); the first failure in directory order is the // step's. Resolving a pin places a repository beside the workspace, which two directories may share: one at a time. var pins sync.Mutex diff --git a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go index a3fdf7dc9..2acdce793 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go @@ -146,6 +146,7 @@ func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) { func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) { m := readForgeManifest(t) // node-tools: the module pulls its asks through the tool runner's seat pull (1911). + // artifact-store: a group run's members are read from it (design 47 G §465). // go-package-registry: the Go seat's read side, the fetch's first source (design 51, CAT-BG3, E2). // detected-steps: the state of the verb steps (design 47 G §326, §414), asked by the template state. // kept-builds: the builds kept for the artifact store's holder (keptbuilds.go), by the template objects (W12.5). @@ -155,7 +156,7 @@ func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *test findings := map[string]any{"name": "findings", "storage": "key-value", "template": "state"} findingEvents := map[string]any{"name": "finding-events", "storage": "stream", "template": "events", "options": map[string]any{ "events": []any{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, "max-age-days": float64(7), "max-mib": float64(16)}} - if want := []any{detected, kept, "forge-account", "go-toolchain", "go-package-registry", "node-tools", findings, findingEvents}; !reflect.DeepEqual(m.Requires, want) { + if want := []any{detected, kept, "artifact-store", "forge-account", "go-toolchain", "go-package-registry", "node-tools", findings, findingEvents}; !reflect.DeepEqual(m.Requires, want) { t.Errorf("requires = %v, want %v", m.Requires, want) } keeps := false diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index edbf844b3..fa75df4c1 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -63,6 +63,7 @@ func main() { } m.Run = StepRunner(m.Home) m.GoSeatFile = m.Home + "/go-package-registry.json" + m.ArtifactStoreFile = m.Home + "/artifact-store.json" m.Settings = m.Home + "/settings.json" m.ReadManifest = askManifest m.AskInput = stdio.Ask diff --git a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go index 99e7eb34e..c59075693 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go @@ -172,12 +172,13 @@ func TestTheModuleDeclaresBubblewrapAndTheCapabilityItNeeds(t *testing.T) { } // 1587-S1, CAT-BG1 (design 51 *Publishing*, ADR 0614 §2): the module publishes a done archive through the seat -// oci-artifact-registry's verbs, which it invokes (the served-verb edge orders it after the holder), and no longer -// requires the artifact store or is told its address: nothing writes into the store but its holder. +// oci-artifact-registry's verbs, which it invokes (the served-verb edge orders it after the holder), and only reads +// the artifact store (§465): nothing writes into the store but its holder. func TestTheModuleInvokesTheArtifactSeatsVerbsAndWritesNoStore(t *testing.T) { m := readManifest(t) - if slices.Contains(m.Requires, "artifact-store") { - t.Errorf("requires %v still name artifact-store", m.Requires) + // It requires the provision to read a group run's package members' builds (design 47 G §465), never to write. + if !slices.Contains(m.Requires, "artifact-store") { + t.Errorf("requires %v lack artifact-store, which a group run's members are read from", m.Requires) } for _, seat := range []string{ArtifactSeat, GoPackageSeat} { // a Go package's too (CAT-BG2) for _, verb := range []string{"publish", "publish-part"} { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 5f0f36909..e6e00fd9a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -84,6 +84,8 @@ type Module struct { // packages*): bound, the mesh's Go modules are fetched through it first, Go's public proxy second, and the forge's // git over the account's ssh last (Go's direct, kept for this milestone only); unbound, as before, directly. GoSeatFile string + // ArtifactStoreFile is the binding of the provision artifact-store, read for a group run's package members (§465). + ArtifactStoreFile string // Call asks a seat's verb over the bus: the publish of a done archive (publish.go). stdio.Ask when nil; a test's // fake holder. Call callFunc diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index fa30c8470..b4a1eb975 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -112,6 +112,7 @@ "max-mib": 1024 } }, + "artifact-store", "forge-account", "go-toolchain", "go-package-registry", @@ -137,6 +138,7 @@ } ], "binds": { + "artifact-store": "${dir:state}/artifact-store.json", "go-package-registry": "${dir:state}/go-package-registry.json" }, "own-secrets": { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/builds.go b/modules/mesh-build-node/cmd/mesh-build-node/builds.go new file mode 100644 index 000000000..0945353f8 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/builds.go @@ -0,0 +1,70 @@ +package main + +// **A group run's package members, laid from their builds** (design 47 G §465, §373): a fetch naming `builds` reads each +// member's build (the npm package's packed tarball, package.go's packed) from the artifact store by its digest, checks +// it and places it as the local package it names in each TypeScript module's node_modules: nothing is fetched by a +// version, and none is reserved. A member that does not match its digest, or a store that does not answer, fails the +// step naming the member. + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" +) + +// storedBuild is one member's build, read and checked. +type storedBuild struct { + member sdkstep.FetchBuild + tgz []byte +} + +// readBuilds reads every member of builds from the artifact store. +func (m *Module) readBuilds(ctx context.Context, builds []sdkstep.FetchBuild) ([]storedBuild, error) { + if len(builds) == 0 { + return nil, nil + } + base, err := buildstep.StoreURL(m.ArtifactStoreFile) + if err != nil { + return nil, fmt.Errorf("a group run's members are read from the artifact store: %w", err) + } + var out []storedBuild + for _, b := range builds { + data, err := buildstep.ReadStoredBuild(ctx, m.httpClient(), base, b) + if err != nil { + return nil, err + } + out = append(out, storedBuild{b, data}) + } + return out, nil +} + +// names says whether a member of the builds is the package pkg. +func namesPackage(builds []storedBuild, pkg string) bool { + for _, b := range builds { + if b.member.Package == pkg { + return true + } + } + return false +} + +// layBuilds places each member in dir's node_modules as the package it names, in place of any other. +func (m *Module) layBuilds(dir string, builds []storedBuild) error { + for _, b := range builds { + into := filepath.Join(dir, "node_modules", filepath.FromSlash(b.member.Package)) + if err := os.RemoveAll(into); err != nil { + return err + } + if err := unpackPackage(b.tgz, into); err != nil { + return fmt.Errorf("the build of %s (%s %s at %s): %w", b.member.Package, b.member.Module, b.member.Artifact, b.member.Digest, err) + } + if m.Say != nil { + m.Say("laid %s from its build %s (%s %s)", b.member.Package, b.member.Digest, b.member.Module, b.member.Artifact) + } + } + return nil +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/builds_test.go b/modules/mesh-build-node/cmd/mesh-build-node/builds_test.go new file mode 100644 index 000000000..0d63bbcc7 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/builds_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" +) + +// Design 47 G §465, §373 (BN7): a fetch naming builds reads each member from the artifact store by its digest, checks the +// bytes and lays it in node_modules as the package named; a mismatch, or a store that does not answer, fails naming it. +func TestAFetchLaysAGroupMembersBuildFromTheArtifactStore(t *testing.T) { + tgz := tgzOf(t, "@novox/lib", "0.0.0") + digest := digestOf(tgz) + store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v2/lib/pkg/blobs/"+digest { + http.NotFound(w, r) + return + } + w.Write(tgz) + })) + defer store.Close() + m := &Module{ArtifactStoreFile: filepath.Join(t.TempDir(), "artifact-store.json")} + os.WriteFile(m.ArtifactStoreFile, []byte(`{"at":"`+strings.TrimPrefix(store.URL, "http://")+`"}`), 0o600) + member := sdkstep.FetchBuild{Package: "@novox/lib", Module: "lib", Artifact: "pkg", Digest: digest} + + got, err := m.readBuilds(context.Background(), []sdkstep.FetchBuild{member}) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := m.layBuilds(dir, got); err != nil { + t.Fatal(err) + } + if b, err := os.ReadFile(filepath.Join(dir, "node_modules", "@novox", "lib", "dist", "index.js")); err != nil || !strings.Contains(string(b), "0.0.0") { + t.Fatalf("the member was not laid: %q, %v", b, err) + } + if !namesPackage(got, "@novox/lib") || namesPackage(got, "@novox/other") { + t.Error("namesPackage is wrong") + } + bad := member + bad.Digest = "sha256:" + strings.Repeat("0", 64) + for _, c := range []sdkstep.FetchBuild{bad, {Package: "@novox/lib", Module: "lib", Artifact: "missing", Digest: digest}} { + if _, err := m.readBuilds(context.Background(), []sdkstep.FetchBuild{c}); err == nil || !strings.Contains(err.Error(), "@novox/lib") { + t.Errorf("member %+v: error %v", c, err) + } + } + store.Close() + if _, err := m.readBuilds(context.Background(), []sdkstep.FetchBuild{member}); err == nil || !strings.Contains(err.Error(), "@novox/lib") { + t.Errorf("a store that does not answer: %v", err) + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go index 962076296..1b5d1049f 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go @@ -102,6 +102,11 @@ func (m *Module) fetchTool() StepTool { "package": map[string]any{"type": "object", "description": "optional: the group's tagged version of the SDK, {name, version}, placed from the registry in each TypeScript member whatever its lock names (ADR 0607 §3); or the group's package member, {repository, commit, ref} (ADR 0500 §5), packed here, publishing nothing, and served to each TypeScript member whose lock names its version, in place of the registry's", "properties": map[string]any{"name": map[string]any{"type": "string"}, "version": map[string]any{"type": "string"}, "repository": map[string]any{"type": "string"}, "commit": map[string]any{"type": "string"}, "ref": map[string]any{"type": "string"}}}, + "builds": map[string]any{"type": "array", "description": "optional: a group run's package members, each by its build's digest, read from the artifact store and laid as the package it names in each module's node_modules (design 47 G §465)", + "items": map[string]any{"type": "object", "properties": map[string]any{ + "package": map[string]any{"type": "string"}, "module": map[string]any{"type": "string"}, + "artifact": map[string]any{"type": "string"}, "digest": map[string]any{"type": "string"}, + }, "required": []string{"package", "module", "artifact", "digest"}}}, "base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"}, }, "required": []string{"repository", "commit"}, @@ -124,6 +129,10 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err e if err != nil { return nil, err } + buildsAsked, err := buildstep.BuildsArg(args["builds"]) + if err != nil { + return nil, err + } m.CleanWorkspaces(m.now()) id := sdkstep.NewID() done := m.spaces().Hold(id) @@ -135,10 +144,10 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err e } } }() - return m.fetchInto(ctx, req, group, id) + return m.fetchInto(ctx, req, group, id, buildsAsked) } -func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group *groupPackage, id string) (any, error) { +func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group *groupPackage, id string, buildsAsked []sdkstep.FetchBuild) (any, error) { repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base var err error root := filepath.Join(m.Work, id) @@ -181,6 +190,11 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group } served, res.Group = &p, p.Name+"@"+p.Version+" "+p.Integrity } + // A group run's package members, each read from the artifact store by its build's digest (builds.go, §465). + members, err := m.readBuilds(ctx, buildsAsked) + if err != nil { + return nil, err + } dirs, packages, err := m.typeScriptModules(ctx, tree) if err != nil { return nil, err @@ -193,6 +207,9 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group if err := m.installDev(ctx, root, dir, d); err != nil { return nil, err } + if err := m.layBuilds(dir, members); err != nil { + return nil, err + } res.Installed = append(res.Installed, d) continue } @@ -201,8 +218,13 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group return nil, err } if refused == "" { - // The latest published SDK, or the group's (ADR 0573 §2, ADR 0607 §3). - if err := m.placeSDK(ctx, dir, d, served, group); err != nil { + // The latest published SDK, or the group's (ADR 0573 §2, ADR 0607 §3); the SDK as a member's build is laid from it. + if !namesPackage(members, sdkPackage) { + if err := m.placeSDK(ctx, dir, d, served, group); err != nil { + return nil, err + } + } + if err := m.layBuilds(dir, members); err != nil { return nil, err } if res.SDK == nil { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/main.go b/modules/mesh-build-node/cmd/mesh-build-node/main.go index c1abbf75f..eb037cd9e 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/main.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/main.go @@ -65,6 +65,7 @@ func main() { } m.Floor = floorOf(m.Node) m.Settings = filepath.Join(home, "settings.json") + m.ArtifactStoreFile = filepath.Join(home, "artifact-store.json") // Outside a launch by the runtime — a build, a check — there is no node and no state to write. if m.Node != "" { go m.Start(ctx) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go index 3d28b5550..32b9b6681 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go @@ -238,8 +238,9 @@ func TestTheModuleInvokesTheRegistrySeatsAndHoldsNoWriteCredential(t *testing.T) } } } - if slices.Contains(m.Requires, "artifact-store") { - t.Errorf("requires %v still name artifact-store", m.Requires) + // It requires the provision to read a group run's package members' builds (design 47 G §465), never to write. + if !slices.Contains(m.Requires, "artifact-store") { + t.Errorf("requires %v lack artifact-store, which a group run's members are read from", m.Requires) } for _, r := range m.Resources { if r["id"] == "registry" { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/module.go b/modules/mesh-build-node/cmd/mesh-build-node/module.go index 8f1aea066..f51485aff 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/module.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/module.go @@ -51,6 +51,8 @@ type Module struct { // Detected and ReadDetected are the Put and the read of the state detected-steps (design 47 G §326, §414). Detected buildstep.Put ReadDetected func(key string) (json.RawMessage, error) + // ArtifactStoreFile is the binding of the provision artifact-store, read for a group run's package members (§465). + ArtifactStoreFile string // ForgetDetected deletes one key of detected-steps; RecordAct writes an act in the hand-act log of the seat // mesh-hand-acts (§463, §480): a drop writes it first. ForgetDetected func(key string) error diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index a056ac734..4678e794e 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -66,12 +66,14 @@ ], "requires": [ {"name": "detected-steps", "storage": "key-value", "template": "state", "options": {"history": 1}}, + "artifact-store", "forge-account", "npm-package-registry", "node-toolchain", "node-tools" ], "binds": { + "artifact-store": "${dir:state}/artifact-store.json", "npm-package-registry": "${dir:state}/package-registry.json" }, "invokes": [