Join: merge A3 (wip/registry-a3)
Conflicts, both sides kept: artifact-store-tools' invokes (the assigner's and indexer's read tools, followed through views, beside the registry seats' hold, release and remove), its main (the views reader and the seat asker), its manifest row; gitea's invokes (the indexer's read tool, the hand act and the builder's versions) and its consumes of the indexer's views. gitea's account test takes the packages account's names; repoCheckFacts, removed on main, is not imported. No catalogue module reads the indexer's state. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
@@ -77,7 +77,7 @@ func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir
|
||||
if !within(realDir, realRoot) {
|
||||
return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module: it resolves to %s, and the module is %s", art.Name, art.From, realRoot, realDir)
|
||||
}
|
||||
return m.packInto(ctx, workspace, module, name, art.Name, root)
|
||||
return m.packInto(ctx, workspace, module, name, art.Name, "archive", root)
|
||||
}
|
||||
|
||||
// pinnedArchive fetches the file an artifact is pinned to, checks it against the pin, lays it out as the artifact says
|
||||
@@ -111,7 +111,7 @@ func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name stri
|
||||
} else if err := unpackUpstream(file, laid, art.Within); err != nil {
|
||||
return nil, fmt.Errorf("%s: %s (%s): %w", art.Name, art.URL, art.Version, err)
|
||||
}
|
||||
return m.packInto(ctx, workspace, module, name, art.Name, laid)
|
||||
return m.packInto(ctx, workspace, module, name, art.Name, "archive", laid)
|
||||
}
|
||||
|
||||
// pinnedFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes are still the
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
@@ -38,6 +39,7 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps {
|
||||
return buildstep.AskSteps{
|
||||
Fetch: byName["build_go_fetch"], Build: byName["build_go_build"], Digest: byName["build_go_digest"], Test: byName["build_go_test"],
|
||||
Imports: byName[ImportsTool],
|
||||
Compare: buildstep.CompareStep(buildstep.GeneratedRootGo, func(ws string) string { return filepath.Join(m.Work, ws, "tree") }),
|
||||
Toolchain: func(ctx context.Context) (delivery.ToolchainUsed, error) {
|
||||
installed, err := m.Probe(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -34,23 +34,27 @@ type BuildResult struct {
|
||||
Binary string `json:"binary"`
|
||||
Digest string `json:"digest"`
|
||||
Size int `json:"size"`
|
||||
// Reference is where the archive is in the artifact store (novox/hq ADR 0498 §1):
|
||||
// artifact-store://<module>/<artifact>/blobs/<digest>.
|
||||
// Reference is where the archive is, in the form the seat it was published through answers (novox/hq ADR 0605
|
||||
// §9): oci-artifact-registry/<module>/<artifact>@<digest>. The builder writes the older form beside it while a
|
||||
// counted node-engine reads only that (ADR 0624 §1, §3).
|
||||
Reference string `json:"reference"`
|
||||
}
|
||||
|
||||
func (m *Module) buildTool() StepTool {
|
||||
return StepTool{
|
||||
Name: "build_go_build",
|
||||
Description: "Build one Go bundle or archive artifact of a module of a fetched workspace with the build agent's recipe, by the " +
|
||||
"installed Go and no network, and pack it. Answers a step id at once; the step ends on its key with the " +
|
||||
"artifact's digest, size and the archive's path.",
|
||||
Description: "Build one Go bundle, archive or Go package artifact of a module of a fetched workspace with the build agent's " +
|
||||
"recipe, by the installed Go and no network, and pack it (a Go package as Go's module zip); a bundle or archive is " +
|
||||
"published on oci-artifact-registry, a Go package given a version on go-package-registry under its module path. " +
|
||||
"Answers a step id at once; the step ends on its key with the artifact's digest, size and reference.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
|
||||
"module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"},
|
||||
"artifact": map[string]any{"type": "string", "description": "the artifact's name in the module's manifest"},
|
||||
"version": map[string]any{"type": "string", "description": "the version the builder gave a package at the trunk's claim; absent for a candidate, whose package is not published"},
|
||||
"seat": map[string]any{"type": "string", "description": "the registry seat the artifact is published through, as the build ask names it"},
|
||||
"publish": map[string]any{"type": "boolean", "description": "true for a merged trunk commit alone: the build writes the shared Go caches; without it (a candidate) they are read beneath a throwaway layer"},
|
||||
},
|
||||
"required": []string{"workspace", "module", "artifact"},
|
||||
@@ -109,6 +113,11 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
if art == nil {
|
||||
return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", "))
|
||||
}
|
||||
if art.Kind == catalogue.ArtifactPackage && art.Language == "go" {
|
||||
version, _ := args["version"].(string)
|
||||
seat, _ := args["seat"].(string)
|
||||
return m.buildGoPackage(ctx, workspace, dir, *art, version, seat)
|
||||
}
|
||||
if art.Kind == "archive" {
|
||||
return m.buildArchive(ctx, workspace, module, manifest.Module, dir, *art)
|
||||
}
|
||||
@@ -149,7 +158,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
if err := m.insideModule(workspace, dir, out); err != nil {
|
||||
return nil, fmt.Errorf("the output directory of %s: %w", art.Name, err)
|
||||
}
|
||||
res, err := m.packInto(ctx, workspace, module, manifest.Module, art.Name, out)
|
||||
res, err := m.packInto(ctx, workspace, module, manifest.Module, art.Name, "bundle", out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -157,20 +166,20 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// packInto packs a directory with mesh-sdk's Pack, as the build agent packs every artifact, and pushes the archive to
|
||||
// the artifact store from this process, after the build step ended (novox/hq ADR 0498 §1 §2): a candidate's as a trunk
|
||||
// build's, under <module>/<artifact> by its digest. The step answers the reference; a push that fails fails the step
|
||||
// `not stored:`. The packed bytes never stay on the node: they are held in this process until the push answered (the
|
||||
// lead's decision L9).
|
||||
func (m *Module) packInto(ctx context.Context, workspace, module, name, artifact, root string) (BuildResult, error) {
|
||||
// packInto packs a directory with mesh-sdk's Pack, as the build agent packs every artifact, and publishes the archive
|
||||
// on the seat oci-artifact-registry from this process, after the build step ended (design 51 *Publishing*; publish.go):
|
||||
// a candidate's as a trunk build's, under <module>/<artifact> by its digest, of kind (bundle or archive). The step
|
||||
// answers the reference; a publish the seat refuses fails the step `not stored:`. The packed bytes never stay on the
|
||||
// node: they are held in this process until the publish answered (the lead's decision L9).
|
||||
func (m *Module) packInto(ctx context.Context, workspace, module, name, artifact, kind, root string) (BuildResult, error) {
|
||||
archive, digest, err := buildstep.Pack(root)
|
||||
if err != nil {
|
||||
return BuildResult{}, fmt.Errorf("packing %s: %w", artifact, err)
|
||||
}
|
||||
if err := m.push(ctx, name+"/"+artifact, digest, archive); err != nil {
|
||||
if err := m.publishArchive(ctx, name, artifact, kind, digest, archive); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("%s%s: %v", delivery.NotStored, artifact, err)
|
||||
}
|
||||
return BuildResult{Artifact: artifact, Digest: digest, Size: len(archive), Reference: buildstep.BlobReference(name, artifact, digest)}, nil
|
||||
return BuildResult{Artifact: artifact, Digest: digest, Size: len(archive), Reference: catalogue.SeatReference(ArtifactSeat, name, artifact, digest)}, nil
|
||||
}
|
||||
|
||||
// buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the installed go
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -310,15 +311,58 @@ func (m *Module) goEnv() []string {
|
||||
if m.GoModCache != "" {
|
||||
modcache = m.GoModCache
|
||||
}
|
||||
return append([]string{
|
||||
env := []string{
|
||||
"GOTOOLCHAIN=local",
|
||||
"GOFLAGS=-mod=mod",
|
||||
"GOMODCACHE=" + modcache,
|
||||
"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
|
||||
"GOPRIVATE=git.novox.be",
|
||||
"GIT_TERMINAL_PROMPT=0",
|
||||
"GIT_SSH_COMMAND=ssh -o BatchMode=yes",
|
||||
}, m.threadEnv()...)
|
||||
}
|
||||
if proxy := m.goSeat(); proxy != "" {
|
||||
// The Go seat first, the public proxy second, the forge's git last (CAT-BG3, E11: direct goes in the first
|
||||
// change after mesh-sdk's and mesh-contracts' first numbered versions are in the Go seat); no GOPRIVATE, and
|
||||
// the checksum database off for the forge's paths, which go.sum and the builder's h1: digest guard instead.
|
||||
env = append(env, "GOPROXY="+proxy+","+PublicGoProxy+",direct", "GONOSUMDB="+ForgeGoPaths, "GOPRIVATE=")
|
||||
} else {
|
||||
env = append(env, "GOPRIVATE="+ForgeGoPaths)
|
||||
}
|
||||
return append(append(env, "GIT_TERMINAL_PROMPT=0", "GIT_SSH_COMMAND=ssh -o BatchMode=yes"), m.threadEnv()...)
|
||||
}
|
||||
|
||||
// PublicGoProxy is Go's public module proxy; ForgeGoPaths the module paths the forge serves.
|
||||
const (
|
||||
PublicGoProxy = "https://proxy.golang.org"
|
||||
ForgeGoPaths = "git.novox.be"
|
||||
)
|
||||
|
||||
// goSeat is the Go seat's proxy URL from its binding (`at`, and `serves` scheme, port and go-path), empty when the
|
||||
// module holds no binding of it or the binding names no address.
|
||||
func (m *Module) goSeat() string {
|
||||
if m.GoSeatFile == "" {
|
||||
return ""
|
||||
}
|
||||
raw, err := os.ReadFile(m.GoSeatFile)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
var b struct {
|
||||
At string `json:"at"`
|
||||
Serves struct {
|
||||
Scheme string `json:"scheme"`
|
||||
Port int `json:"port"`
|
||||
GoPath string `json:"go-path"`
|
||||
} `json:"serves"`
|
||||
}
|
||||
if json.Unmarshal(raw, &b) != nil || b.At == "" || b.Serves.GoPath == "" {
|
||||
return ""
|
||||
}
|
||||
scheme := b.Serves.Scheme
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
if b.Serves.Port == 0 {
|
||||
return fmt.Sprintf("%s://%s%s", scheme, b.At, b.Serves.GoPath)
|
||||
}
|
||||
return fmt.Sprintf("%s://%s:%d%s", scheme, b.At, b.Serves.Port, b.Serves.GoPath)
|
||||
}
|
||||
|
||||
// goEnvFor is goEnv for a command run in dir: a directory that vendors (vendor/modules.txt) is read from its vendor
|
||||
|
||||
@@ -145,6 +145,20 @@ func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// CAT-BG3: with the Go seat bound, the fetch goes through it first, Go's public proxy second and the forge's git
|
||||
// last, with no GOPRIVATE and the checksum database off for the forge's paths.
|
||||
seat := filepath.Join(t.TempDir(), "go-package-registry.json")
|
||||
if err := os.WriteFile(seat, []byte(`{"at":"forge.mesh","serves":{"scheme":"http","port":3000,"go-path":"/api/packages/novox/go"}}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.GoSeatFile = seat
|
||||
env := strings.Join(m.goEnv(), " ")
|
||||
for _, want := range []string{"GOPROXY=http://forge.mesh:3000/api/packages/novox/go,https://proxy.golang.org,direct", "GONOSUMDB=git.novox.be", "GOPRIVATE= "} {
|
||||
if !strings.Contains(env+" ", want) {
|
||||
t.Errorf("with the Go seat bound the environment lacks %q: %s", want, env)
|
||||
}
|
||||
}
|
||||
m.GoSeatFile = ""
|
||||
if downloads != 2 || verifies != 2 {
|
||||
t.Errorf("%d downloads and %d verifies, want 2 each (go.mod and sub/dir/go.mod):\n%s", downloads, verifies, lines)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -143,10 +144,10 @@ func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) {
|
||||
// template in the files/ archive (novox/hq ADR 0349 §2).
|
||||
func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) {
|
||||
m := readForgeManifest(t)
|
||||
// artifact-store: the module pushes a done build's archive itself (ADR 0498 §2).
|
||||
// node-tools: the module pulls its asks through the tool runner's seat pull (1911).
|
||||
if len(m.Requires) != 4 || m.Requires[0] != "forge-account" || m.Requires[1] != "go-toolchain" || m.Requires[2] != "artifact-store" || m.Requires[3] != "node-tools" {
|
||||
t.Errorf("requires = %v, want [forge-account go-toolchain artifact-store node-tools]", m.Requires)
|
||||
// go-package-registry: the Go seat's read side, the fetch's first source (design 51, CAT-BG3, E2).
|
||||
if want := []any{"forge-account", "go-toolchain", "go-package-registry", "node-tools"}; !slices.Equal(m.Requires, want) {
|
||||
t.Errorf("requires = %v, want %v", m.Requires, want)
|
||||
}
|
||||
keeps := false
|
||||
for _, s := range m.State {
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
// The build of a Go package (CAT-BG2; novox/hq design 51 *Publishing*, ADR 0605 §3, §9): the directory the artifact
|
||||
// names (its one path, else the module's own) packed as Go's module zip, every file under <module path>@<version>/,
|
||||
// as Go's module proxy protocol serves it; its digest is the zip's sha256 (no h1: sum is answered: nothing in the contract carries
|
||||
// it on, the lead's ruling on A3n). A
|
||||
// trunk build given a version by the builder publishes it through go-package-registry under its Go module path (the
|
||||
// name the forge's Go seat takes, the lead's ruling 3 on A3k) and that version; a candidate's publishes nothing. The
|
||||
// zip is made in this process, after nothing ran: packing a tree runs no code of it.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/mod/modfile"
|
||||
"golang.org/x/mod/module"
|
||||
modzip "golang.org/x/mod/zip"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// GoPackageSeat is the registry seat a Go package is published on.
|
||||
const GoPackageSeat = "go-package-registry"
|
||||
|
||||
// candidateVersion is the version a candidate's zip is laid out under: no version was given, and nothing is published.
|
||||
const candidateVersion = "v0.0.0"
|
||||
|
||||
func (m *Module) buildGoPackage(ctx context.Context, workspace, dir string, art catalogue.Artifact, version, seat string) (BuildResult, error) {
|
||||
root := dir
|
||||
switch len(art.Paths) {
|
||||
case 0:
|
||||
case 1:
|
||||
root = filepath.Join(dir, filepath.FromSlash(strings.TrimSuffix(art.Paths[0], "/")))
|
||||
default:
|
||||
return BuildResult{}, fmt.Errorf("the Go package %s names %d paths; a Go package is one directory holding its go.mod", art.Name, len(art.Paths))
|
||||
}
|
||||
if err := m.insideModule(workspace, dir, root); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("the directory of %s: %w", art.Name, err)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(root, "go.mod"))
|
||||
if err != nil {
|
||||
return BuildResult{}, fmt.Errorf("the Go package %s has no go.mod in %s: %w", art.Name, art.Paths, err)
|
||||
}
|
||||
path := modfile.ModulePath(raw)
|
||||
if path == "" {
|
||||
return BuildResult{}, fmt.Errorf("the go.mod of the Go package %s names no module", art.Name)
|
||||
}
|
||||
if version != "" && seat != GoPackageSeat {
|
||||
return BuildResult{}, fmt.Errorf("the Go package %s is given the version %s and the seat %q: a Go package is published through %s", art.Name, version, seat, GoPackageSeat)
|
||||
}
|
||||
zipVersion := candidateVersion
|
||||
if version != "" {
|
||||
zipVersion = "v" + strings.TrimPrefix(version, "v")
|
||||
}
|
||||
file, err := os.CreateTemp(filepath.Join(m.Work, workspace), "gopackage-*.zip")
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
defer os.Remove(file.Name()) // the packed file never stays on the node (the lead's decision L9)
|
||||
defer file.Close()
|
||||
if err := modzip.CreateFromDir(file, module.Version{Path: path, Version: zipVersion}, root); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("packing the Go package %s (%s@%s): %w", art.Name, path, zipVersion, err)
|
||||
}
|
||||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
body, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
d := sha256.Sum256(body)
|
||||
digest := "sha256:" + hex.EncodeToString(d[:])
|
||||
res := BuildResult{Artifact: art.Name, Digest: digest, Size: len(body)}
|
||||
if version == "" {
|
||||
return res, nil
|
||||
}
|
||||
if err := m.publishOn(ctx, seat, path, "package", version, digest, transferName(path, version, digest), body); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("%s%s: %v", delivery.NotStored, art.Name, err)
|
||||
}
|
||||
res.Reference = path + "@" + version
|
||||
return res, nil
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
// The late publish (CAT-BG2): a numbered bundle or archive's held digest published under its version, by the build
|
||||
// modules' one late publish (buildstep/latepublish.go).
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// NumberedPattern is the event type this module consumes for the late publish, as its manifest's `consumes` names it.
|
||||
const NumberedPattern = buildstep.NumberedPattern
|
||||
|
||||
// LatePublish publishes a numbered bundle or archive's held digest under its version on the artifact seat; a Go package
|
||||
// publishes at its build, an image is mesh-build-image's.
|
||||
func (m *Module) LatePublish(ctx context.Context, body []byte) (bool, error) {
|
||||
call := m.Call
|
||||
if call == nil {
|
||||
call = stdio.Ask
|
||||
}
|
||||
return buildstep.LatePublish(ctx, call, body, ArtifactSeat, "bundle", "archive")
|
||||
}
|
||||
|
||||
// hearNumbered hears the builder's version.numbered for the late publish.
|
||||
func (m *Module) hearNumbered(ctx context.Context) { buildstep.HearNumbered(ctx, m.LatePublish) }
|
||||
@@ -72,6 +72,9 @@ type TestResult struct {
|
||||
// JudgedBy is the program that ran the level (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
|
||||
// runner workspace.
|
||||
JudgedBy string `json:"judged_by,omitempty"`
|
||||
// OutOfStep are the generated files of a contracts module in the workspace out of step with what the generator writes
|
||||
// (CAT-BG5, buildstep.OutOfStep): the unit level fails on any, Why naming each. Absent when none is.
|
||||
OutOfStep []string `json:"out-of-step,omitempty"`
|
||||
// By is the build module and the SDK it was built with (sdkstep.By): the checker names it when a failure hides
|
||||
// its cause.
|
||||
By string `json:"by,omitempty"`
|
||||
|
||||
@@ -54,7 +54,7 @@ func main() {
|
||||
m.Home = u.HomeDir
|
||||
}
|
||||
m.Run = StepRunner(m.Home)
|
||||
m.RegistryFile = m.Home + "/registry"
|
||||
m.GoSeatFile = m.Home + "/go-package-registry.json"
|
||||
m.Settings = m.Home + "/settings.json"
|
||||
m.ReadKept = readOf(stdio.State(KeptState)).Get
|
||||
m.ReadManifest = askManifest
|
||||
@@ -82,6 +82,8 @@ func main() {
|
||||
if m.Node != "" {
|
||||
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
|
||||
go func() { _ = puller.Pull(ctx) }()
|
||||
// The builder's version.numbered, for the late publish of a held digest (latepublish.go).
|
||||
go m.hearNumbered(ctx)
|
||||
}
|
||||
if err := stdio.Serve("", append(m.Tools(steps), m.SeatTools(puller)...)); err != nil {
|
||||
logf("%v", err)
|
||||
|
||||
@@ -75,6 +75,17 @@ func TestManifestDeclaresTheAccountTheCapabilityAndThePackages(t *testing.T) {
|
||||
if want := []string{"bubblewrap", "git", "openssh", "postgresql"}; !slices.Equal(packages, want) {
|
||||
t.Errorf("packages = %v, want %v", packages, want)
|
||||
}
|
||||
// buildstep, which mesh-build-node and mesh-build-image require, is a Go package of this module, numbered in
|
||||
// series 0.1 (CAT-BG6; ADR 0606): its users accept a series, never a commit.
|
||||
declared := false
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Name == "buildstep" {
|
||||
declared = a.Kind == "package" && a.Language == "go" && slices.Equal(a.Paths, []string{"buildstep"}) && a.Series == "0.1"
|
||||
}
|
||||
}
|
||||
if !declared {
|
||||
t.Errorf("artifacts %+v declare no Go package buildstep in series 0.1", m.Build.Artifacts)
|
||||
}
|
||||
}
|
||||
|
||||
// The work and cache directories are outside the home (ADR 0466 §1), owned by the account.
|
||||
@@ -159,21 +170,32 @@ func TestTheModuleDeclaresBubblewrapAndTheCapabilityItNeeds(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1587-S1 (ADR 0498 §2): the module requires the artifact store and is told its address in a file of its home, which no
|
||||
// run step is given (a run step has no network either, ADR 0448).
|
||||
func TestTheModuleRequiresTheArtifactStoreAndIsToldItsAddress(t *testing.T) {
|
||||
// 1587-S1, CAT-BG1 (design 51 *Publishing*, ADR 0614 §2): the module publishes a done archive through the seat
|
||||
// oci-artifact-registry's verbs, which it invokes (the served-verb edge orders it after the holder), and no longer
|
||||
// requires the artifact store or is told its address: nothing writes into the store but its holder.
|
||||
func TestTheModuleInvokesTheArtifactSeatsVerbsAndWritesNoStore(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
if !slices.Contains(m.Requires, "artifact-store") {
|
||||
t.Errorf("requires %v lack artifact-store", m.Requires)
|
||||
if slices.Contains(m.Requires, "artifact-store") {
|
||||
t.Errorf("requires %v still name artifact-store", m.Requires)
|
||||
}
|
||||
r := resource(t, m, "registry")
|
||||
// The template (files/registry) is read below; decoding leaves it unread in the resource.
|
||||
if r["type"] != "file" || r["path"] != "${dir:state}/registry" || r["template"] != "files/registry" {
|
||||
t.Errorf("the registry resource is %v", r)
|
||||
for _, seat := range []string{ArtifactSeat, GoPackageSeat} { // a Go package's too (CAT-BG2)
|
||||
for _, verb := range []string{"publish", "publish-part"} {
|
||||
if !slices.Contains(m.Invokes, "seat:"+seat+"."+verb) {
|
||||
t.Errorf("invokes %v lack the seat %s's %s", m.Invokes, seat, verb)
|
||||
}
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(moduleDir, "files", "registry"))
|
||||
if err != nil || strings.TrimSpace(string(raw)) != "${seat:mesh-artifact-store:reach}" {
|
||||
t.Errorf("files/registry is %q (%v), want the store seat's reach", raw, err)
|
||||
// The late publish hears the builder's version.numbered (latepublish.go).
|
||||
if !slices.Contains(m.Consumes, NumberedPattern) {
|
||||
t.Errorf("consumes %v lack %s", m.Consumes, NumberedPattern)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == "registry" {
|
||||
t.Errorf("the store's address is still placed: %v", r)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(moduleDir, "files", "registry")); err == nil {
|
||||
t.Error("files/registry is still shipped")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -67,10 +67,13 @@ type Module struct {
|
||||
// Inside is the integration level's own process, started shut off in bubblewrap by a test step (level.go): its
|
||||
// commands run as they are, never shut off a second time.
|
||||
Inside bool
|
||||
// RegistryFile holds the artifact store's address (host:port), placed by the manifest; Push puts a blob there
|
||||
// (buildstep.PushBlob when nil; a test's fake store).
|
||||
RegistryFile string
|
||||
Push func(ctx context.Context, registry, repository, digest string, body []byte) error
|
||||
// GoSeatFile is the binding of the provision go-package-registry, the Go seat's read side (design 51 *Go
|
||||
// packages*): bound, the mesh's Go modules are fetched through it first, Go's public proxy second, and the forge's
|
||||
// git over the account's ssh last (Go's direct, kept for this milestone only); unbound, as before, directly.
|
||||
GoSeatFile string
|
||||
// Call asks a seat's verb over the bus: the publish of a done archive (publish.go). stdio.Ask when nil; a test's
|
||||
// fake holder.
|
||||
Call callFunc
|
||||
// Executable is this program's path, which the integration level starts again shut off; os.Executable when empty.
|
||||
Executable string
|
||||
// NodeToolchains is where the Node build module keeps its installed toolchains, bound read-only into the
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
// The publish of a done build's archive (novox/hq design 51 *Publishing*, ADR 0611; CAT-BG1): from this module's own
|
||||
// process, after the build step ended, through the seat oci-artifact-registry's verbs publish and publish-part, which
|
||||
// the manifest `invokes`, with the build modules' one sender (buildstep.PublishOne): the archive in chunks, only what the holder
|
||||
// lacks. No build step reaches the store, and this module no longer writes into it directly (push.go is gone): the
|
||||
// holder checks the build with the builder and says published or refused on the seat's events.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
)
|
||||
|
||||
// ArtifactSeat is the registry seat a bundle's or an archive's archive is published on.
|
||||
const ArtifactSeat = buildstep.ArtifactSeat
|
||||
|
||||
// transferName is the transfer's name, this module's own (buildstep.TransferName).
|
||||
func transferName(module, artifact, digest string) string {
|
||||
return buildstep.TransferName(module, artifact, digest)
|
||||
}
|
||||
|
||||
// publishArchive publishes body, of digest, as <module>/<artifact> of kind (bundle or archive) on the artifact seat.
|
||||
func (m *Module) publishArchive(ctx context.Context, module, artifact, kind, digest string, body []byte) error {
|
||||
return m.publishOn(ctx, ArtifactSeat, module+"/"+artifact, kind, "", digest, transferName(module, artifact, digest), body)
|
||||
}
|
||||
|
||||
// publishOn publishes body through buildstep.PublishOne: through Call when a test set it, else the runtime's request
|
||||
// over the bus (stdio.Ask).
|
||||
func (m *Module) publishOn(ctx context.Context, seat, name, kind, version, digest, transfer string, body []byte) error {
|
||||
call := m.Call
|
||||
if call == nil {
|
||||
call = stdio.Ask
|
||||
}
|
||||
return buildstep.PublishOne(ctx, call, seat, name, kind, version, digest, transfer, body)
|
||||
}
|
||||
|
||||
// callFunc is the runtime's request and reply over the bus, as stdio.Ask makes it.
|
||||
type callFunc = func(key string, body any) (json.RawMessage, error)
|
||||
@@ -0,0 +1,286 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back.
|
||||
var pushed sync.Map
|
||||
|
||||
// fakeStore is the seat oci-artifact-registry's holder as m's publish meets it (publish.go): publish opens a transfer
|
||||
// and answers the part it lacks, publish-part takes the chunks in sequence, and the last one stores the whole part
|
||||
// when it hashes to its digest. Each blob is kept by name and digest; fail, when set, is the holder's refusal at the
|
||||
// last chunk; asked are the addresses m called.
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte // "<name>@<digest>"
|
||||
fail error
|
||||
open map[string]registry.PublishInput
|
||||
chunks map[string][]byte
|
||||
asked []string
|
||||
// published are the publishes whose last chunk was stored, in order.
|
||||
published []registry.PublishInput
|
||||
}
|
||||
|
||||
func withStore(t *testing.T, m *Module) *fakeStore {
|
||||
t.Helper()
|
||||
s := &fakeStore{blobs: map[string][]byte{}, open: map[string]registry.PublishInput{}, chunks: map[string][]byte{}}
|
||||
m.Call = func(key string, body any) (json.RawMessage, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.asked = append(s.asked, key)
|
||||
raw, _ := json.Marshal(body)
|
||||
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
|
||||
if seat != ArtifactSeat && seat != GoPackageSeat {
|
||||
return nil, errors.New("nats: no responders available for request")
|
||||
}
|
||||
switch verb {
|
||||
case "publish":
|
||||
var in registry.PublishInput
|
||||
_ = json.Unmarshal(raw, &in)
|
||||
if (seat == ArtifactSeat) != (in.Kind == "bundle" || in.Kind == "archive") || in.Transfer == "" || strings.ContainsAny(in.Transfer, "/. *>") {
|
||||
t.Errorf("publish of kind %q, transfer %q", in.Kind, in.Transfer)
|
||||
}
|
||||
s.open[in.Transfer] = in
|
||||
// A digest held under the name already travels no more (the late publish, CAT-BG2).
|
||||
if _, held := s.blobs[in.Name+"@"+in.Digest]; held {
|
||||
s.published = append(s.published, in)
|
||||
return json.Marshal(registry.PublishAnswer{Transfer: in.Transfer, Held: true})
|
||||
}
|
||||
return json.Marshal(registry.PublishAnswer{Transfer: in.Transfer, Lacks: []string{in.Digest}})
|
||||
case "publish-part":
|
||||
var c registry.PublishPartInput
|
||||
_ = json.Unmarshal(raw, &c)
|
||||
data, _ := base64.StdEncoding.DecodeString(c.Data)
|
||||
s.chunks[c.Transfer] = append(s.chunks[c.Transfer], data...)
|
||||
if !c.Last {
|
||||
return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part})
|
||||
}
|
||||
in, whole := s.open[c.Transfer], s.chunks[c.Transfer]
|
||||
delete(s.chunks, c.Transfer)
|
||||
if s.fail != nil {
|
||||
return nil, s.fail
|
||||
}
|
||||
sum := sha256.Sum256(whole)
|
||||
if "sha256:"+hex.EncodeToString(sum[:]) != in.Digest {
|
||||
t.Errorf("published %s with bytes of another digest", in.Digest)
|
||||
}
|
||||
s.blobs[in.Name+"@"+in.Digest] = whole
|
||||
s.published = append(s.published, in)
|
||||
pushed.Store(in.Digest, whole)
|
||||
return json.Marshal(registry.PublishPartAnswer{Transfer: c.Transfer, Part: c.Part, Complete: true})
|
||||
}
|
||||
return nil, errors.New("nats: no responders available for request")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// archiveOf is the archive a build pushed, by its digest.
|
||||
func archiveOf(t *testing.T, res BuildResult) []byte {
|
||||
t.Helper()
|
||||
raw, ok := pushed.Load(res.Digest)
|
||||
if !ok {
|
||||
t.Fatalf("nothing was pushed under %s", res.Digest)
|
||||
}
|
||||
return raw.([]byte)
|
||||
}
|
||||
|
||||
// 1587-S2 (ADR 0498 §1): a done build answers a reference of the stated form whose tail is the build's digest, and the
|
||||
// store holds the archive there; a bundle and an archive alike.
|
||||
func TestADoneBuildAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
store := withStore(t, m)
|
||||
res, st := runBuild(t, m, states, "tools")
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if want := "oci-artifact-registry/demo/tools@" + res.Digest; res.Reference != want {
|
||||
t.Errorf("reference %q, want %q", res.Reference, want)
|
||||
}
|
||||
if store.blobs["demo/tools@"+res.Digest] == nil || store.asked[0] != "seat:oci-artifact-registry.publish" {
|
||||
t.Errorf("the seat was asked %v and holds %d blobs, not demo/tools@%s", store.asked, len(store.blobs), res.Digest)
|
||||
}
|
||||
a, states2, _ := archiveModule(t, "")
|
||||
withStore(t, a)
|
||||
arch, st := runBuild(t, a, states2, "files")
|
||||
if st.State != delivery.StepPassed || arch.Reference != "oci-artifact-registry/demo/files@"+arch.Digest {
|
||||
t.Errorf("the archive artifact: %s %q, reference %q", st.State, st.Reason, arch.Reference)
|
||||
}
|
||||
}
|
||||
|
||||
// 1587-S3 (ADR 0498 §2), CAT-BG1: a publish the seat's holder refuses fails the step `not stored: <artifact>: <the
|
||||
// holder's words>`, and so does a seat nothing holds.
|
||||
func TestARefusedPushFailsTheStepNotStored(t *testing.T) {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
store := withStore(t, m)
|
||||
store.fail = errors.New("refused: the forge did not take it: disk full")
|
||||
if _, st := runBuild(t, m, states, "tools"); st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, delivery.NotStored+"tools: ") || !strings.Contains(st.Reason, "disk full") {
|
||||
t.Errorf("the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
m.Call = func(string, any) (json.RawMessage, error) {
|
||||
return nil, errors.New("nats: no responders available for request")
|
||||
}
|
||||
if _, st := runBuild(t, m, states, "tools"); st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, delivery.NotStored+"tools: ") ||
|
||||
!strings.Contains(st.Reason, "oci-artifact-registry") {
|
||||
t.Errorf("no holder: the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// 1587-S4 (the lead's decision L9): the packed file is not left on the node once its push answered, a candidate's
|
||||
// included: the workspace holds no archive, pushed or refused.
|
||||
func TestThePackedFileLeavesTheNodeOnceItsPushAnswered(t *testing.T) {
|
||||
for _, fail := range []error{nil, errors.New("refused")} {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
withStore(t, m).fail = fail
|
||||
runBuild(t, m, states, "tools")
|
||||
var archives []string
|
||||
filepath.WalkDir(m.Work, func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil && strings.HasSuffix(p, ".tar.gz") {
|
||||
archives = append(archives, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if len(archives) != 0 {
|
||||
t.Errorf("push answered %v: archives left on the node: %v", fail, archives)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CAT-BG2 (REG S23, ADR 0605 §3, §9; the lead's ruling 3 on A3k): a Go package is built as Go's module zip of the
|
||||
// directory its artifact names, every file under <module path>@<version>/, its digest the zip's.
|
||||
// A trunk build given a version publishes it through go-package-registry under its Go module path and that version,
|
||||
// and answers <module path>@<version>; a candidate's (no version) publishes nothing and answers its digest alone. A
|
||||
// directory with no go.mod fails the step naming it.
|
||||
func TestAGoPackageIsBuiltAsAModuleZipAndPublishedUnderItsModulePath(t *testing.T) {
|
||||
const pkg = `{"name":"go","kind":"package","language":"go","paths":["go"],"series":"0.1"}`
|
||||
m, states, _ := buildModule(t, manifestWith(pkg), "unused")
|
||||
src := filepath.Join(m.Work, "ws1", "tree", "modules", "demo", "go")
|
||||
for name, body := range map[string]string{"go.mod": "module git.example/demo/go\n\ngo 1.26\n", "a.go": "package demo\n",
|
||||
"vendor/x/x.go": "package x\n"} {
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(src, name)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
run := func(args map[string]any) (BuildResult, delivery.Step) {
|
||||
t.Helper()
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == "build_go_build" {
|
||||
if _, err := tool.Run(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res BuildResult
|
||||
if st.State == delivery.StepPassed {
|
||||
_ = json.Unmarshal(st.Result, &res)
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
store := withStore(t, m)
|
||||
args := map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "go"}
|
||||
cand, st := run(args)
|
||||
if st.State != delivery.StepPassed || !strings.HasPrefix(cand.Digest, "sha256:") || cand.Reference != "" || len(store.asked) != 0 {
|
||||
t.Fatalf("a candidate's Go package: %s %q, %+v, asked %v", st.State, st.Reason, cand, store.asked)
|
||||
}
|
||||
args["version"], args["seat"] = "v0.1.15", GoPackageSeat
|
||||
res, st := run(args)
|
||||
if st.State != delivery.StepPassed || res.Reference != "git.example/demo/go@v0.1.15" {
|
||||
t.Fatalf("a trunk Go package: %s %q, %+v", st.State, st.Reason, res)
|
||||
}
|
||||
if len(store.published) != 1 {
|
||||
t.Fatalf("published %+v", store.published)
|
||||
}
|
||||
in := store.published[0]
|
||||
if in.Name != "git.example/demo/go" || in.Version != "v0.1.15" || in.Kind != "package" || in.Digest != res.Digest {
|
||||
t.Errorf("the publish: %+v", in)
|
||||
}
|
||||
z, err := zip.NewReader(bytes.NewReader(store.blobs[in.Name+"@"+in.Digest]), int64(len(store.blobs[in.Name+"@"+in.Digest])))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for _, f := range z.File {
|
||||
names = append(names, f.Name)
|
||||
}
|
||||
if strings.Join(names, " ") != "git.example/demo/go@v0.1.15/a.go git.example/demo/go@v0.1.15/go.mod" {
|
||||
t.Errorf("the zip holds %v", names)
|
||||
}
|
||||
if err := os.Remove(filepath.Join(src, "go.mod")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, st := run(args); st.State != delivery.StepFailed || !strings.Contains(st.Reason, "go.mod") {
|
||||
t.Errorf("no go.mod: %s %q", st.State, st.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// CAT-BG2 (design 51 *The versions*): the builder's version.numbered for a bundle or an archive publishes the held
|
||||
// digest under its version, sending nothing; a digest the holder no longer holds fails the event (offered again),
|
||||
// and what is not this module's to publish is let pass.
|
||||
func TestTheNumberedEventPublishesAHeldDigestUnderItsVersion(t *testing.T) {
|
||||
digest := "sha256:" + strings.Repeat("ab", 32)
|
||||
other := "sha256:" + strings.Repeat("cd", 32)
|
||||
numbered := func(kind, seat, digest string) []byte {
|
||||
raw, err := delivery.EncodeVersionNumbered(delivery.VersionNumbered{Name: "mod/tools", Kind: kind, Version: "0.1.4",
|
||||
Module: "mod", Fingerprint: "fp", Commit: "c0", Digest: digest, Seat: seat})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
body []byte
|
||||
did, fail bool
|
||||
}{
|
||||
{"a held bundle is tagged", numbered("bundle", ArtifactSeat, digest), true, false},
|
||||
{"an archive with no seat named goes to the artifact seat", numbered("archive", "", digest), true, false},
|
||||
{"a digest the holder lacks fails, sending nothing", numbered("bundle", ArtifactSeat, other), false, true},
|
||||
{"a Go package is not published late", numbered("package", GoPackageSeat, digest), false, false},
|
||||
{"an image is mesh-build-image's", numbered("image", "container-image-registry", digest), false, false},
|
||||
{"no digest, nothing to publish", numbered("bundle", ArtifactSeat, ""), false, false},
|
||||
{"an unreadable body fails", []byte(`{"name":"mod/tools"}`), false, true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
m := &Module{}
|
||||
s := withStore(t, m)
|
||||
s.blobs["mod/tools@"+digest] = []byte("held")
|
||||
did, err := m.LatePublish(t.Context(), c.body)
|
||||
if did != c.did || (err != nil) != c.fail {
|
||||
t.Fatalf("did %v, err %v; want did %v, fail %v", did, err, c.did, c.fail)
|
||||
}
|
||||
for _, k := range s.asked {
|
||||
if strings.HasSuffix(k, ".publish-part") {
|
||||
t.Errorf("a late publish sent a part: %v", s.asked)
|
||||
}
|
||||
}
|
||||
if !c.did {
|
||||
return
|
||||
}
|
||||
if len(s.published) != 1 {
|
||||
t.Fatalf("published %v", s.published)
|
||||
}
|
||||
p := s.published[0]
|
||||
if p.Name != "mod/tools" || p.Version != "0.1.4" || p.Digest != digest || p.Fingerprint != "fp" || p.Commit != "c0" {
|
||||
t.Errorf("published %+v", p)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
package main
|
||||
|
||||
// The push of a done build's archive (novox/hq ADR 0498 §1 §2; issue 1587): from this module's own process, after the
|
||||
// build step ended, with mesh-sdk's one client (buildstep.PushBlob), to the artifact store whose address the manifest
|
||||
// binds into RegistryFile. No build step reaches the store: a run step has no network (ADR 0448).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
)
|
||||
|
||||
// registry is the artifact store's address, host:port, as the manifest bound it into RegistryFile.
|
||||
func (m *Module) registry() (string, error) {
|
||||
raw, err := os.ReadFile(m.RegistryFile)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the artifact store's address cannot be read: %w", err)
|
||||
}
|
||||
addr := strings.TrimSpace(string(raw))
|
||||
if addr == "" || strings.Contains(addr, "${") || strings.HasPrefix(addr, ":") || strings.HasSuffix(addr, ":") {
|
||||
return "", fmt.Errorf("the artifact store's address in %s is %q, not host:port", m.RegistryFile, addr)
|
||||
}
|
||||
return addr, nil
|
||||
}
|
||||
|
||||
// push puts body into the artifact store as a blob of repository (<module>/<artifact>) by its digest: Push when a test
|
||||
// set it, else buildstep.PushBlob.
|
||||
func (m *Module) push(ctx context.Context, repository, digest string, body []byte) error {
|
||||
addr, err := m.registry()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m.Push != nil {
|
||||
return m.Push(ctx, addr, repository, digest, body)
|
||||
}
|
||||
return buildstep.PushBlob(ctx, &http.Client{Timeout: 10 * time.Minute}, addr, repository, digest, body)
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back.
|
||||
var pushed sync.Map
|
||||
|
||||
// fakeStore makes m push to a store of the test's own: its address in RegistryFile, each blob kept by repository and
|
||||
// digest; fail, when set, is the store's refusal.
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte // "<repository>@<digest>"
|
||||
fail error
|
||||
addr string
|
||||
}
|
||||
|
||||
func withStore(t *testing.T, m *Module) *fakeStore {
|
||||
t.Helper()
|
||||
s := &fakeStore{blobs: map[string][]byte{}}
|
||||
m.RegistryFile = filepath.Join(t.TempDir(), "registry")
|
||||
if err := os.WriteFile(m.RegistryFile, []byte("store.test:5000\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Push = func(ctx context.Context, registry, repository, digest string, body []byte) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.addr = registry
|
||||
if s.fail != nil {
|
||||
return s.fail
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if "sha256:"+hex.EncodeToString(sum[:]) != digest {
|
||||
t.Errorf("pushed %s with bytes of another digest", digest)
|
||||
}
|
||||
s.blobs[repository+"@"+digest] = append([]byte(nil), body...)
|
||||
pushed.Store(digest, s.blobs[repository+"@"+digest])
|
||||
return nil
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// archiveOf is the archive a build pushed, by its digest.
|
||||
func archiveOf(t *testing.T, res BuildResult) []byte {
|
||||
t.Helper()
|
||||
raw, ok := pushed.Load(res.Digest)
|
||||
if !ok {
|
||||
t.Fatalf("nothing was pushed under %s", res.Digest)
|
||||
}
|
||||
return raw.([]byte)
|
||||
}
|
||||
|
||||
// 1587-S2 (ADR 0498 §1): a done build answers a reference of the stated form whose tail is the build's digest, and the
|
||||
// store holds the archive there; a bundle and an archive alike.
|
||||
func TestADoneBuildAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
store := withStore(t, m)
|
||||
res, st := runBuild(t, m, states, "tools")
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if want := "artifact-store://demo/tools/blobs/" + res.Digest; res.Reference != want {
|
||||
t.Errorf("reference %q, want %q", res.Reference, want)
|
||||
}
|
||||
if store.addr != "store.test:5000" || store.blobs["demo/tools@"+res.Digest] == nil {
|
||||
t.Errorf("the store at %q holds %d blobs, not demo/tools@%s", store.addr, len(store.blobs), res.Digest)
|
||||
}
|
||||
a, states2, _ := archiveModule(t, "")
|
||||
withStore(t, a)
|
||||
arch, st := runBuild(t, a, states2, "files")
|
||||
if st.State != delivery.StepPassed || arch.Reference != "artifact-store://demo/files/blobs/"+arch.Digest {
|
||||
t.Errorf("the archive artifact: %s %q, reference %q", st.State, st.Reason, arch.Reference)
|
||||
}
|
||||
}
|
||||
|
||||
// 1587-S3 (ADR 0498 §2): a push the store refuses fails the step `not stored: <artifact>: <the store's words>`, and so
|
||||
// does a store whose address the manifest did not place.
|
||||
func TestARefusedPushFailsTheStepNotStored(t *testing.T) {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
store := withStore(t, m)
|
||||
store.fail = errors.New("store.test:5000 refused the blob: 500 Internal Server Error disk full")
|
||||
if _, st := runBuild(t, m, states, "tools"); st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, delivery.NotStored+"tools: ") || !strings.Contains(st.Reason, "disk full") {
|
||||
t.Errorf("the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
m.RegistryFile = filepath.Join(t.TempDir(), "absent")
|
||||
if _, st := runBuild(t, m, states, "tools"); st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, delivery.NotStored+"tools: ") {
|
||||
t.Errorf("no address: the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// 1587-S4 (the lead's decision L9): the packed file is not left on the node once its push answered, a candidate's
|
||||
// included: the workspace holds no archive, pushed or refused.
|
||||
func TestThePackedFileLeavesTheNodeOnceItsPushAnswered(t *testing.T) {
|
||||
for _, fail := range []error{nil, errors.New("refused")} {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
withStore(t, m).fail = fail
|
||||
runBuild(t, m, states, "tools")
|
||||
var archives []string
|
||||
filepath.WalkDir(m.Work, func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil && strings.HasSuffix(p, ".tar.gz") {
|
||||
archives = append(archives, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if len(archives) != 0 {
|
||||
t.Errorf("push answered %v: archives left on the node: %v", fail, archives)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,12 +3,16 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -19,6 +23,8 @@ import (
|
||||
"github.com/distribution/distribution/v3/registry/handlers"
|
||||
_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
|
||||
|
||||
registry "git.novox.be/novox/mesh-contracts/go/seats/containerimageregistry"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -34,23 +40,68 @@ func realStore(t *testing.T, m *Module) string {
|
||||
srv := httptest.NewServer(handlers.NewApp(context.Background(), config))
|
||||
t.Cleanup(srv.Close)
|
||||
addr := strings.TrimPrefix(srv.URL, "http://")
|
||||
m.RegistryFile = filepath.Join(t.TempDir(), "registry")
|
||||
if err := os.WriteFile(m.RegistryFile, []byte(addr+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
// The seat's holder as distribution holds it: the publish through the seat's verbs (publish.go, played by withStore),
|
||||
// each whole part then stored in the store under <module>/<artifact> by its digest.
|
||||
s := withStore(t, m)
|
||||
inner := m.Call
|
||||
m.Call = func(key string, body any) (json.RawMessage, error) {
|
||||
raw, err := inner(key, body)
|
||||
var a registry.PublishPartAnswer
|
||||
if err != nil || !strings.HasSuffix(key, ".publish-part") || json.Unmarshal(raw, &a) != nil || !a.Complete {
|
||||
return raw, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for k, b := range s.blobs {
|
||||
name, digest, _ := strings.Cut(k, "@")
|
||||
if err := uploadBlob(srv.URL, name, digest, b); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
delete(s.blobs, k)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
m.Push = nil
|
||||
return addr
|
||||
}
|
||||
|
||||
// resolve reads a done artifact's reference from the store, as the controller will: artifact-store://<module>/
|
||||
// <artifact>/blobs/<digest> is <store>/v2/<module>/<artifact>/blobs/<digest>.
|
||||
// uploadBlob is the holder's own write into its store: an upload opened, then its bytes put by their digest (the
|
||||
// distribution protocol's two requests), as distribution stores a part it took whole.
|
||||
func uploadBlob(base, name, digest string, body []byte) error {
|
||||
resp, err := http.Post(base+"/v2/"+name+"/blobs/uploads/", "", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
loc, err := url.Parse(resp.Header.Get("Location"))
|
||||
if err != nil || resp.StatusCode != http.StatusAccepted {
|
||||
return fmt.Errorf("the store opened no upload of %s: %s %v", digest, resp.Status, err)
|
||||
}
|
||||
b, _ := url.Parse(base)
|
||||
u := b.ResolveReference(loc)
|
||||
q := u.Query()
|
||||
q.Set("digest", digest)
|
||||
u.RawQuery = q.Encode()
|
||||
req, _ := http.NewRequest(http.MethodPut, u.String(), bytes.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
put, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
put.Body.Close()
|
||||
if put.StatusCode != http.StatusCreated {
|
||||
return fmt.Errorf("the store did not take %s: %s", digest, put.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolve reads a done artifact's reference from the store as a node-engine does (catalogue.Routed): the seat's form
|
||||
// oci-artifact-registry/<module>/<artifact>@<digest> is <store>/v2/<module>/<artifact>/blobs/<digest> (ADR 0605 §9).
|
||||
func resolve(t *testing.T, addr string, a delivery.BuiltArtifact) []byte {
|
||||
t.Helper()
|
||||
path, ok := strings.CutPrefix(a.Reference, "artifact-store://")
|
||||
if !ok {
|
||||
t.Fatalf("%s's reference %q is not the artifact store's", a.Name, a.Reference)
|
||||
if _, _, ok := catalogue.InRegistrySeat(a.Reference); !ok {
|
||||
t.Fatalf("%s's reference %q is not in the form a registry seat answers", a.Name, a.Reference)
|
||||
}
|
||||
resp, err := http.Get("http://" + addr + "/v2/" + path)
|
||||
resp, err := http.Get(catalogue.Routed(a.Reference, addr))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -101,7 +152,7 @@ func TestADoneBuildsArtifactsResolveInARealArtifactStore(t *testing.T) {
|
||||
t.Errorf("the done key is not deliverable, missing %v", missing)
|
||||
}
|
||||
for _, a := range done.Artifacts {
|
||||
if !strings.HasSuffix(a.Reference, "/blobs/"+a.Digest) {
|
||||
if !strings.HasSuffix(a.Reference, "@"+a.Digest) {
|
||||
t.Errorf("%s: the reference %q does not end in its digest %s", a.Name, a.Reference, a.Digest)
|
||||
}
|
||||
sum := sha256.Sum256(resolve(t, addr, a))
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
sdkstep "git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
@@ -100,9 +101,42 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
||||
return nil, err
|
||||
}
|
||||
res.JudgedBy = RunnerInstalled
|
||||
if level == LevelUnit {
|
||||
if err := compareGenerated(tree, res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// compareGenerated regenerates the generated Go files of each contracts module in the tree (the tree's own manifest and
|
||||
// each modules/<m>/module.json; CAT-BG5) and fails the level on any out of step, Why naming each path.
|
||||
func compareGenerated(tree string, res *TestResult) error {
|
||||
manifests, _ := filepath.Glob(filepath.Join(tree, "modules", "*", "module.json"))
|
||||
if _, err := os.Stat(filepath.Join(tree, "module.json")); err == nil {
|
||||
manifests = append([]string{filepath.Join(tree, "module.json")}, manifests...)
|
||||
}
|
||||
for _, mf := range manifests {
|
||||
dir := filepath.Dir(mf)
|
||||
paths, err := buildstep.OutOfStep(dir, buildstep.GeneratedRootGo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(tree, dir)
|
||||
for _, p := range paths {
|
||||
res.OutOfStep = append(res.OutOfStep, filepath.ToSlash(filepath.Join(rel, p)))
|
||||
}
|
||||
}
|
||||
if len(res.OutOfStep) > 0 {
|
||||
why := "generated files out of step with the seat generator (regenerate them): " + strings.Join(res.OutOfStep, ", ")
|
||||
if res.Why != "" {
|
||||
why = res.Why + "; " + why
|
||||
}
|
||||
res.Passed, res.Why = false, why
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// testAsCandidate runs a level as the candidate's program (ADR 0535 §1): the integration level is refused first as the
|
||||
// installed program would refuse it (no postgres), then the whole level runs in the candidate's one shut-off process.
|
||||
func (m *Module) testAsCandidate(ctx context.Context, workspace, level string, packages []string, pattern string, names []string) (*TestResult, error) {
|
||||
|
||||
@@ -178,6 +178,40 @@ func TestUnitLevelOfAFixtureWithNoTestsSaysSo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// CAT-BG5: a contracts module (a package artifact naming generated-from) whose generated Go files are out of step with
|
||||
// the seat generator fails the unit level, each path named in its out-of-step and its why; its TypeScript is not this
|
||||
// module's to compare.
|
||||
func TestUnitLevelOfAContractsModuleOutOfStepFailsNamingEachPath(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "empty")
|
||||
tree := filepath.Join(m.Work, ws, "tree")
|
||||
for p, text := range map[string]string{
|
||||
"module.json": `{"module":"c","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"],"generated-from":"."}]}}`,
|
||||
"seats/index.json": `{"format":1,"seats":["one-store"]}`,
|
||||
"seats/one-store/seat.json": `{"format":1,"seat":"one-store","scope":"mesh","grain":"seat-list","decision":"ADR 1","verbs":[{"name":"query","input":{"type":"object"}}]}`,
|
||||
"go/go.mod": "module example.org/c/go\n",
|
||||
"go/stray.go": "package stray\n",
|
||||
"ts/package.json": `{"name":"@example/c"}`,
|
||||
"ts/stray.ts": "// not this module's\n",
|
||||
} {
|
||||
full := filepath.Join(tree, filepath.FromSlash(p))
|
||||
_ = os.MkdirAll(filepath.Dir(full), 0o755)
|
||||
if err := os.WriteFile(full, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
res, st := runUnit(t, m, states, ws)
|
||||
stray := false
|
||||
for _, p := range res.OutOfStep {
|
||||
stray = stray || p == "go/stray.go"
|
||||
if !strings.HasPrefix(p, "go/") {
|
||||
t.Errorf("%s is not under the Go root", p)
|
||||
}
|
||||
}
|
||||
if st.State != delivery.StepPassed || res.Passed || !stray || !strings.Contains(res.Why, "go/stray.go") {
|
||||
t.Errorf("result = %+v (%s %s)", res, st.State, st.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// A package that does not build fails the level with the compiler's words.
|
||||
func TestUnitLevelOfAPackageThatDoesNotBuildFailsWithTheCompilersLine(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "broken")
|
||||
|
||||
Reference in New Issue
Block a user