Compare commits
145
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0786408109 | ||
|
|
59c28971cb | ||
|
|
035c950ff2 | ||
|
|
5a79776625 | ||
|
|
b9cdca78b0 | ||
|
|
fde561075d | ||
|
|
889fa09ea4 | ||
|
|
3a25b6e1f8 | ||
|
|
22992da15d | ||
|
|
b6247f837d | ||
|
|
9c5b7e14e1 | ||
|
|
c0873728e2 | ||
|
|
d108487396 | ||
|
|
eed6be0f4a | ||
|
|
51900367a0 | ||
|
|
77596f37a3 | ||
|
|
56f842c3af | ||
|
|
075b61901c | ||
|
|
49caf17ff5 | ||
|
|
cac09a4e20 | ||
|
|
29b4c02709 | ||
|
|
c3d3194bc8 | ||
|
|
d0463fb510 | ||
|
|
6bf054b982 | ||
|
|
fbbf28d21c | ||
|
|
fb89990e2b | ||
|
|
d0a3784be2 | ||
|
|
555ed4963a | ||
|
|
edb6d3b7d3 | ||
|
|
8928e6f972 | ||
|
|
2d8c8f6d75 | ||
|
|
9194e15072 | ||
|
|
77578a882f | ||
|
|
3f9081d00e | ||
|
|
7842001c53 | ||
|
|
09e749bc1a | ||
|
|
ea8692b08f | ||
|
|
3cf75580b5 | ||
|
|
a22fdd015b | ||
|
|
dcf6e86d02 | ||
|
|
98ca97fc5b | ||
|
|
db67a0e55e | ||
|
|
34277ef60f | ||
|
|
a3b51a9d9a | ||
|
|
e5608f7b00 | ||
|
|
5a18eb5f4a | ||
|
|
138a1e1c4b | ||
|
|
f1fbd91f0d | ||
|
|
5ddc3ff312 | ||
|
|
b5908b486e | ||
|
|
aa57bbcd44 | ||
|
|
67a585600d | ||
|
|
1caf728a4b | ||
|
|
02816c0f7a | ||
|
|
7bf63b0ac5 | ||
|
|
426db57d3f | ||
|
|
1d995d2bef | ||
|
|
92225cd528 | ||
|
|
35e4764d35 | ||
|
|
b7b9f2a3da | ||
|
|
68183ee441 | ||
|
|
cff5ed46b7 | ||
|
|
7451e7edf4 | ||
|
|
529c156f64 | ||
|
|
47df3a4d60 | ||
|
|
2d2e831659 | ||
|
|
d3ebaf72eb | ||
|
|
183b9680c1 | ||
|
|
74baa53787 | ||
|
|
332aa75afd | ||
|
|
9e2d1164f0 | ||
|
|
48b08c74ab | ||
|
|
260be1c3e6 | ||
|
|
224a7a66a5 | ||
|
|
5a6d082e8c | ||
|
|
7b5f7f84ac | ||
|
|
3508be1476 | ||
|
|
7c5d11e55d | ||
|
|
3dcaa7a996 | ||
|
|
848249cf54 | ||
|
|
692fc3e1ab | ||
|
|
ce8d52ed0b | ||
|
|
f5153a490c | ||
|
|
8815f950b0 | ||
|
|
c948f3a61c | ||
|
|
91877e3421 | ||
|
|
77b89ff182 | ||
|
|
e7ca330e1f | ||
|
|
b4e90e5fce | ||
|
|
e2220df5e8 | ||
|
|
0ec446ab75 | ||
|
|
3c0496d1d3 | ||
|
|
97a111e70a | ||
|
|
eb4f16431c | ||
|
|
c4fd31aa7e | ||
|
|
09d5360bce | ||
|
|
b4d7a32abf | ||
|
|
6aa4f135c9 | ||
|
|
4915f69857 | ||
|
|
843e408556 | ||
|
|
4ef2fa2dff | ||
|
|
8288042eb7 | ||
|
|
3bbae41037 | ||
|
|
77f3963d83 | ||
|
|
49a28f8537 | ||
|
|
c770f7947d | ||
|
|
4b8e7f11cf | ||
|
|
0d2686b5eb | ||
|
|
839be0eb64 | ||
|
|
dc144a4e7d | ||
|
|
bfc42e4f4f | ||
|
|
c4176107f2 | ||
|
|
a3794ea156 | ||
|
|
e8cb2ee470 | ||
|
|
07b3c5e77a | ||
|
|
f3100d1116 | ||
|
|
64a96a13f7 | ||
|
|
f1fb73e12c | ||
|
|
d2faada61b | ||
|
|
f436d1dace | ||
|
|
12ff1cfbc4 | ||
|
|
7bf0de0ff5 | ||
|
|
59b13d091d | ||
|
|
ce72a366bf | ||
|
|
d9876e5296 | ||
|
|
62e67578eb | ||
|
|
1319432e55 | ||
|
|
0cf7b2344f | ||
|
|
aa5d263c08 | ||
|
|
769a8d6f6a | ||
|
|
34b310b9f8 | ||
|
|
33d8b71cf7 | ||
|
|
aa0eaa6c52 | ||
|
|
d5804927b3 | ||
|
|
3793175894 | ||
|
|
b70f246622 | ||
|
|
cdbdb47474 | ||
|
|
689e61081d | ||
|
|
c83f48b821 | ||
|
|
db5e6c5b48 | ||
|
|
b9dcf19c3f | ||
|
|
e3a714d5f0 | ||
|
|
08cec8cb1c | ||
|
|
e4cbffbaa9 | ||
|
|
f178aab641 |
@@ -27,3 +27,4 @@ modules/mesh-provisioner-client/cmd/mesh-provisioner-client/mesh-provisioner-cli
|
||||
modules/brother/cmd/brother/brother
|
||||
modules/mesh-build-go/cmd/mesh-build-go/mesh-build-go
|
||||
modules/mesh-config/mesh-config
|
||||
modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/imagebuild
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,92 @@
|
||||
// Command imagebuild refuses any manifest but modules/buildah providing the rootless-image-build provision
|
||||
// (novox/hq ADR 0487 §3, issue 1704): the image builder's tool is buildah alone, and docker, whose daemon holds
|
||||
// the machine's root, never stands in for it. A second provider would let a node compose mesh-build-image over
|
||||
// something that is not the rootless builder it was designed against.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Provision is the provision only Provider may provide.
|
||||
const Provision = "rootless-image-build"
|
||||
|
||||
// Provider is the one module allowed to provide it.
|
||||
const Provider = "buildah"
|
||||
|
||||
// Finding is one manifest refused: a module other than buildah providing the provision.
|
||||
type Finding struct {
|
||||
Module string
|
||||
}
|
||||
|
||||
// Check reads every manifest under root/modules and answers which ones it refuses, and whether buildah provides.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Finding
|
||||
provided := false
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Provides []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"provides"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
for _, p := range m.Provides {
|
||||
if p.Name != Provision {
|
||||
continue
|
||||
}
|
||||
if m.Module == Provider {
|
||||
provided = true
|
||||
} else {
|
||||
out = append(out, Finding{Module: m.Module})
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if !provided {
|
||||
out = append(out, Finding{Module: Provider})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
if f.Module == Provider {
|
||||
fmt.Printf("%s does not provide %s (novox/hq ADR 0487 §3)\n", Provider, Provision)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%s provides %s: only %s may, and docker never (novox/hq ADR 0487 §3)\n", f.Module, Provision, Provider)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("imagebuild: rootless-image-build is provided only by buildah")
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func catalogue(t *testing.T, manifests map[string]string) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for name, body := range manifests {
|
||||
dir := filepath.Join(root, "modules", name)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
const buildah = `{"module":"buildah","provides":[{"name":"rootless-image-build","reach":"machine"}]}`
|
||||
|
||||
func TestOnlyBuildahProvidesTheRootlessImageBuild(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{"buildah": buildah, "go": `{"module":"go","provides":[{"name":"go-toolchain"}]}`})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("found %v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerAndAnySecondProviderAreRefused(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{
|
||||
"buildah": buildah,
|
||||
"docker": `{"module":"docker","provides":[{"name":"rootless-image-build"}]}`,
|
||||
"other": `{"module":"other","provides":[{"name":"x"},{"name":"rootless-image-build"}]}`,
|
||||
})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []Finding{{Module: "docker"}, {Module: "other"}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("found %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildahMustProvideIt(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{"buildah": `{"module":"buildah"}`})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []Finding{{Module: "buildah"}}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("found %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it stands: this is the check itself, run here too.
|
||||
func TestTheCatalogueGivesTheProvisionToBuildahAlone(t *testing.T) {
|
||||
got, err := Check("../..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("the catalogue's providers of rootless-image-build are wrong: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/imageplace
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,91 @@
|
||||
// Command imageplace refuses an image artifact whose Dockerfile is not a file inside its own module's directory
|
||||
// (novox/hq ADR 0487 §1, issue 1707): the image builder builds a module from that module's directory alone, so
|
||||
// a `from` that leaves it (../other/Dockerfile) or names a file that is not there could never build.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Finding is one image refused: Module's artifact Artifact names From, and Why says what is wrong with it.
|
||||
type Finding struct {
|
||||
Module, Artifact, From, Why string
|
||||
}
|
||||
|
||||
// Check reads every manifest under root/modules and answers which images it refuses.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Finding
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
From string `json:"from"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
dir := filepath.Dir(path)
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != "image" {
|
||||
continue
|
||||
}
|
||||
f := Finding{Module: m.Module, Artifact: a.Name, From: a.From}
|
||||
clean := filepath.Clean(a.From)
|
||||
switch {
|
||||
case a.From == "" || filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)):
|
||||
f.Why = "is not a path inside the module's own directory"
|
||||
default:
|
||||
info, err := os.Stat(filepath.Join(dir, clean))
|
||||
if err != nil || info.IsDir() {
|
||||
f.Why = "is not a file in the module's directory"
|
||||
}
|
||||
}
|
||||
if f.Why != "" {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Printf("%s: image %s builds from %q, which %s (novox/hq ADR 0487 §1)\n", f.Module, f.Artifact, f.From, f.Why)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("imageplace: every image builds from a Dockerfile in its own module's directory")
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// module writes modules/<name>/module.json with one image artifact from `from`, and the files named.
|
||||
func module(t *testing.T, root, name, from string, files ...string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", name)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := `{"module":"` + name + `","build":{"artifacts":[{"name":"server","kind":"image","from":"` + from + `"},{"name":"files","kind":"archive","from":"../elsewhere"}]}}`
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
p := filepath.Join(dir, f)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte("FROM scratch\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestADockerfileInTheModulesOwnDirectoryPasses(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "own", "Dockerfile", "Dockerfile")
|
||||
module(t, root, "nested", "image/Dockerfile", "image/Dockerfile")
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("found %v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADockerfileOutsideTheModulesDirectoryFails(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "other", "Dockerfile", "Dockerfile")
|
||||
module(t, root, "leaves", "../other/Dockerfile")
|
||||
module(t, root, "absolute", "/etc/Dockerfile")
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var mods []string
|
||||
for _, f := range got {
|
||||
mods = append(mods, f.Module)
|
||||
}
|
||||
if want := []string{"absolute", "leaves"}; !reflect.DeepEqual(mods, want) {
|
||||
t.Errorf("refused %v (%v), want %v", mods, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMissingDockerfileFails(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "gone", "Dockerfile")
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Module != "gone" {
|
||||
t.Errorf("found %v, want gone refused", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it stands: this is the check itself, run here too.
|
||||
func TestTheCataloguesImagesBuildFromTheirOwnDirectories(t *testing.T) {
|
||||
got, err := Check("../..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("an image builds from outside its module: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# The modules the check covers (novox/hq ADR 0507 §9, issue 1838), one a line; issue 1838 widens it.
|
||||
pacman
|
||||
kernel
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/particulars
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,159 @@
|
||||
// Command particulars keeps this mesh's node names out of the modules it covers (novox/hq ADR 0507 §9,
|
||||
// issue 1838): a module is for any mesh, so no manifest, file, code string, identifier, comment, test name or
|
||||
// fixture of a covered module names a node of this one.
|
||||
//
|
||||
// The names are in the file `particulars` at the catalogue's root, one a line, `#` comments — the one place
|
||||
// they may be written. The modules covered are in `covered` at checks/particulars/covered, one a line. A name is matched
|
||||
// whole and in any case; a hyphen, an underscore or a CamelCase boundary parts one, so a longer word holding a name is another word.
|
||||
// The forge is not a node name: its host (git.novox.be) and the organisation path (novox/<repo>) are blanked
|
||||
// before a line is read.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 naming file:line and the word, 2 when a list cannot be read.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// forge is the forge's host and organisation path, blanked before a line is read; a path segment novox/ elsewhere
|
||||
// (testdata/novox/x, /srv/novox/data) is not blanked.
|
||||
var forge = regexp.MustCompile(`(?i)git\.novox\.be(?:/novox/[\w.-]+)?|(?:^|[^/\w.-])novox/[\w.-]+`)
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
os.Exit(check(root))
|
||||
}
|
||||
|
||||
// check runs the check over a catalogue and returns the exit code: 0 clean, 1 a finding, 2 a list unreadable.
|
||||
func check(root string) int {
|
||||
names, err := readList(filepath.Join(root, "particulars"))
|
||||
if err == nil && len(names) == 0 {
|
||||
err = fmt.Errorf("particulars lists no name — the check would pass on anything")
|
||||
}
|
||||
var covered []string
|
||||
if err == nil {
|
||||
covered, err = readList(filepath.Join(root, "checks/particulars/covered"))
|
||||
if err == nil && len(covered) == 0 {
|
||||
err = fmt.Errorf("covered lists no module — the check would pass on anything")
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "particulars:", err)
|
||||
return 2
|
||||
}
|
||||
findings, err := run(root, names, covered)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "particulars:", err)
|
||||
return 2
|
||||
}
|
||||
for _, f := range findings {
|
||||
fmt.Println(f)
|
||||
}
|
||||
if len(findings) > 0 {
|
||||
fmt.Printf("particulars: %d node name(s) in a module for any mesh (novox/hq ADR 0507 §9) — say what it shows, not where it was seen\n", len(findings))
|
||||
return 1
|
||||
}
|
||||
fmt.Printf("particulars: none of %d node names in the %d covered modules\n", len(names), len(covered))
|
||||
return 0
|
||||
}
|
||||
|
||||
// readList reads a list: one entry a line; blank lines and lines starting with # are not entries.
|
||||
func readList(path string) ([]string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
var out []string
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out, sc.Err()
|
||||
}
|
||||
|
||||
// run reads every text file of each covered module and returns the findings, sorted.
|
||||
func run(root string, names, covered []string) ([]string, error) {
|
||||
var out []string
|
||||
for _, m := range covered {
|
||||
dir := filepath.Join(root, "modules", m)
|
||||
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
switch d.Name() {
|
||||
case "node_modules", "dist", "vendor", ".git":
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(root, path)
|
||||
// A path is read too: a testdata directory named after a node is a finding.
|
||||
for _, n := range hits(filepath.ToSlash(rel), names) {
|
||||
out = append(out, fmt.Sprintf("%s:0: node name %q in the path", filepath.ToSlash(rel), n))
|
||||
}
|
||||
if d.Name() == "go.sum" {
|
||||
return nil
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if bytes.IndexByte(b, 0) >= 0 {
|
||||
return nil
|
||||
}
|
||||
out = append(out, scan(filepath.ToSlash(rel), string(b), names)...)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// scan returns a finding for each node name in src, whole-word and in any case.
|
||||
func scan(file, src string, names []string) []string {
|
||||
var out []string
|
||||
for i, line := range strings.Split(src, "\n") {
|
||||
for _, n := range hits(line, names) {
|
||||
out = append(out, fmt.Sprintf("%s:%d: node name %q — %s", file, i+1, n, strings.Join(strings.Fields(line), " ")))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hits returns the node names one line holds: as a word (a hyphen or underscore parts it), as a part of a
|
||||
// CamelCase identifier (TestAce, novoxTree), the forge's host and organisation path blanked first.
|
||||
func hits(line string, names []string) []string {
|
||||
line = forge.ReplaceAllStringFunc(line, func(s string) string { return strings.Repeat(" ", len(s)) })
|
||||
var out []string
|
||||
for _, n := range names {
|
||||
q := regexp.QuoteMeta(n)
|
||||
camel := regexp.QuoteMeta(strings.ToUpper(n[:1]) + n[1:])
|
||||
rx := regexp.MustCompile(`(?i:(?:^|[^A-Za-z0-9])` + q + `(?:$|[^A-Za-z0-9]))|[a-z0-9]` + camel + `(?:$|[^a-z])|(?:^|[^A-Za-z0-9])` + q + `[A-Z0-9]`)
|
||||
if rx.MatchString(line) {
|
||||
out = append(out, strings.ToLower(n))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var names = []string{"novox", "ace", "shanks", "g14"}
|
||||
|
||||
func TestANodeNameIsRefusedWhereItIsWritten(t *testing.T) {
|
||||
cases := map[string]struct{ file, src string }{
|
||||
"manifest string": {"modules/kernel/module.json", "{\n \"description\": \"as on Shanks\"\n}\n"},
|
||||
"file line": {"modules/kernel/files/boot.sh", "#!/bin/sh\necho run on g14\n"},
|
||||
"go identifier": {"modules/kernel/cmd/x/a.go", "package main\n\nfunc novox() {}\n"},
|
||||
"test name": {"modules/kernel/cmd/x/a_test.go", "package main\n\nfunc TestAce(t *testing.T) {}\n\nfunc TestOnAce_x(t *testing.T) {}\n"},
|
||||
}
|
||||
for name, c := range cases {
|
||||
if got := scan(c.file, c.src, names); len(got) == 0 {
|
||||
t.Errorf("%s: %q not refused", name, c.src)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFindingNamesTheLineAndTheWord(t *testing.T) {
|
||||
got := scan("f.sh", "a\nb\nrun on ACE now\n", names)
|
||||
if len(got) != 1 || !strings.HasPrefix(got[0], "f.sh:3:") || !strings.Contains(got[0], "ace") {
|
||||
t.Errorf("got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheForgeIsNotANodeName(t *testing.T) {
|
||||
for _, s := range []string{
|
||||
"see novox/hq issue 5",
|
||||
"module git.novox.be/novox/mesh-sdk/go v1.2.3",
|
||||
"https://git.novox.be/novox/mesh-catalog",
|
||||
"git.novox.be",
|
||||
"space trace surface grace g14x",
|
||||
} {
|
||||
if got := scan("f", s, names); len(got) != 0 {
|
||||
t.Errorf("%q refused: %v", s, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeNameInAPathOrJoinedIsRefused(t *testing.T) {
|
||||
for _, s := range []string{
|
||||
"modules/kernel/testdata/novox/meminfo",
|
||||
"/srv/novox/data",
|
||||
"testdata/shanks-meminfo.txt",
|
||||
"on-shanks",
|
||||
"shanks-desktop",
|
||||
"mesh-novox",
|
||||
"func novoxTree() {}",
|
||||
"aceFiles := 1",
|
||||
"g14Meminfo := 2",
|
||||
"novox_tree := 3",
|
||||
} {
|
||||
if got := scan("f", s, names); len(got) == 0 {
|
||||
t.Errorf("%q not refused", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeNamedPathIsRefusedThroughRun(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
p := filepath.Join(root, "modules/kernel/testdata/ace/x")
|
||||
os.MkdirAll(filepath.Dir(p), 0o755)
|
||||
os.WriteFile(p, []byte("clean\n"), 0o644)
|
||||
got, err := run(root, names, []string{"kernel"})
|
||||
if err != nil || len(got) != 1 || !strings.HasPrefix(got[0], "modules/kernel/testdata/ace/x:0: ") {
|
||||
t.Errorf("got %v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckReturnsOneOnAFindingAndTwoOnAnUnreadableList(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
write := func(p, s string) {
|
||||
p = filepath.Join(root, p)
|
||||
os.MkdirAll(filepath.Dir(p), 0o755)
|
||||
os.WriteFile(p, []byte(s), 0o644)
|
||||
}
|
||||
write("particulars", "ace\n")
|
||||
write("checks/particulars/covered", "kernel\n")
|
||||
write("modules/kernel/a", "clean\n")
|
||||
if got := check(root); got != 0 {
|
||||
t.Errorf("clean = %d, want 0", got)
|
||||
}
|
||||
write("modules/kernel/a", "on ace\n")
|
||||
if got := check(root); got != 1 {
|
||||
t.Errorf("finding = %d, want 1", got)
|
||||
}
|
||||
os.Remove(filepath.Join(root, "particulars"))
|
||||
if got := check(root); got != 2 {
|
||||
t.Errorf("no list = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyCoveredModulesAreRead(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
write := func(p, s string) {
|
||||
p = filepath.Join(root, p)
|
||||
os.MkdirAll(filepath.Dir(p), 0o755)
|
||||
os.WriteFile(p, []byte(s), 0o644)
|
||||
}
|
||||
write("modules/kernel/files/a", "on novox\n")
|
||||
write("modules/other/files/a", "on novox\n")
|
||||
got, err := run(root, names, []string{"kernel"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || !strings.Contains(got[0], "modules/kernel/files/a:1") {
|
||||
t.Errorf("got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// Command toolchains refuses any manifest but the owner's declaring a language's toolchain package (novox/hq
|
||||
// ADR 0477 clause 4, issue 1573): the package go only in modules/go, and nodejs and npm only in modules/nodejs.
|
||||
// ADR 0477 clause 4, issue 1573): the package go only in modules/go, nodejs and npm only in modules/nodejs, and buildah (issue 1702) only in modules/buildah.
|
||||
// Everything that compiles or runs that language requires the owner's provision instead (go-toolchain,
|
||||
// node-toolchain), so the package is declared once, and a node without the owner refuses the module that
|
||||
// needs it at resolution rather than failing at its first build. The owners must declare their packages.
|
||||
@@ -19,9 +19,10 @@ import (
|
||||
|
||||
// Owners names, for each toolchain package, the one module that declares it.
|
||||
var Owners = map[string]string{
|
||||
"go": "go",
|
||||
"nodejs": "nodejs",
|
||||
"npm": "nodejs",
|
||||
"buildah": "buildah",
|
||||
"go": "go",
|
||||
"nodejs": "nodejs",
|
||||
"npm": "nodejs",
|
||||
}
|
||||
|
||||
// Finding is one refusal: Module declares Package, which Owner alone may, or (Missing) Owner does not declare
|
||||
@@ -100,5 +101,5 @@ func main() {
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("toolchains: go, nodejs and npm are declared only by their owners")
|
||||
fmt.Println("toolchains: go, nodejs, npm and buildah are declared only by their owners")
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ func TestOnlyTheOwnerDeclaresAToolchainPackage(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{
|
||||
"go": `{"module":"go","resources":[{"id":"go","type":"package","package":"go"}]}`,
|
||||
"nodejs": `{"module":"nodejs","resources":[{"id":"nodejs","type":"package","package":"nodejs"},{"id":"npm","type":"package","package":"npm"}]}`,
|
||||
"buildah": `{"module":"buildah","resources":[{"id":"b","type":"package","package":"buildah"}]}`,
|
||||
"lab": `{"module":"lab","resources":[{"id":"git","type":"package","package":"git"},{"id":"go","type":"package","package":"go"}]}`,
|
||||
"builder": `{"module":"builder","resources":[{"id":"n","type":"package","package":"nodejs"},{"id":"dir","type":"directory","path":"/go"}]}`,
|
||||
})
|
||||
@@ -39,6 +40,23 @@ func TestOnlyTheOwnerDeclaresAToolchainPackage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecondDeclarerOfBuildahIsRefused(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{
|
||||
"go": `{"module":"go","resources":[{"id":"go","type":"package","package":"go"}]}`,
|
||||
"nodejs": `{"module":"nodejs","resources":[{"id":"nodejs","type":"package","package":"nodejs"},{"id":"npm","type":"package","package":"npm"}]}`,
|
||||
"buildah": `{"module":"buildah","resources":[{"id":"b","type":"package","package":"buildah"}]}`,
|
||||
"imagery": `{"module":"imagery","resources":[{"id":"b","type":"package","package":"buildah"}]}`,
|
||||
})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []Finding{{Module: "imagery", Package: "buildah", Owner: "buildah"}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("found %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheOwnersMustExistAndDeclareTheirPackages(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{
|
||||
"go": `{"module":"go","resources":[]}`,
|
||||
@@ -47,8 +65,8 @@ func TestTheOwnersMustExistAndDeclareTheirPackages(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Errorf("an owner missing its packages gave %v, want three findings (go, nodejs, npm)", got)
|
||||
if len(got) != 4 {
|
||||
t.Errorf("an owner missing its packages gave %v, want four findings (buildah, go, nodejs, npm)", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -50,11 +50,20 @@
|
||||
# every directory and archive says `unknown` (move-out, module or program) and a valid `knows`; `move-out`
|
||||
# only under a home; and no module declares a path inside a directory of another module.
|
||||
# checks/sshdir (ADR 0415 §6) stays while main runs it; it goes under issue 1035 once main runs checks/owned.
|
||||
# 12. the image builder (novox/hq ADR 0487 §3, issue 1704): only modules/buildah provides rootless-image-build.
|
||||
# 13. the image's place (novox/hq ADR 0487 §1, issue 1707): every image builds from a Dockerfile in its own module's directory.
|
||||
set -eu
|
||||
|
||||
(cd checks/words && go test -count=1 . && go run . ../..)
|
||||
# Node names (novox/hq ADR 0507 §9, issue 1838): no node name of this mesh in a covered module; the names are in
|
||||
# `particulars`, the covered modules in checks/particulars/covered.
|
||||
(cd checks/particulars && go test -count=1 . && go run . ../..)
|
||||
# The toolchains (novox/hq ADR 0477 clause 4): the package go only in modules/go, nodejs and npm only in modules/nodejs.
|
||||
(cd checks/toolchains && go test -count=1 . && go run . ../..)
|
||||
# The image builder (novox/hq ADR 0487 §3): only modules/buildah provides rootless-image-build, docker never.
|
||||
(cd checks/imagebuild && go test -count=1 . && go run . ../..)
|
||||
# The image's place (novox/hq ADR 0487 §1): every image builds from a Dockerfile in its own module's directory.
|
||||
(cd checks/imageplace && go test -count=1 . && go run . ../..)
|
||||
# The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
|
||||
(cd checks/agenthome && go test -count=1 . && go run . ../..)
|
||||
# The mail-store seat (novox/hq ADR 0378 §3): only mesh-mail may claim it.
|
||||
@@ -190,6 +199,19 @@ for m in $touched; do
|
||||
# another process) run here and nowhere else: the unit suite stays under a second (novox/hq playbook 07).
|
||||
(cd "modules/$m" && CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go vet -tags integration ${modfile:+"-modfile=$modfile"} ./... &&
|
||||
CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go test -tags integration ${modfile:+"-modfile=$modfile"} $race -count=1 ./...)
|
||||
# The docker tests of the bus's snapshot program and entrypoint (novox/hq ADR 0480 clause 6) start throwaway nats
|
||||
# containers, and the Go toolchain container holds no docker, so above they skip with a "NOT RUN" line. Where the
|
||||
# build seat says docker exists (MESH_CHECK_DOCKER=1) they run here by name, and a missing docker fails the check
|
||||
# (MESH_REQUIRE_DOCKER=1 makes the tests fail instead of skip). Without it the step says it did not run.
|
||||
if [ "$m" = nats ]; then
|
||||
if [ "${MESH_CHECK_DOCKER:-}" = 1 ]; then
|
||||
(cd modules/nats && MESH_REQUIRE_DOCKER=1 CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go test -tags integration ${modfile:+"-modfile=$modfile"} -count=1 \
|
||||
-run 'TestTheDeclaredDumpRunsInTheBussImage|TestTheEntrypointReloadsTheServerWithoutDroppingAConnection|TestTheStoreSurvivesTheNewDeclarationTheOldAndTheNewAgain|TestASnapshotOfALiveBusRestoresToTheSameContent|TestTheSnapshotUserCannotChangeTheBus' \
|
||||
./cmd/mesh-nats-snapshot) || { echo "the bus's docker tests failed"; exit 1; }
|
||||
else
|
||||
echo "NOT RUN: the bus's five docker tests (modules/nats/cmd/mesh-nats-snapshot) skipped: this check has no docker (MESH_CHECK_DOCKER is not 1)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
if [ -n "$untestable" ]; then
|
||||
echo "the check fails: the tests of$untestable could not run (novox/hq issue 385)"
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# buildah
|
||||
|
||||
The rootless image builder as a module (novox/hq ADR 0487, issue 1702).
|
||||
|
||||
- **The only module that declares the package `buildah`** (`checks/toolchains`), and the only one that provides
|
||||
`rootless-image-build` (`checks/imagebuild`; docker never does).
|
||||
- **Provides `rootless-image-build` with the machine's reach**: mesh-build-image requires it, so it composes only on a
|
||||
machine where this module is assigned, and a node without it is refused at resolution naming `buildah`.
|
||||
- **No account and no subordinate-id word.** The node-engine makes every `user` account with `useradd`, which
|
||||
allocates the account 65536 subordinate uids and gids; mesh-build-image's own account gets its ranges that way and
|
||||
its health proves them (it reads /etc/subuid and /etc/subgid).
|
||||
- The helpers (an OCI runtime, passt, containers-common) come as buildah's package dependencies.
|
||||
- Holds no long-running resource, so it adds nothing to `health-undeclared`.
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"module": "buildah",
|
||||
"version": "1",
|
||||
"capabilities": [
|
||||
"package-manager",
|
||||
"user-namespaces"
|
||||
],
|
||||
"provides": [
|
||||
{
|
||||
"name": "rootless-image-build",
|
||||
"reach": "machine"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "buildah",
|
||||
"type": "package",
|
||||
"package": "buildah"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package main
|
||||
|
||||
// Every call a hook prints for a session to copy must be accepted by claude_code_wait's own argument check
|
||||
// (issue 1757): the Stop hook's suggested watched and said calls, and the SessionStart hook's suggested
|
||||
// adopt call.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// suggestedCall extracts the one JSON object embedded in a hook's message, right after "claude_code_wait "
|
||||
// — what a session is expected to copy verbatim — and parses it into the arguments a tool call takes.
|
||||
func suggestedCall(t *testing.T, msg string) map[string]any {
|
||||
t.Helper()
|
||||
i := strings.Index(msg, "claude_code_wait ")
|
||||
if i < 0 {
|
||||
t.Fatalf("no suggested call in: %s", msg)
|
||||
}
|
||||
rest := msg[i+len("claude_code_wait "):]
|
||||
start := strings.Index(rest, "{")
|
||||
if start < 0 {
|
||||
t.Fatalf("no JSON object in: %s", msg)
|
||||
}
|
||||
depth, end := 0, -1
|
||||
for j, c := range rest[start:] {
|
||||
switch c {
|
||||
case '{':
|
||||
depth++
|
||||
case '}':
|
||||
depth--
|
||||
if depth == 0 {
|
||||
end = start + j + 1
|
||||
}
|
||||
}
|
||||
if end >= 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if end < 0 {
|
||||
t.Fatalf("unbalanced JSON object in: %s", msg)
|
||||
}
|
||||
var args map[string]any
|
||||
if err := json.Unmarshal([]byte(rest[start:end]), &args); err != nil {
|
||||
t.Fatalf("the suggested call is not valid JSON: %v (%s)", err, rest[start:end])
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// Issue 1757: the Stop hook's suggested watched call carries `by`, and the tool accepts it as printed.
|
||||
func TestTheStopHooksSuggestedWatchedCallIsAcceptedByTheTool(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
store := newMemWaits()
|
||||
waiting = &Waiting{Store: store, Now: func() time.Time { return t0 }}
|
||||
t.Cleanup(func() { waiting = nil })
|
||||
s := serve(t, p, &bus{kept: map[string]map[string]any{}}, memConfig{})
|
||||
|
||||
text, isErr := s.call(t, "claude_code_wait", map[string]any{"op": "add", "session": "verify-0489-test", "by": "fix-1757@g14",
|
||||
"what": "the operator's Approve", "read": map[string]any{"address": "mesh-delivery.recheck"}, "bound_minutes": 10})
|
||||
if isErr {
|
||||
t.Fatalf("add: %s", text)
|
||||
}
|
||||
|
||||
f, err := readWaitsFile(waitsFilePath(p, "verify-0489-test"))
|
||||
if err != nil || len(f.Waits) != 1 {
|
||||
t.Fatalf("%+v %v", f, err)
|
||||
}
|
||||
_, reason := waitBlocksStop("verify-0489-test", f.Waits[0], t0.Add(time.Minute))
|
||||
if !strings.Contains(reason, "\"by\":\"fix-1757\"") {
|
||||
t.Fatalf("the suggested call does not carry by: %s", reason)
|
||||
}
|
||||
args := suggestedCall(t, reason)
|
||||
|
||||
text, isErr = s.call(t, "claude_code_wait", args)
|
||||
if isErr {
|
||||
t.Fatalf("the suggested watched call, as printed, was refused: %s", text)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1757: the Stop hook's suggested said call (past its bound) carries `by` too.
|
||||
func TestTheStopHooksSuggestedSaidCallIsAcceptedByTheTool(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
store := newMemWaits()
|
||||
clock := t0
|
||||
waiting = &Waiting{Store: store, Now: func() time.Time { return clock }}
|
||||
t.Cleanup(func() { waiting = nil })
|
||||
s := serve(t, p, &bus{kept: map[string]map[string]any{}}, memConfig{})
|
||||
|
||||
text, isErr := s.call(t, "claude_code_wait", map[string]any{"op": "add", "session": "verify-0489-test", "by": "fix-1757@g14",
|
||||
"what": "the operator's Approve", "read": map[string]any{"address": "mesh-delivery.recheck"}, "bound_minutes": 1})
|
||||
if isErr {
|
||||
t.Fatalf("add: %s", text)
|
||||
}
|
||||
|
||||
clock = t0.Add(2 * time.Minute) // past its 1-minute bound, for the live tool call too
|
||||
f, err := readWaitsFile(waitsFilePath(p, "verify-0489-test"))
|
||||
if err != nil || len(f.Waits) != 1 {
|
||||
t.Fatalf("%+v %v", f, err)
|
||||
}
|
||||
_, reason := waitBlocksStop("verify-0489-test", f.Waits[0], clock)
|
||||
if !strings.Contains(reason, "\"by\":\"fix-1757\"") {
|
||||
t.Fatalf("the suggested call does not carry by: %s", reason)
|
||||
}
|
||||
args := suggestedCall(t, reason)
|
||||
|
||||
text, isErr = s.call(t, "claude_code_wait", args)
|
||||
if isErr {
|
||||
t.Fatalf("the suggested said call, as printed, was refused: %s", text)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1757: the SessionStart hook's suggested adopt call already carries `by` — checked, as asked, so a
|
||||
// regression here is caught the same way as the Stop hook's.
|
||||
func TestTheSessionStartHooksSuggestedAdoptCallIsAcceptedByTheTool(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
store := newMemWaits()
|
||||
waiting = &Waiting{Store: store, Now: func() time.Time { return t0 }}
|
||||
t.Cleanup(func() { waiting = nil })
|
||||
s := serve(t, p, &bus{kept: map[string]map[string]any{}}, memConfig{})
|
||||
|
||||
text, isErr := s.call(t, "claude_code_wait", map[string]any{"op": "add", "session": "sess-old", "by": "fix-1757@g14",
|
||||
"what": "the operator's Approve", "read": map[string]any{"address": "mesh-delivery.recheck"}, "bound_minutes": 10})
|
||||
if isErr {
|
||||
t.Fatalf("add: %s", text)
|
||||
}
|
||||
|
||||
msg := SessionStartMessage("sess-new", WaitsDirPath(p), t0.Add(time.Minute))
|
||||
if !strings.Contains(msg, "\"by\":\"fix-1757@g14\"") {
|
||||
t.Fatalf("the suggested adopt call does not carry by: %s", msg)
|
||||
}
|
||||
args := suggestedCall(t, msg)
|
||||
|
||||
text, isErr = s.call(t, "claude_code_wait", args)
|
||||
if isErr {
|
||||
t.Fatalf("the suggested adopt call, as printed, was refused: %s", text)
|
||||
}
|
||||
}
|
||||
@@ -52,12 +52,12 @@ func waitBlocksStop(session string, w Wait, now time.Time) (bool, string) {
|
||||
}
|
||||
return true, fmt.Sprintf("wait %s (%s) is open and nothing follows it yet. Read where it stands with %s, then "+
|
||||
"%s, and mark it watched: claude_code_wait "+
|
||||
"{\"op\":\"watched\",\"session\":%q,\"id\":%q,\"watcher\":\"...\"}.", w.ID, w.What, w.Read.Address, watcherAdvice(w),
|
||||
session, w.ID)
|
||||
"{\"op\":\"watched\",\"session\":%q,\"by\":%q,\"id\":%q,\"watcher\":\"...\"}.", w.ID, w.What, w.Read.Address,
|
||||
watcherAdvice(w), session, w.By, w.ID)
|
||||
}
|
||||
return true, fmt.Sprintf("wait %s (%s) is past its bound and the operator has not been told. Tell the operator what "+
|
||||
"waits, since when, and what you are doing about it, then mark it: claude_code_wait "+
|
||||
"{\"op\":\"said\",\"session\":%q,\"id\":%q}.", w.ID, w.What, session, w.ID)
|
||||
"{\"op\":\"said\",\"session\":%q,\"by\":%q,\"id\":%q}.", w.ID, w.What, session, w.By, w.ID)
|
||||
}
|
||||
|
||||
// watcherAdvice is what a session is told to do to follow a wait (ADR 0489 §3, review round 1 item 3): a
|
||||
@@ -113,15 +113,34 @@ func readRefusalCounters(waitsDir, session string) map[string]int {
|
||||
return m
|
||||
}
|
||||
|
||||
// writeRefusalCounters writes the counter file beside the waits file. Its caller decides what an error here
|
||||
// means: harmless when the stop is already allowed, but when the stop was about to be refused, a cap that
|
||||
// cannot be written is a cap that cannot take effect (ADR 0489 §5 "never loops", review round 1 item 2).
|
||||
// writeRefusalCounters keeps the counter file beside the waits file in step with m, through fsys like every
|
||||
// other write into the agent's home (novox/hq ADR 0266, issue 1749): called from the Stop hook, which
|
||||
// already runs as the agent's own account (fsys is then its zero value, a plain write or remove, as
|
||||
// before), and from the tool's AdoptName, which runs as the operator's account and must reach a confined
|
||||
// node's agent home through the sudo helper fsys carries such a write through. Its caller decides what an
|
||||
// error here means: harmless when the stop is already allowed, but when the stop was about to be refused, a
|
||||
// cap that cannot be written is a cap that cannot take effect (ADR 0489 §5 "never loops", review round 1
|
||||
// item 2).
|
||||
//
|
||||
// Empty, m is removed rather than written as "{}" (issue 1749, follow-up): a stop with nothing to record —
|
||||
// every stop of every session with no waits, the ordinary case — must leave neither the file nor, the first
|
||||
// time, the waits folder itself; removing a file that was never there is not an error.
|
||||
func writeRefusalCounters(waitsDir, session string, m map[string]int) error {
|
||||
path := refusalCountersPath(waitsDir, session)
|
||||
if len(m) == 0 {
|
||||
if err := fsys.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(refusalCountersPath(waitsDir, session), raw, 0o600)
|
||||
if err := fsys.MkdirAll(waitsDir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
return fsys.WriteFile(path, raw, 0o600)
|
||||
}
|
||||
|
||||
// notice says a fail-open or let-through reason on both the log (stderr) and the screen: Claude Code shows a
|
||||
|
||||
@@ -170,13 +170,72 @@ func TestRunStopKeepsItsCounterBesideTheWaitsFile(t *testing.T) {
|
||||
if !strings.Contains(stdout.String(), "systemMessage") || !strings.Contains(stdout.String(), "unwatched") {
|
||||
t.Fatalf("the let-through notice did not reach stdout: %s", stdout.String())
|
||||
}
|
||||
raw, err := os.ReadFile(refusalCountersPath(waitsDir, "sess-6"))
|
||||
// Once let through, the counters are empty again: the file is removed, not left holding "{}" (issue
|
||||
// 1749, follow-up).
|
||||
if _, err := os.Stat(refusalCountersPath(waitsDir, "sess-6")); !os.IsNotExist(err) {
|
||||
t.Fatalf("the counter file was not removed once its counters emptied: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1749, follow-up: a stop for a session with no waits at all leaves nothing behind — not the counter
|
||||
// file, not even the waits folder a node that never added a wait would otherwise gain on every stop of
|
||||
// every session.
|
||||
func TestStopWithNoWaitsLeavesNoFolderAndNoFile(t *testing.T) {
|
||||
waitsDir := filepath.Join(t.TempDir(), "waits") // not created: this is what the hook is handed cold
|
||||
var stdout, stderr bytes.Buffer
|
||||
stdin := strings.NewReader(`{"session_id":"sess-none","stop_hook_active":false}`)
|
||||
if code := runStop(waitsDir, stdin, &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("code %d, stderr %s", code, stderr.String())
|
||||
}
|
||||
if _, err := os.Stat(waitsDir); !os.IsNotExist(err) {
|
||||
t.Fatalf("a stop with no waits created the waits folder: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1749, follow-up: a refusal writes the counter file (there is something to record).
|
||||
func TestStopWithARefusalWritesTheCounterFile(t *testing.T) {
|
||||
waitsDir := t.TempDir()
|
||||
f := WaitsFile{Session: "sess-7", At: stopT0, Waits: []Wait{openWait("w-7", 10)}}
|
||||
writeWaitsFixture(t, filepath.Join(waitsDir, "sess-7.json"), f)
|
||||
var stdout, stderr bytes.Buffer
|
||||
stdin := strings.NewReader(`{"session_id":"sess-7","stop_hook_active":false}`)
|
||||
if code := runStop(waitsDir, stdin, &stdout, &stderr); code != 2 {
|
||||
t.Fatalf("code %d, stderr %s", code, stderr.String())
|
||||
}
|
||||
raw, err := os.ReadFile(refusalCountersPath(waitsDir, "sess-7"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var counters map[string]int
|
||||
if json.Unmarshal(raw, &counters) != nil || len(counters) != 0 {
|
||||
t.Fatalf("the counter file was not reset: %s", raw)
|
||||
if json.Unmarshal(raw, &counters) != nil || counters["w-7"] != 1 {
|
||||
t.Fatalf("the counter file: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1749, follow-up: once the wait is done (no longer in the waits file), the next stop removes the
|
||||
// counter file rather than leaving a stale "{}" behind.
|
||||
func TestStopAfterTheWaitIsDoneRemovesTheCounterFile(t *testing.T) {
|
||||
waitsDir := t.TempDir()
|
||||
path := filepath.Join(waitsDir, "sess-8.json")
|
||||
writeWaitsFixture(t, path, WaitsFile{Session: "sess-8", At: stopT0, Waits: []Wait{openWait("w-8", 10)}})
|
||||
stdin := strings.NewReader(`{"session_id":"sess-8","stop_hook_active":false}`)
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := runStop(waitsDir, stdin, &stdout, &stderr); code != 2 {
|
||||
t.Fatalf("refusal: code %d, stderr %s", code, stderr.String())
|
||||
}
|
||||
if _, err := os.Stat(refusalCountersPath(waitsDir, "sess-8")); err != nil {
|
||||
t.Fatalf("the counter file was not written: %v", err)
|
||||
}
|
||||
|
||||
// The wait is done: it is no longer in the waits file (DoneWait removes it, waits.go).
|
||||
writeWaitsFixture(t, path, WaitsFile{Session: "sess-8", At: stopT0, Waits: []Wait{}})
|
||||
var stdout2, stderr2 bytes.Buffer
|
||||
stdin2 := strings.NewReader(`{"session_id":"sess-8","stop_hook_active":false}`)
|
||||
if code := runStop(waitsDir, stdin2, &stdout2, &stderr2); code != 0 {
|
||||
t.Fatalf("after done: code %d, stderr %s", code, stderr2.String())
|
||||
}
|
||||
if _, err := os.Stat(refusalCountersPath(waitsDir, "sess-8")); !os.IsNotExist(err) {
|
||||
t.Fatalf("the counter file was not removed after the wait was done: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -273,3 +332,31 @@ func TestHookRegisterStillRefusesEveryHook(t *testing.T) {
|
||||
t.Fatalf("a Stop hook was not refused: %v %s", isError, text)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1749: adopt's refusal-counter reset writes into the agent's home (beside the waits file) and must
|
||||
// go through fsys like every other such write (novox/hq ADR 0266) — on a confined node the module runs as
|
||||
// the operator's account, which cannot write there directly; only the helper, as the agent's own account,
|
||||
// can. The confined-node fake (agentFiles, agent_account_test.go) proves it: a write that reaches the real
|
||||
// filesystem only through the helper's request/response protocol bumps its count.
|
||||
func TestWriteRefusalCountersGoesThroughTheHelper(t *testing.T) {
|
||||
f, home, counted := agentFiles(t)
|
||||
was := fsys
|
||||
fsys = f
|
||||
t.Cleanup(func() { fsys = was })
|
||||
|
||||
waitsDir := filepath.Join(home, ".claude", "nox-mesh", "waits")
|
||||
if err := writeRefusalCounters(waitsDir, "sess-1", map[string]int{"w-1": 2}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if counted.asked.Load() == 0 {
|
||||
t.Fatal("the counter file was written directly, never through the helper")
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(waitsDir, "sess-1.refusals.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m map[string]int
|
||||
if json.Unmarshal(raw, &m) != nil || m["w-1"] != 2 {
|
||||
t.Fatalf("the written counters: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ machine's resources a seat, these tools become its verbs.
|
||||
|
||||
## Tools
|
||||
|
||||
All five only read.
|
||||
All six only read.
|
||||
|
||||
| tool | what | replaces |
|
||||
|---|---|---|
|
||||
@@ -32,6 +32,7 @@ All five only read.
|
||||
| `disk_top` | Over a window. `by=unit` (the default) ranks systemd units and containers by bytes moved, from their cgroups. `by=container` ranks containers by name, and `by=process` ranks processes | `iotop` |
|
||||
| `disk_filesystems` | Each filesystem that stores data, said once however many places it is mounted: size, used and free GiB, used % as `df` counts it, inodes used % | `df -h`, `df -i` |
|
||||
| `disk_devices` | Each disk without sampling: model, size, whether it spins, scheduler, queue limit, what it is built on, and since boot the GiB read and written, the busy share and the wait per read and per write | `lsblk`, `cat /proc/diskstats` |
|
||||
| `disk_folders` | Which folders fill a disk, like `du -x`: the largest folders under `path` (default `/`) to `depth` levels (default 2, at most 3), `limit` of them (default 20, at most 100), with bytes and GiB. Sizes are allocated bytes, a hard link once. It stays on the path's one filesystem: a mount below it is named, never entered. It reads every folder the tool runner can read (docker's included) and no content; folders it cannot read are counted and named. After 25 s (the tool runner gives a call 30 s) it answers what it has measured, marked `partial`, with `stopped_at` (novox/hq issue 1804) | `du -x`, `ncdu` |
|
||||
| `cpu_top` | Samples a window (`seconds`, default 3, at most 10). It answers the machine's CPU in use (% of all CPUs and as CPUs busy, from `/proc/stat`), then `by=unit` (the default) ranks systemd units and containers by CPU, from their cgroups' cpu.stat, busiest first (`limit`, default 15), with the busy time no unit accounts for; `by=container` ranks containers by name | `top`, `systemd-cgtop` |
|
||||
|
||||
### How each number is read
|
||||
@@ -87,4 +88,5 @@ cover:
|
||||
- processes ranked, one that started inside the window left out, unreadable ones counted;
|
||||
- filesystems said once, mount escapes undone, a statfs that fails said;
|
||||
- the window's bounds;
|
||||
- folders ranked biggest first to the depth asked, a mount never entered, unreadable folders named, a walk past its bound answered as partial with where it stopped, a hard link counted once;
|
||||
- the manifest: tools listed equal tools served, all read-only, no resources, no machine named.
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FolderBudget is how long a folder walk may take before it answers what it has measured.
|
||||
const FolderBudget = 25 * time.Second
|
||||
|
||||
// namedUnreadable is how many unreadable folders are named; the rest are only counted.
|
||||
const namedUnreadable = 50
|
||||
|
||||
// Folder is one folder's size: everything under it on the same filesystem.
|
||||
type Folder struct {
|
||||
Path string `json:"path"`
|
||||
Depth int `json:"depth"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
GiB float64 `json:"gib"`
|
||||
}
|
||||
|
||||
// FolderReport is the answer of disk_folders.
|
||||
type FolderReport struct {
|
||||
Path string `json:"path"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
GiB float64 `json:"gib"`
|
||||
Largest []Folder `json:"largest"`
|
||||
Partial bool `json:"partial"`
|
||||
StoppedAt string `json:"stopped_at,omitempty"`
|
||||
Unreadable int `json:"unreadable"`
|
||||
UnreadableFolders []string `json:"unreadable_folders,omitempty"`
|
||||
Skipped []string `json:"other_filesystems_skipped,omitempty"`
|
||||
Said []string `json:"said"`
|
||||
}
|
||||
|
||||
func (m *Machine) readDir(real string) ([]os.DirEntry, error) {
|
||||
if m.ReadDir != nil {
|
||||
return m.ReadDir(real)
|
||||
}
|
||||
return os.ReadDir(real)
|
||||
}
|
||||
|
||||
func (m *Machine) now() time.Time {
|
||||
if m.Clock != nil {
|
||||
return m.Clock()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func devOf(path string, info fs.FileInfo) uint64 {
|
||||
if st, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||
return uint64(st.Dev)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type inodeKey struct{ dev, ino uint64 }
|
||||
|
||||
// walk is one folder measurement.
|
||||
type walk struct {
|
||||
m *Machine
|
||||
maxDepth int
|
||||
dev uint64
|
||||
deadline time.Time
|
||||
seen map[inodeKey]bool
|
||||
folders []Folder
|
||||
bad []string
|
||||
skipped []string
|
||||
stopped string
|
||||
}
|
||||
|
||||
// Folders measures path like du -x: allocated bytes, a hard link once, never entering another filesystem.
|
||||
func (m *Machine) Folders(path string, depth, limit int) (FolderReport, error) {
|
||||
if !filepath.IsAbs(path) {
|
||||
return FolderReport{}, fmt.Errorf("path must be absolute, not %q", path)
|
||||
}
|
||||
path = filepath.Clean(path)
|
||||
info, err := os.Stat(m.path(path))
|
||||
if err != nil {
|
||||
return FolderReport{}, fmt.Errorf("path %s: %w", path, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return FolderReport{}, fmt.Errorf("path %s is not a folder", path)
|
||||
}
|
||||
budget := m.Budget
|
||||
if budget == 0 {
|
||||
budget = FolderBudget
|
||||
}
|
||||
w := &walk{m: m, maxDepth: depth, deadline: m.now().Add(budget), seen: map[inodeKey]bool{}}
|
||||
w.dev = w.device(path, info)
|
||||
total := w.dir(path, 0)
|
||||
|
||||
sort.SliceStable(w.folders, func(i, j int) bool {
|
||||
if w.folders[i].Bytes != w.folders[j].Bytes {
|
||||
return w.folders[i].Bytes > w.folders[j].Bytes
|
||||
}
|
||||
return w.folders[i].Path < w.folders[j].Path
|
||||
})
|
||||
if len(w.folders) > limit {
|
||||
w.folders = w.folders[:limit]
|
||||
}
|
||||
sort.Strings(w.bad)
|
||||
named := w.bad
|
||||
if len(named) > namedUnreadable {
|
||||
named = named[:namedUnreadable]
|
||||
}
|
||||
r := FolderReport{
|
||||
Path: path, Bytes: total, GiB: gibOf(total), Largest: w.folders,
|
||||
Partial: w.stopped != "", StoppedAt: w.stopped,
|
||||
Unreadable: len(w.bad), UnreadableFolders: named, Skipped: w.skipped,
|
||||
}
|
||||
if r.Largest == nil {
|
||||
r.Largest = []Folder{}
|
||||
}
|
||||
var said []string
|
||||
if r.Partial {
|
||||
said = append(said, fmt.Sprintf("PARTIAL: the walk stopped after %s at %s; sizes are what was measured so far", budget, r.StoppedAt))
|
||||
}
|
||||
said = append(said, fmt.Sprintf("%s holds %.1f GiB on its own filesystem", path, r.GiB))
|
||||
if r.Unreadable > 0 {
|
||||
said = append(said, fmt.Sprintf("%d folders could not be read (%s), so their size is missing", r.Unreadable, strings.Join(named, ", ")))
|
||||
}
|
||||
if len(r.Skipped) > 0 {
|
||||
said = append(said, "other filesystems not entered: "+strings.Join(r.Skipped, ", "))
|
||||
}
|
||||
r.Said = said
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (w *walk) device(path string, info fs.FileInfo) uint64 {
|
||||
if w.m.DevOf != nil {
|
||||
return w.m.DevOf(path, info)
|
||||
}
|
||||
return devOf(path, info)
|
||||
}
|
||||
|
||||
// dir is the allocated bytes under a folder; it records the folder when it is within the depth.
|
||||
func (w *walk) dir(path string, depth int) int64 {
|
||||
if w.stopped != "" {
|
||||
return 0
|
||||
}
|
||||
if w.m.now().After(w.deadline) {
|
||||
w.stopped = path
|
||||
return 0
|
||||
}
|
||||
entries, err := w.m.readDir(w.m.path(path))
|
||||
if err != nil {
|
||||
w.bad = append(w.bad, path)
|
||||
return 0
|
||||
}
|
||||
var total int64
|
||||
for _, e := range entries {
|
||||
child := filepath.Join(path, e.Name())
|
||||
info, err := os.Lstat(w.m.path(child))
|
||||
if err != nil {
|
||||
continue // gone since it was listed
|
||||
}
|
||||
if info.IsDir() {
|
||||
if w.device(child, info) != w.dev {
|
||||
w.skipped = append(w.skipped, child)
|
||||
continue
|
||||
}
|
||||
total += w.allocated(info, false) + w.dir(child, depth+1)
|
||||
continue
|
||||
}
|
||||
total += w.allocated(info, true)
|
||||
}
|
||||
if depth >= 1 && depth <= w.maxDepth {
|
||||
w.folders = append(w.folders, Folder{Path: path, Depth: depth, Bytes: total, GiB: gibOf(total)})
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// allocated is the bytes an entry occupies on disk, as du counts them; a file with several links once.
|
||||
func (w *walk) allocated(info fs.FileInfo, file bool) int64 {
|
||||
st, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return info.Size()
|
||||
}
|
||||
if file && st.Nlink > 1 {
|
||||
k := inodeKey{uint64(st.Dev), uint64(st.Ino)}
|
||||
if w.seen[k] {
|
||||
return 0
|
||||
}
|
||||
w.seen[k] = true
|
||||
}
|
||||
return st.Blocks * 512
|
||||
}
|
||||
|
||||
func gibOf(b int64) float64 { return math.Round(float64(b)/gib*100) / 100 }
|
||||
@@ -0,0 +1,170 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const mb = 1 << 20
|
||||
|
||||
// megabytes writes a file of n MiB under the fake root.
|
||||
func (f *fake) megabytes(path string, n int) { f.file(path, strings.Repeat("x", n*mb)) }
|
||||
|
||||
func aFullDisk(t *testing.T) *fake {
|
||||
f := newFake(t)
|
||||
f.megabytes("var/lib/docker/overlay2/a/layer", 8)
|
||||
f.megabytes("var/lib/docker/overlay2/b/layer", 4)
|
||||
f.megabytes("var/log/big.log", 3)
|
||||
f.megabytes("home/jo/video.mkv", 6)
|
||||
f.megabytes("home/jo/deep/er/est/file", 1)
|
||||
f.dir("empty")
|
||||
return f
|
||||
}
|
||||
|
||||
func folders(t *testing.T, m *Machine, args map[string]any) FolderReport {
|
||||
t.Helper()
|
||||
out, err := tool(t, m, "disk_folders").Run(args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.(FolderReport)
|
||||
}
|
||||
|
||||
func paths(r FolderReport) string {
|
||||
var p []string
|
||||
for _, f := range r.Largest {
|
||||
p = append(p, f.Path)
|
||||
}
|
||||
return strings.Join(p, ",")
|
||||
}
|
||||
|
||||
// 1804/AC1: given a disk with folders of known sizes, when disk_folders runs with the defaults, then it answers
|
||||
// the largest folders to depth 2, biggest first, with bytes and GiB, and the total of the path.
|
||||
func TestFoldersAreRankedBiggestFirstToDepthTwoByDefault(t *testing.T) {
|
||||
r := folders(t, aFullDisk(t).machine(), nil)
|
||||
if got := paths(r); !strings.HasPrefix(got, "/var,/var/lib,") || !strings.Contains(got, "/home,/home/jo") {
|
||||
t.Fatalf("ranked %s", got)
|
||||
}
|
||||
if r.Path != "/" || r.Partial || r.Unreadable != 0 {
|
||||
t.Errorf("%+v", r)
|
||||
}
|
||||
top := r.Largest[0]
|
||||
if top.Path != "/var" || top.Depth != 1 || top.Bytes < 15*mb || top.Bytes > 16*mb || top.GiB <= 0 {
|
||||
t.Errorf("top %+v", top)
|
||||
}
|
||||
for _, f := range r.Largest {
|
||||
if f.Depth > 2 || f.Depth < 1 {
|
||||
t.Errorf("%+v is outside depth 2", f)
|
||||
}
|
||||
if strings.Contains(f.Path, "overlay2") {
|
||||
t.Errorf("%s is deeper than 2", f.Path)
|
||||
}
|
||||
}
|
||||
if r.Bytes < 22*mb || r.Bytes > 24*mb {
|
||||
t.Errorf("total %d", r.Bytes)
|
||||
}
|
||||
}
|
||||
|
||||
// 1804/AC2: given a path, a depth and a limit, then only that folder is measured, depth is held at 3 and limit at 100.
|
||||
func TestFoldersTakeAPathADepthAndALimitHeldAtThreeAndAHundred(t *testing.T) {
|
||||
f := aFullDisk(t)
|
||||
r := folders(t, f.machine(), map[string]any{"path": "/var/lib/docker", "depth": 1.0})
|
||||
if r.Path != "/var/lib/docker" || paths(r) != "/var/lib/docker/overlay2" {
|
||||
t.Errorf("%s: %s", r.Path, paths(r))
|
||||
}
|
||||
r = folders(t, f.machine(), map[string]any{"path": "/home", "depth": 9.0, "limit": 2.0})
|
||||
if len(r.Largest) != 2 {
|
||||
t.Errorf("limit 2 gave %d", len(r.Largest))
|
||||
}
|
||||
r = folders(t, f.machine(), map[string]any{"path": "/home", "depth": 9.0})
|
||||
if got := paths(r); !strings.Contains(got, "/home/jo/deep/er") || strings.Contains(got, "/home/jo/deep/er/est") {
|
||||
t.Errorf("depth 3 under /home is /home/jo/deep/er: %s", got)
|
||||
}
|
||||
tl := tool(t, f.machine(), "disk_folders")
|
||||
for _, bad := range []map[string]any{{"depth": 0.0}, {"limit": 0.0}, {"depth": "deep"}, {"path": "relative/dir"}, {"path": "/nope"}, {"path": "/var/log/big.log"}} {
|
||||
if _, err := tl.Run(bad); err == nil {
|
||||
t.Errorf("%v was accepted", bad)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 120; i++ {
|
||||
f.file(fmt.Sprintf("many/d%03d/f", i), "x")
|
||||
}
|
||||
if r := folders(t, f.machine(), map[string]any{"path": "/many", "limit": 500.0}); len(r.Largest) != 100 {
|
||||
t.Errorf("limit held at %d, not 100", len(r.Largest))
|
||||
}
|
||||
}
|
||||
|
||||
// 1804/AC3: given a mount below the path, then it is never entered or counted, and is named.
|
||||
func TestFoldersNeverCrossIntoAnotherFilesystem(t *testing.T) {
|
||||
f := aFullDisk(t)
|
||||
m := f.machine()
|
||||
m.DevOf = func(path string, info fs.FileInfo) uint64 {
|
||||
if path == "/var/lib/docker" {
|
||||
return 7
|
||||
}
|
||||
return 1
|
||||
}
|
||||
r := folders(t, m, nil)
|
||||
if strings.Contains(paths(r), "docker") || len(r.Skipped) != 1 || r.Skipped[0] != "/var/lib/docker" {
|
||||
t.Errorf("%s, skipped %v", paths(r), r.Skipped)
|
||||
}
|
||||
if r.Bytes > 14*mb {
|
||||
t.Errorf("total %d includes the mount", r.Bytes)
|
||||
}
|
||||
}
|
||||
|
||||
// 1804/AC4: given folders it cannot read, then they are counted and named and the rest is still answered.
|
||||
func TestUnreadableFoldersAreCountedAndNamed(t *testing.T) {
|
||||
f := aFullDisk(t)
|
||||
m := f.machine()
|
||||
real := os.ReadDir
|
||||
m.ReadDir = func(p string) ([]os.DirEntry, error) {
|
||||
if strings.HasSuffix(p, "/var/log") || strings.HasSuffix(p, "/overlay2/b") {
|
||||
return nil, fmt.Errorf("open %s: permission denied", p)
|
||||
}
|
||||
return real(p)
|
||||
}
|
||||
r := folders(t, m, map[string]any{"depth": 3.0})
|
||||
if r.Unreadable != 2 || strings.Join(r.UnreadableFolders, ",") != "/var/lib/docker/overlay2/b,/var/log" {
|
||||
t.Errorf("%d: %v", r.Unreadable, r.UnreadableFolders)
|
||||
}
|
||||
if r.Partial || !strings.Contains(strings.Join(r.Said, "\n"), "2 folders could not be read") {
|
||||
t.Errorf("%+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// 1804/AC5: given a walk that outlasts its bound, then it answers what it measured, partial, with where it stopped.
|
||||
func TestALongWalkAnswersPartialAndSaysWhereItStopped(t *testing.T) {
|
||||
f := aFullDisk(t)
|
||||
m := f.machine()
|
||||
now := time.Unix(0, 0)
|
||||
m.Clock = func() time.Time { now = now.Add(10 * time.Second); return now }
|
||||
r := folders(t, m, nil)
|
||||
if !r.Partial || r.StoppedAt == "" || len(r.Said) < 2 || !strings.HasPrefix(r.Said[0], "PARTIAL") || !strings.Contains(r.Said[0], r.StoppedAt) {
|
||||
t.Errorf("%+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// 1804/AC6: a hard link is counted once, and the tool changes nothing.
|
||||
func TestAHardLinkIsCountedOnce(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.megabytes("a/big", 5)
|
||||
if err := os.Link(f.root+"/a/big", f.root+"/a/again"); err != nil {
|
||||
t.Skip(err)
|
||||
}
|
||||
r := folders(t, f.machine(), nil)
|
||||
if r.Bytes > 6*mb {
|
||||
t.Errorf("counted twice: %d", r.Bytes)
|
||||
}
|
||||
}
|
||||
|
||||
// 1804/AC7: the bound sits below the 30 s the tool runner gives a call, so a partial answer reaches its caller.
|
||||
func TestTheWalkBoundIsBelowTheToolRunnersCallLimit(t *testing.T) {
|
||||
if FolderBudget >= 30*time.Second {
|
||||
t.Errorf("the budget %s is not below the 30 s a call gets", FolderBudget)
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -53,6 +54,13 @@ type Machine struct {
|
||||
Run Runner
|
||||
Wait func(time.Duration)
|
||||
Statfs func(path string) (FSStat, error)
|
||||
|
||||
// The folder walk's seams, all optional: the folder reader, the filesystem a path is on, the clock and
|
||||
// the time the walk may take (60 s when zero).
|
||||
ReadDir func(path string) ([]os.DirEntry, error)
|
||||
DevOf func(path string, info fs.FileInfo) uint64
|
||||
Clock func() time.Time
|
||||
Budget time.Duration
|
||||
}
|
||||
|
||||
// Here is the machine this process runs on.
|
||||
|
||||
@@ -136,6 +136,34 @@ func Tools(m *Machine) []stdio.Tool {
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "disk_folders",
|
||||
Description: "Which folders fill a disk, like du -x: the largest folders under a path, measured on disk (allocated bytes, a hard link " +
|
||||
"once), staying on the path's one filesystem (a mount below it is named, never entered). path defaults to /, depth to 2 (at most 3), " +
|
||||
"limit to 20 (at most 100). Sizes in bytes and GiB. It reads every folder the tool runner can read, none of the content; folders " +
|
||||
"it cannot read are counted and named. It stops after 25 s and answers what it measured so far, marked partial, with where it stopped. " +
|
||||
"Replaces du. (r)",
|
||||
Input: map[string]any{
|
||||
"path": map[string]any{"type": "string", "description": "the absolute folder to measure (default /)"},
|
||||
"depth": map[string]any{"type": "integer", "description": "how many levels below path to rank (default 2, at most 3)"},
|
||||
"limit": map[string]any{"type": "integer", "description": "how many folders to answer (default 20, at most 100)"},
|
||||
},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
depth, err := count(args, "depth", 2, 1, 3)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
limit, err := count(args, "limit", 20, 1, 100)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
path := text(args, "path")
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
return m.Folders(path, depth, limit)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "disk_filesystems",
|
||||
Description: "The space of every filesystem that stores data (not proc, tmpfs or overlay layers): size, used and free in GiB, " +
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
"disk_load",
|
||||
"disk_top",
|
||||
"disk_filesystems",
|
||||
"disk_folders",
|
||||
"disk_devices",
|
||||
"cpu_top"
|
||||
],
|
||||
|
||||
@@ -160,6 +160,7 @@ A failure is an error naming how it failed, never an empty answer.
|
||||
| `docker_disk_usage` | r | `docker system df -v`: total, active and reclaimable per kind, with the largest of each |
|
||||
| `docker_networks` | r | networks, subnets, and the containers on each |
|
||||
| `docker_volumes` | r | volumes, who mounts each, whether the mesh holds one of them, anonymous or not, and sizes if asked |
|
||||
| `docker_volumes_remove_anonymous` | a | leftover anonymous volumes no container mounts in any state (so none the mesh holds), optionally created before a time; each other volume answered with why it stays; none can be named. **A dry run unless `dry_run` is false, which needs `why`. Never forced** (issue 1796). Sizes only when `sizes` is true (docker is slow on very many volumes); at most `limit` (default 200) per call |
|
||||
| `docker_events` | r | the runtime's events over a window ending now (default 60 min, at most 24 h), without exec noise unless asked; **an exec's command line is shown with any secret it carried as `[redacted: <what it was>]`** |
|
||||
| `docker_daemon_config` | r | `daemon.json` as on disk, `docker info`'s essentials, and keys the daemon has not taken yet |
|
||||
| `docker_unlabelled` | r | the containers the mesh does not hold: the cleanup list |
|
||||
|
||||
@@ -944,16 +944,9 @@ func (c *Client) Volumes(ctx context.Context, unmountedOnly, sizes bool) (map[st
|
||||
}
|
||||
size := map[string]string{}
|
||||
if sizes {
|
||||
out, err := c.docker(ctx, "system", "df", "--verbose", "--format", "{{json .}}")
|
||||
if err != nil {
|
||||
if size, err = c.volumeSizes(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var v struct{ Volumes []map[string]any }
|
||||
if err := json.Unmarshal([]byte(out), &v); err == nil {
|
||||
for _, vol := range v.Volumes {
|
||||
size[fmt.Sprint(vol["Name"])] = fmt.Sprint(vol["Size"])
|
||||
}
|
||||
}
|
||||
}
|
||||
vols := []map[string]any{}
|
||||
if len(lines(names)) > 0 {
|
||||
@@ -1215,3 +1208,134 @@ func (c *Client) DaemonConfig(ctx context.Context) (map[string]any, error) {
|
||||
answer["note"] = "keys read only at start take effect at the daemon's next restart; with live-restore on, a restart keeps every container running"
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// volumeSizes is every volume's size by name, from docker system df -v, which walks every volume:
|
||||
// on a machine with a thousand of them it does not answer within a call's bound.
|
||||
func (c *Client) volumeSizes(ctx context.Context) (map[string]string, error) {
|
||||
size := map[string]string{}
|
||||
out, err := c.docker(ctx, "system", "df", "--verbose", "--format", "{{json .}}")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var v struct{ Volumes []map[string]any }
|
||||
if err := json.Unmarshal([]byte(out), &v); err == nil {
|
||||
for _, vol := range v.Volumes {
|
||||
size[fmt.Sprint(vol["Name"])] = fmt.Sprint(vol["Size"])
|
||||
}
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// RemoveVolumesAsk is what docker_volumes_remove_anonymous is asked to take.
|
||||
type RemoveVolumesAsk struct {
|
||||
DryRun bool
|
||||
Why string
|
||||
CreatedBefore *time.Time
|
||||
Sizes bool
|
||||
Limit int // how many to remove in one call; 0 is 200
|
||||
}
|
||||
|
||||
// RemoveAnonymousVolumes removes the leftover anonymous volumes (novox/hq issue 1796): those docker
|
||||
// labels anonymous, which no container mounts in any state, so none a mesh-held container holds. The
|
||||
// classification is Volumes' own, not a copy. Every other volume is answered with why it stays. Each
|
||||
// is removed by its own `docker volume rm`, never forced and never `volume prune`, so one the daemon
|
||||
// refuses is reported and the rest still go.
|
||||
func (c *Client) RemoveAnonymousVolumes(ctx context.Context, a RemoveVolumesAsk) (map[string]any, error) {
|
||||
if !a.DryRun && strings.TrimSpace(a.Why) == "" {
|
||||
return nil, fmt.Errorf("why is required when dry_run is false")
|
||||
}
|
||||
all, err := c.Volumes(ctx, false, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
candidates := []string{}
|
||||
skipped := []map[string]any{}
|
||||
skip := func(name, why string) { skipped = append(skipped, map[string]any{"name": name, "why": why}) }
|
||||
for _, v := range all["volumes"].([]map[string]any) {
|
||||
name := fmt.Sprint(v["name"])
|
||||
switch by := v["mounted_by"].([]map[string]any); {
|
||||
case v["anonymous"] != true:
|
||||
skip(name, "not anonymous: a named volume is data")
|
||||
case v["mesh_held"] == true:
|
||||
skip(name, "mounted by a container the mesh holds")
|
||||
case len(by) > 0:
|
||||
skip(name, fmt.Sprintf("mounted by %v (%v)", by[0]["container"], by[0]["state"]))
|
||||
case a.CreatedBefore != nil && !volumeOlder(fmt.Sprint(v["created"]), *a.CreatedBefore):
|
||||
skip(name, "not created before "+a.CreatedBefore.Format(time.RFC3339)+" (created "+fmt.Sprint(v["created"])+")")
|
||||
default:
|
||||
candidates = append(candidates, name)
|
||||
}
|
||||
}
|
||||
answer := map[string]any{"dry_run": a.DryRun, "candidates": len(candidates)}
|
||||
// Sizes are asked for, never needed: the listing walks every volume and may not answer, and
|
||||
// then the call answers without them. Nothing below depends on it.
|
||||
sizes := map[string]string{}
|
||||
if a.Sizes {
|
||||
if got, err := c.volumeSizes(ctx); err != nil {
|
||||
answer["sizes_unknown"] = err.Error()
|
||||
} else {
|
||||
sizes = got
|
||||
answer["sizes"] = sizes
|
||||
}
|
||||
}
|
||||
sum := func(names []string) int64 {
|
||||
var t int64
|
||||
for _, n := range names {
|
||||
t += max(Bytes(sizes[n]), 0)
|
||||
}
|
||||
return t
|
||||
}
|
||||
if a.DryRun {
|
||||
answer["skipped"] = skipped
|
||||
answer["would_remove"] = candidates
|
||||
if len(sizes) > 0 {
|
||||
answer["would_free_bytes"] = sum(candidates)
|
||||
}
|
||||
answer["note"] = "nothing was removed: call again with dry_run false and why to remove them (limit volumes at most per call)"
|
||||
return answer, nil
|
||||
}
|
||||
limit := a.Limit
|
||||
if limit <= 0 {
|
||||
limit = 200
|
||||
}
|
||||
take := candidates
|
||||
if len(take) > limit {
|
||||
take = candidates[:limit]
|
||||
}
|
||||
removed := []string{}
|
||||
// Chunks of a hundred in one `docker volume rm`, so a thousand volumes are ten calls; a chunk
|
||||
// docker refuses any of is done again one by one, so what it refuses is named.
|
||||
for i := 0; i < len(take); i += 100 {
|
||||
chunk := take[i:min(i+100, len(take))]
|
||||
if _, err := c.docker(ctx, append([]string{"volume", "rm"}, chunk...)...); err == nil {
|
||||
removed = append(removed, chunk...)
|
||||
continue
|
||||
}
|
||||
for _, name := range chunk {
|
||||
if _, err := c.docker(ctx, "volume", "rm", name); err != nil {
|
||||
// The batch took it before it refused another (or before its bound ran out):
|
||||
// a volume that is no longer there is a removed one.
|
||||
if strings.Contains(strings.ToLower(err.Error()), "no such volume") {
|
||||
removed = append(removed, name)
|
||||
continue
|
||||
}
|
||||
skip(name, "docker refused: "+err.Error())
|
||||
continue
|
||||
}
|
||||
removed = append(removed, name)
|
||||
}
|
||||
}
|
||||
answer["skipped"] = skipped
|
||||
answer["removed"], answer["count"], answer["why"] = removed, len(removed), a.Why
|
||||
answer["remaining"] = len(candidates) - len(take)
|
||||
if len(sizes) > 0 {
|
||||
answer["freed_bytes"] = sum(removed)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// volumeOlder is whether a volume's CreatedAt is before the cutoff; one that cannot be read is not.
|
||||
func volumeOlder(created string, cutoff time.Time) bool {
|
||||
t, err := time.Parse(time.RFC3339Nano, created)
|
||||
return err == nil && t.Before(cutoff)
|
||||
}
|
||||
|
||||
@@ -431,3 +431,145 @@ func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *test
|
||||
t.Fatalf("a full buffer reaching past the window: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1796/AC1: only anonymous volumes nothing mounts go, each answered; a named, a mounted and a too young one stay with why.
|
||||
func TestRemoveAnonymousVolumesTakesOnlyTheUnmountedAnonymousOnes(t *testing.T) {
|
||||
f := machine().
|
||||
on("docker volume ls --quiet", Ran{Stdout: "webdata\nanonmounted\nold\nyoung\nnamed\n"}).
|
||||
on("docker volume inspect", Ran{Stdout: `[
|
||||
{"Name":"webdata","Driver":"local","CreatedAt":"2026-01-01T00:00:00Z","Labels":{"com.docker.volume.anonymous":""}},
|
||||
{"Name":"old","Driver":"local","CreatedAt":"2026-09-01T00:00:00Z","Labels":{"com.docker.volume.anonymous":""}},
|
||||
{"Name":"young","Driver":"local","CreatedAt":"2026-10-04T11:00:00Z","Labels":{"com.docker.volume.anonymous":""}},
|
||||
{"Name":"named","Driver":"local","CreatedAt":"2026-01-01T00:00:00Z"}]`}).
|
||||
on("docker system df --verbose", Ran{Stdout: `{"Volumes":[{"Name":"old","Size":"1.5MB"},{"Name":"young","Size":"1kB"}]}`}).
|
||||
on("docker volume rm old", Ran{Stdout: "old\n"})
|
||||
c := client(f, 1000)
|
||||
before := time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)
|
||||
got, err := c.RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{DryRun: true, Sizes: true, CreatedBefore: &before})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.ran("docker volume rm") || !reflect.DeepEqual(got["would_remove"], []string{"old"}) {
|
||||
t.Fatalf("dry run: %v %+v", got, f.calls)
|
||||
}
|
||||
if got["would_free_bytes"] != int64(1500000) || got["sizes"].(map[string]string)["old"] != "1.5MB" {
|
||||
t.Fatalf("dry run size: %v", got)
|
||||
}
|
||||
skipped := got["skipped"].([]map[string]any)
|
||||
why := map[string]string{}
|
||||
for _, s := range skipped {
|
||||
why[s["name"].(string)] = s["why"].(string)
|
||||
}
|
||||
if len(why) != 3 || !strings.Contains(why["webdata"], "mesh holds") || why["young"] == "" || why["named"] == "" {
|
||||
t.Fatalf("skipped: %v", skipped)
|
||||
}
|
||||
got, err = c.RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{Why: "leftovers", Sizes: true, CreatedBefore: &before})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !f.ran("docker volume rm old") || got["count"] != 1 || !reflect.DeepEqual(got["removed"], []string{"old"}) {
|
||||
t.Fatalf("removed: %v %+v", got, f.calls)
|
||||
}
|
||||
if got["freed_bytes"] != int64(1500000) {
|
||||
t.Fatalf("freed: %v", got)
|
||||
}
|
||||
for _, c := range f.calls {
|
||||
if len(c.args) > 1 && c.args[0] == "volume" && c.args[1] == "rm" && (len(c.args) != 3 || c.args[2] != "old") {
|
||||
t.Errorf("reached too far: %v", c.args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1796/AC2: a volume that is mounted by a mesh-held container is never removed; the tool takes no name.
|
||||
func TestRemoveAnonymousVolumesRefusesANamedVolumeAndKeepsWhatTheMeshHolds(t *testing.T) {
|
||||
for _, tool := range tools(client(&fake{}, 1000)) {
|
||||
if tool.Name != "docker_volumes_remove_anonymous" {
|
||||
continue
|
||||
}
|
||||
for _, k := range []string{"name", "names", "volume", "volumes"} {
|
||||
if _, err := tool.Run(map[string]any{k: "x", "dry_run": false, "why": "w"}); err == nil {
|
||||
t.Errorf("%s accepted", k)
|
||||
}
|
||||
}
|
||||
if _, err := tool.Run(map[string]any{"dry_run": false}); err == nil {
|
||||
t.Error("a removal without why was accepted")
|
||||
}
|
||||
if _, err := tool.Run(map[string]any{"created_before": "yesterday"}); err == nil {
|
||||
t.Error("a bad time was accepted")
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatal("tool not served")
|
||||
}
|
||||
|
||||
// 1796/AC3: on a machine with a very large set of volumes docker's size listing does not answer in
|
||||
// time; the tool still answers, and removal never depends on it.
|
||||
func TestRemoveAnonymousVolumesAnswersWhenTheSizeListingDoesNot(t *testing.T) {
|
||||
f := (&fake{}).
|
||||
on("docker ps --all --quiet --no-trunc", Ran{}).
|
||||
on("docker volume ls --quiet", Ran{Stdout: "a1\na2\na3\n"}).
|
||||
on("docker volume inspect", Ran{Stdout: `[{"Name":"a1","Labels":{"com.docker.volume.anonymous":""}},{"Name":"a2","Labels":{"com.docker.volume.anonymous":""}},{"Name":"a3","Labels":{"com.docker.volume.anonymous":""}}]`}).
|
||||
on("docker system df", Ran{Status: 124, Err: "no answer within 20 s"}).
|
||||
on("docker volume rm", Ran{Stdout: "ok\n"})
|
||||
c := client(f, 1000)
|
||||
got, err := c.RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{DryRun: true})
|
||||
if err != nil || f.ran("docker system df") || !reflect.DeepEqual(got["would_remove"], []string{"a1", "a2", "a3"}) {
|
||||
t.Fatalf("default dry run: %v %v %+v", got, err, f.calls)
|
||||
}
|
||||
got, err = c.RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{DryRun: true, Sizes: true})
|
||||
if err != nil || !reflect.DeepEqual(got["would_remove"], []string{"a1", "a2", "a3"}) || got["sizes_unknown"] == nil {
|
||||
t.Fatalf("sizes asked, df silent: %v %v", got, err)
|
||||
}
|
||||
got, err = c.RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{Why: "w", Sizes: true, Limit: 2})
|
||||
if err != nil || got["count"] != 2 || got["remaining"] != 1 || !f.ran("docker volume rm a1 a2") {
|
||||
t.Fatalf("removal: %v %v %+v", got, err, f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// 1796/AC4: a batch docker half-did is settled by name: what is already gone counts as removed, what is in use stays with why.
|
||||
func TestRemoveAnonymousVolumesCountsWhatAFailedBatchAlreadyRemoved(t *testing.T) {
|
||||
f := (&fake{}).
|
||||
on("docker ps --all --quiet --no-trunc", Ran{}).
|
||||
on("docker volume ls --quiet", Ran{Stdout: "a1\na2\na3\n"}).
|
||||
on("docker volume inspect", Ran{Stdout: `[{"Name":"a1","Labels":{"com.docker.volume.anonymous":""}},{"Name":"a2","Labels":{"com.docker.volume.anonymous":""}},{"Name":"a3","Labels":{"com.docker.volume.anonymous":""}}]`}).
|
||||
on("docker volume rm a1 a2 a3", Ran{Status: 1, Stdout: "a1\n", Stderr: "Error response from daemon: remove a2: volume is in use - [c1]\n"}).
|
||||
on("docker volume rm a1", Ran{Status: 1, Stderr: "Error response from daemon: get a1: no such volume\n"}).
|
||||
on("docker volume rm a2", Ran{Status: 1, Stderr: "Error response from daemon: remove a2: volume is in use - [c1]\n"}).
|
||||
on("docker volume rm a3", Ran{Stdout: "a3\n"})
|
||||
got, err := client(f, 1000).RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{Why: "w"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sk := got["skipped"].([]map[string]any)
|
||||
if !reflect.DeepEqual(got["removed"], []string{"a1", "a3"}) || got["count"] != 2 || len(sk) != 1 || sk[0]["name"] != "a2" || !strings.Contains(sk[0]["why"].(string), "in use") {
|
||||
t.Fatalf("%v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1796/AC5: removal goes in batches of a hundred names.
|
||||
func TestRemoveAnonymousVolumesRemovesInBatchesOfAHundred(t *testing.T) {
|
||||
names, entries := []string{}, []string{}
|
||||
for i := 0; i < 250; i++ {
|
||||
n := fmt.Sprintf("v%03d", i)
|
||||
names = append(names, n)
|
||||
entries = append(entries, `{"Name":"`+n+`","Labels":{"com.docker.volume.anonymous":""}}`)
|
||||
}
|
||||
f := (&fake{}).
|
||||
on("docker ps --all --quiet --no-trunc", Ran{}).
|
||||
on("docker volume ls --quiet", Ran{Stdout: strings.Join(names, "\n")}).
|
||||
on("docker volume inspect", Ran{Stdout: "[" + strings.Join(entries, ",") + "]"}).
|
||||
on("docker volume rm", Ran{Stdout: "ok\n"})
|
||||
got, err := client(f, 1000).RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{Why: "w", Limit: 250})
|
||||
if err != nil || got["count"] != 250 {
|
||||
t.Fatalf("%v %v", got, err)
|
||||
}
|
||||
sizes := []int{}
|
||||
for _, c := range f.calls {
|
||||
if len(c.args) > 1 && c.args[0] == "volume" && c.args[1] == "rm" {
|
||||
sizes = append(sizes, len(c.args)-2)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(sizes, []int{100, 100, 50}) {
|
||||
t.Fatalf("batches %v", sizes)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"math"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
@@ -286,6 +287,40 @@ func tools(c *Client) []stdio.Tool {
|
||||
return c.Volumes(ctx, flag(args, "unmounted", false), flag(args, "sizes", false))
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "docker_volumes_remove_anonymous",
|
||||
Description: "Remove the leftover anonymous volumes (novox/hq issue 1796): only a volume docker labels anonymous (as docker_volumes says), which no container mounts in any state " +
|
||||
"and so none the mesh holds, and with created_before only one created before it. Every other volume is answered with why it stays; a named volume is data and is never taken, " +
|
||||
"and no volume can be named. Each is removed on its own, never forced; the answer has the count and names removed. With sizes true the answer has each volume's size and the bytes freed; docker's size listing is slow on many volumes, so it is asked for, and when it does not answer the call answers without it. At most limit are removed per call, in batches. " +
|
||||
"A dry run by default; dry_run false needs why. Replaces docker volume rm and docker volume prune. (a)",
|
||||
Input: map[string]any{
|
||||
"dry_run": map[string]any{"type": "boolean", "description": "list only (default true)"},
|
||||
"why": map[string]any{"type": "string", "description": "why: required when dry_run is false"},
|
||||
"created_before": map[string]any{"type": "string", "description": "RFC 3339 time: only volumes created before it (default: any age)"},
|
||||
"sizes": map[string]any{"type": "boolean", "description": "also read each volume's size and the bytes freed (default false: docker walks every volume for it and on a machine with very many may not answer; the call then answers without sizes)"},
|
||||
"limit": map[string]any{"type": "integer", "description": "how many to remove in one call (default 200, at most 2000); the answer says how many remain, call again for them"},
|
||||
},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
for _, k := range []string{"name", "names", "volume", "volumes"} {
|
||||
if _, ok := args[k]; ok {
|
||||
return nil, fmt.Errorf("%s: this tool takes no volume by name; it removes only unmounted anonymous volumes", k)
|
||||
}
|
||||
}
|
||||
limit, err := bounded(args, "limit", 200, 2000)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ask := RemoveVolumesAsk{DryRun: flag(args, "dry_run", true), Why: optional(args, "why"), Sizes: flag(args, "sizes", false), Limit: limit}
|
||||
if s := optional(args, "created_before"); s != "" {
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("created_before must be an RFC 3339 time such as 2026-10-01T00:00:00Z")
|
||||
}
|
||||
ask.CreatedBefore = &t
|
||||
}
|
||||
return c.RemoveAnonymousVolumes(ctx, ask)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "docker_events",
|
||||
Description: "What the runtime did in a window ending now (default the last 60 minutes, at most 24 hours): containers created, started, died, health changes, images pulled — with mesh_held. Exec events are left out unless asked; " +
|
||||
|
||||
@@ -33,6 +33,7 @@
|
||||
"docker_disk_usage",
|
||||
"docker_networks",
|
||||
"docker_volumes",
|
||||
"docker_volumes_remove_anonymous",
|
||||
"docker_events",
|
||||
"docker_daemon_config",
|
||||
"docker_unlabelled",
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
// **A consumer that cannot be bound yet is bound again, never fatal** (novox/hq issue 1793). When the tool runtime
|
||||
// starts this bundle, the runtime's answer to "bind this module's consumer" can be "no responders" for a few seconds (the
|
||||
// runtime restarted by a delivery, the bus not yet serving the stream). That rejection, left unhandled at the top of
|
||||
// index.ts, ended the bundle: nothing heard `mesh-controller.checked` again, and a verdict that landed later never
|
||||
// reached the forge's statuses. A bind that fails is said and asked again with a growing wait until it holds.
|
||||
|
||||
/** How long to wait before the nth new attempt (1-based): 2 s, 5 s, 10 s, then every 30 s. */
|
||||
export function bindWait(attempt: number): number {
|
||||
return [2_000, 5_000, 10_000][attempt - 1] ?? 30_000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Binds with `bind`, asking again after each failure, until it holds. `sleep` is the wait; `say` how a failure is said.
|
||||
* Never rejects; resolves once the bind holds.
|
||||
*/
|
||||
export async function bindUntilHeld(what: string, bind: () => Promise<unknown>,
|
||||
sleep: (ms: number) => Promise<void> = (ms) => new Promise((r) => setTimeout(r, ms)),
|
||||
say: (line: string) => void = (line) => console.error(line)): Promise<void> {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
await bind();
|
||||
if (attempt > 1) say(`[gitea] ${what} is bound after ${attempt} attempts`);
|
||||
return;
|
||||
} catch (err) {
|
||||
const wait = bindWait(attempt);
|
||||
say(`[gitea] ${what} could not be bound (${err instanceof Error ? err.message : String(err)}); asking again in ${wait / 1000}s`);
|
||||
await sleep(wait);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* **Starts the binds without making the bundle wait for them** (novox/hq issue 1793). The launcher imports the bundle
|
||||
* and then answers the runtime's handshake, which the runtime gives 10 s and then kills the bundle without starting it
|
||||
* again; a bind awaited at the top of the module that stays refused longer ends the bundle all the same. So this
|
||||
* returns at once: the binds run in the order given, each until it holds, and `done` is called after the last.
|
||||
*/
|
||||
export function bindInBackground(binds: [what: string, bind: () => Promise<unknown>][], done: () => void,
|
||||
sleep?: (ms: number) => Promise<void>, say?: (line: string) => void): void {
|
||||
void (async () => {
|
||||
for (const [what, bind] of binds) await bindUntilHeld(what, bind, sleep, say);
|
||||
done();
|
||||
})();
|
||||
}
|
||||
@@ -179,7 +179,7 @@ func newTeamRig(t *testing.T) *teamRig {
|
||||
svc := NewService(Config{Node: "novox", HostFile: host, OriginsFile: origins, Port: 222,
|
||||
Forge: forgehttp.Client{URL: srv.URL, User: "mesh-admin", PasswordFile: pw}, Runner: runner, Log: func(string, ...any) {}})
|
||||
svc.loaded = true
|
||||
svc.loadedFrom["mesh-build-go"], svc.loadedFrom["mesh-build-node"] = true, true
|
||||
svc.loadedFrom["mesh-build-go"], svc.loadedFrom["mesh-build-node"], svc.loadedFrom["mesh-build-image"] = true, true, true
|
||||
return &teamRig{t: t, bus: newBus(), forge: forge, svc: svc, runner: runner}
|
||||
}
|
||||
|
||||
@@ -347,7 +347,7 @@ func TestABuildModulesEntryWaitsForItsOwnBuilderKey(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTheKindOfAModulesKeyAndItsForgeUser(t *testing.T) {
|
||||
for module, want := range map[string]string{"mesh-build-go": "nox-builder", "mesh-build-node": "nox-builder", "ssh-client": "nox-agent", "anything-else": "nox-agent"} {
|
||||
for module, want := range map[string]string{"mesh-build-go": "nox-builder", "mesh-build-node": "nox-builder", "mesh-build-image": "nox-builder", "ssh-client": "nox-agent", "anything-else": "nox-agent"} {
|
||||
got, err := (&Service{}).valuesForUser("ace", module)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("%s: %q, %v; want %q", module, got, err, want)
|
||||
|
||||
@@ -38,7 +38,7 @@ func newWatched(t *testing.T) *watched {
|
||||
r.svc.OnKeys = func(string) { n++ }
|
||||
r.svc.cfg.Log = func(f string, a ...any) { mu.Lock(); logs = append(logs, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
w := &watched{teamRig: r, sources: map[string]*fakeKeys{}, passes: &n, logs: &logs, mu: mu}
|
||||
for _, module := range []string{"ssh-client", "mesh-build-go", "mesh-build-node"} {
|
||||
for _, module := range []string{"ssh-client", "mesh-build-go", "mesh-build-node", "mesh-build-image"} {
|
||||
w.sources[module] = &fakeKeys{}
|
||||
if err := r.svc.WatchSource(context.Background(), module, w.sources[module]); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -154,7 +154,7 @@ func TestTheManifestReadsTheBuildModulesKeys(t *testing.T) {
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys"} {
|
||||
for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys", "mesh-build-image.keys"} {
|
||||
if !slices.Contains(m.Reads, state) {
|
||||
t.Errorf("gitea's reads %v lack %q", m.Reads, state)
|
||||
}
|
||||
|
||||
+15
-11
@@ -47,6 +47,7 @@ import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, composedGate, groupGateS
|
||||
import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
|
||||
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
|
||||
import { keepSiblings, noteComposed, noteVerdict, readSiblings, siblingsFile, siblingsOf, weighSiblings } from "./siblings.js";
|
||||
import { bindInBackground } from "./bind.js";
|
||||
import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
|
||||
|
||||
// Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
|
||||
@@ -590,15 +591,18 @@ if (gitea) {
|
||||
tick(() => pollRepos(client), 60_000);
|
||||
tick(() => pollMerged(client), () => pace.every());
|
||||
tick(() => pollPulls(client), () => pace.every());
|
||||
await on("mesh-controller.checked", (event) => {
|
||||
const c = (event.body ?? {}) as Checked;
|
||||
// A head's own verdict takes its turn after every write of that head's gate already asked for (gate.ts).
|
||||
// A verdict of a check asked before the one now pending is an older run's, and dropped (novox/hq issue 456).
|
||||
if (!c.group && c.owner && c.repo && c.commit) {
|
||||
return gates.verdict(c.owner, c.repo, c.commit, () => setVerdict(client, event), c.id);
|
||||
}
|
||||
return setVerdict(client, event);
|
||||
});
|
||||
await on("node-build-agent.started", buildStarted);
|
||||
console.log("[gitea] watching for new repositories and merged pull requests");
|
||||
// Bound again until they hold, and not awaited (bind.ts, issue 1793): the runtime kills a bundle that has not answered its
|
||||
// handshake in 10 s, so the import must end while a refused bind is still asked again.
|
||||
bindInBackground([
|
||||
["mesh-controller.checked", () => on("mesh-controller.checked", (event) => {
|
||||
const c = (event.body ?? {}) as Checked;
|
||||
// A head's own verdict takes its turn after every write of that head's gate already asked for (gate.ts).
|
||||
// A verdict of a check asked before the one now pending is an older run's, and dropped (novox/hq issue 456).
|
||||
if (!c.group && c.owner && c.repo && c.commit) {
|
||||
return gates.verdict(c.owner, c.repo, c.commit, () => setVerdict(client, event), c.id);
|
||||
}
|
||||
return setVerdict(client, event);
|
||||
})],
|
||||
["node-build-agent.started", () => on("node-build-agent.started", buildStarted)],
|
||||
], () => console.log("[gitea] watching for new repositories and merged pull requests"));
|
||||
}
|
||||
|
||||
@@ -195,7 +195,7 @@ const BuilderUser = "nox-builder"
|
||||
|
||||
// BuildModules are the modules whose keys are the builders' (kind builder), by their name, which is also the name of
|
||||
// the account each runs as (ADR 0466 §1).
|
||||
var BuildModules = []string{"mesh-build-go", "mesh-build-node"}
|
||||
var BuildModules = []string{"mesh-build-go", "mesh-build-node", "mesh-build-image"}
|
||||
|
||||
// KindOfModule is the kind of key a consumer module's entry is made for: builder for a build module, machine for any
|
||||
// other (ssh-client's).
|
||||
|
||||
@@ -12,7 +12,7 @@ func TestTheBuildModulesHaveOneForgeUserAndTheMachinesKeepTheirs(t *testing.T) {
|
||||
if ForgeUser("Not A Node", "builder") != "" {
|
||||
t.Errorf("a name no account could be named for got a user")
|
||||
}
|
||||
for module, want := range map[string]string{"mesh-build-go": "builder", "mesh-build-node": "builder", "ssh-client": "machine", "": "machine"} {
|
||||
for module, want := range map[string]string{"mesh-build-go": "builder", "mesh-build-node": "builder", "mesh-build-image": "builder", "ssh-client": "machine", "": "machine"} {
|
||||
if got := KindOfModule(module); got != want {
|
||||
t.Errorf("KindOfModule(%q) = %s, want %s", module, got, want)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import { mergeableUnder, newestByContext } from "./checks.js";
|
||||
import { MERGE_METHODS, Refused, checkName } from "./guard.js";
|
||||
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js";
|
||||
import { approvedHeadsOf, groupRefusal, memberKey, namesSiblings, notApproved, readSiblings, siblingOutOfDate, siblingsFile } from "./siblings.js";
|
||||
import { announcementOf, askedWords, outOfDateWords, sameCommit, tookFile, tookWords, twelve, verdictAtTrunk } from "./trunk.js";
|
||||
import { announcementOf, askedWords, outOfDateWords, sameCommit, tookFile, tookWords, twelve, verdictAtTrunk, waitsForComposed } from "./trunk.js";
|
||||
|
||||
/** A head named for a merge: at least 12 hex characters of its sha (novox/hq issue 381), up to a sha-256 commit's 64. */
|
||||
const REVIEWED_HEAD = /^[0-9a-f]{12,64}$/;
|
||||
@@ -117,6 +117,14 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
||||
: at.judged ? `was judged against trunk ${twelve(at.judged)}` : "names no trunk commit it was judged against";
|
||||
const took = tookWords(tookFile(deps.stateDir), `${owner}/${repo}`);
|
||||
const head = twelve(pull.head_sha), now = twelve(trunkHead);
|
||||
// **A member that waits for its group's composed check is not announced again** (novox/hq issue 1106): its own
|
||||
// check can only say "composed" once more, so an ask here brought a verdict naming no trunk and the next merge was
|
||||
// refused the same way for ever. The composed check is mesh-delivery's to ask; this says whose verdict is awaited.
|
||||
if (waitsForComposed(checks[CHECK_CONTEXT])) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its merge gate ${checks[CHECK_CONTEXT]?.description}; the ` +
|
||||
`verdict that names its trunk commit is its group's composed check's, and mesh-delivery asks it, this tool asks no check ` +
|
||||
`(main is now ${now}). Ask again once the group's composed check has passed. Nothing was merged`);
|
||||
}
|
||||
if (at.onItsWay) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` +
|
||||
`${now}. Its check is already asked and not yet answered (${checks[CHECK_CONTEXT]?.description}), so it is not ` +
|
||||
|
||||
@@ -542,6 +542,7 @@
|
||||
"mesh-controller.assignments",
|
||||
"ssh-client.keys",
|
||||
"mesh-build-go.keys",
|
||||
"mesh-build-node.keys"
|
||||
"mesh-build-node.keys",
|
||||
"mesh-build-image.keys"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -226,6 +226,10 @@ function repoWrong(repo: Layer | undefined, facts: RepoCheckFacts): boolean {
|
||||
return !!repo && repoStatus(repo, facts).state !== "success";
|
||||
}
|
||||
|
||||
/** What a group member's gate says while the group's composed check is the one that judges it: the member's own check can say
|
||||
* nothing more of it, so asking that check again brings no verdict that names a trunk (novox/hq issue 1106). */
|
||||
export const WAITS_FOR_COMPOSED = "waits for its group's composed check";
|
||||
|
||||
/** The forge's status for the gate. */
|
||||
export function statusFor(c: Checked): CommitStatus {
|
||||
const gate = c.gate ?? { verdict: c.verdict, summary: c.summary };
|
||||
@@ -260,7 +264,7 @@ export function statusesFor(c: Checked, facts: RepoCheckFacts = {}, gateNow?: st
|
||||
// set. The gate stays pending, saying it waits for the group, unless a composed verdict already stands on it.
|
||||
const out: CommitStatus[] = [];
|
||||
if (!(gateNow ?? "").includes(" composed: ")) {
|
||||
out.push({ state: "pending", context: CHECK_CONTEXT, description: "waits for its group's composed check" });
|
||||
out.push({ state: "pending", context: CHECK_CONTEXT, description: WAITS_FOR_COMPOSED });
|
||||
}
|
||||
const repo = c["repo-check"];
|
||||
if (repo) out.push(judgedFirst(c, repoStatus(repo, facts)));
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
// **A consumer that cannot be bound yet is bound again** (novox/hq issue 1793): the runtime's "no responders" at the
|
||||
// bundle's start, seen 2026-10-22 00:28:37 on mesh-catalog#427's verdict, ended the bundle and the verdict was never
|
||||
// written on the forge.
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { bindInBackground, bindUntilHeld, bindWait } from "../dist/bind.js";
|
||||
|
||||
test("a bind refused with no responders is asked again until it holds, and says so", async () => {
|
||||
let calls = 0;
|
||||
const waits: number[] = [];
|
||||
const said: string[] = [];
|
||||
await bindUntilHeld("mesh-controller.checked", async () => {
|
||||
calls++;
|
||||
if (calls < 4) throw new Error("binding gitea's consumer novox_gitea: nats: no responders available for request");
|
||||
}, async (ms) => { waits.push(ms); }, (l) => said.push(l));
|
||||
assert.equal(calls, 4);
|
||||
assert.deepEqual(waits, [2_000, 5_000, 10_000]);
|
||||
assert.equal(said.length, 4);
|
||||
assert.match(said[0], /no responders available for request.*asking again in 2s/);
|
||||
assert.match(said[3], /is bound after 4 attempts/);
|
||||
});
|
||||
|
||||
test("a bind that holds the first time says nothing and waits for nothing", async () => {
|
||||
const said: string[] = [];
|
||||
await bindUntilHeld("x", async () => {}, async () => assert.fail("waited"), (l) => said.push(l));
|
||||
assert.deepEqual(said, []);
|
||||
});
|
||||
|
||||
test("the wait grows to 30 s and stays", () => {
|
||||
assert.deepEqual([1, 2, 3, 4, 50].map(bindWait), [2_000, 5_000, 10_000, 30_000, 30_000]);
|
||||
});
|
||||
|
||||
test("the binds start without being waited for: the caller returns while a bind keeps refusing, and they go in order", async () => {
|
||||
const order: string[] = [];
|
||||
let finished = false;
|
||||
let refusals = 0;
|
||||
const gate = new Promise<void>((r) => setTimeout(r, 0));
|
||||
bindInBackground([
|
||||
["first", async () => { refusals++; if (refusals < 3) throw new Error("nats: no responders available for request"); order.push("first"); }],
|
||||
["second", async () => { order.push("second"); }],
|
||||
], () => { finished = true; }, async () => { await gate; }, () => {});
|
||||
// Returned at once, with the first bind refused and nothing done.
|
||||
assert.equal(finished, false);
|
||||
assert.deepEqual(order, []);
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
assert.deepEqual(order, ["first", "second"]);
|
||||
assert.equal(finished, true);
|
||||
});
|
||||
|
||||
test("a bind that never holds does not stop the caller from returning", () => {
|
||||
let returned = false;
|
||||
bindInBackground([["x", async () => { throw new Error("no responders"); }]], () => assert.fail("done"),
|
||||
() => new Promise(() => {}), () => {});
|
||||
returned = true;
|
||||
assert.equal(returned, true);
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
// **The bundle's import ends while a bind is refused** (novox/hq issue 1793). The launcher imports index.js and then answers
|
||||
// node-tools' handshake, which has 10 s before the bundle is killed and never started again. A bind awaited at the top
|
||||
// of index.ts, refused for longer, ended the bundle at 00:28:37 on 2026-10-22. Run in a child: the polls' timers keep a
|
||||
// process alive, and the child says "imported" and exits as soon as the import resolves.
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
test("importing the bundle resolves while the broker keeps refusing the binds", async () => {
|
||||
const index = pathToFileURL(join(here, "..", "dist", "index.js")).href;
|
||||
const script = `
|
||||
const { useBroker } = await import("@novox/mesh-sdk/messaging");
|
||||
useBroker(() => ({
|
||||
subscribe: async () => { throw new Error("binding gitea's consumer novox_gitea: nats: no responders available for request"); },
|
||||
publish: async () => {}, request: async () => { throw new Error("no responders"); },
|
||||
handle: async () => () => {}, close: async () => {},
|
||||
}));
|
||||
await import(${JSON.stringify(index)});
|
||||
console.log("imported");
|
||||
process.exit(0);`;
|
||||
const env = { ...process.env, MESH_GITEA_TOKEN: "x", MESH_GITEA_URL: "http://127.0.0.1:1" };
|
||||
for (const k of ["MESH_GITEA_STATE_DIR", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD", "MESH_SERVED_MODULE"]) delete env[k];
|
||||
const child = spawn(process.execPath, ["--input-type=module", "-e", script], { cwd: join(here, ".."), env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let out = "";
|
||||
let err = "";
|
||||
child.stdout.on("data", (d) => (out += d));
|
||||
child.stderr.on("data", (d) => (err += d));
|
||||
const ended = new Promise<void>((r) => child.on("exit", () => r()));
|
||||
const timer = setTimeout(() => child.kill("SIGKILL"), 3000);
|
||||
await ended;
|
||||
clearTimeout(timer);
|
||||
assert.match(out, /imported/, `the import did not resolve within 3 s while a bind was refused; stderr: ${err.slice(0, 600)}`);
|
||||
});
|
||||
@@ -339,6 +339,30 @@ test("gitea_merge_pull_request refuses a verdict judged against an earlier trunk
|
||||
assert.equal(state.emitted.length, 4, "a merge asks no check");
|
||||
});
|
||||
|
||||
// **A group member whose gate waits for its group's composed check is not announced again by the merge tool** (novox/hq issue
|
||||
// 1106): the member's own check can only say "composed" again, so each refusal's ask brought a verdict that names no trunk and
|
||||
// the next merge was refused the same way, for ever. The tool says whose verdict is awaited and asks nothing.
|
||||
test("gitea_merge_pull_request asks no check of a member that waits for its group's composed check", async () => {
|
||||
const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED });
|
||||
state.trunk = TRUNK;
|
||||
const before = state.emitted.length;
|
||||
state.statuses = [{ context: "mesh/merge-gate", state: "pending", description: "waits for its group's composed check" },
|
||||
{ context: "mesh/repo-check", state: "success", description: "pass: y" }];
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await assert.rejects(merge(), (err: Error) => {
|
||||
assert.match(err.message, /waits for its group's composed check/);
|
||||
assert.match(err.message, /mesh-delivery asks it, this tool asks no check/);
|
||||
assert.match(err.message, /Nothing was merged/);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
assert.equal(state.emitted.length, before, "no check is announced again and again");
|
||||
assert.equal(state.merged, false);
|
||||
// Once the composed verdict names the member's unmoved trunk, it merges.
|
||||
state.statuses = judgedAt(TRUNK);
|
||||
assert.equal(((await merge()) as Record<string, unknown>).merged, true);
|
||||
});
|
||||
|
||||
// A merge lands only the head that was reviewed (novox/hq issue 381): mesh-controller#197 was reviewed at one head,
|
||||
// force-pushed, checked green at the new one, and merged there, because the tool merged whatever the head was.
|
||||
test("gitea_merge_pull_request merges only the head it is named, and refuses one that is not the head", async () => {
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import type { CommitStatus, GiteaClient, GiteaPull, HeadCheck } from "./client.js";
|
||||
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js";
|
||||
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, WAITS_FOR_COMPOSED } from "./pulls.js";
|
||||
|
||||
/** A commit as a verdict names it: its first twelve characters. */
|
||||
export function twelve(sha: string | undefined): string {
|
||||
@@ -58,6 +58,11 @@ export function checkOnItsWay(gate: HeadCheck | undefined): boolean {
|
||||
return d.startsWith("waiting in the build queue") || d.startsWith("building on ") || d.startsWith("checking again");
|
||||
}
|
||||
|
||||
/** Whether a gate says it waits for its group's composed check. */
|
||||
export function waitsForComposed(gate: HeadCheck | undefined): boolean {
|
||||
return !!gate && gate.state === "pending" && (gate.description ?? "").startsWith(WAITS_FOR_COMPOSED);
|
||||
}
|
||||
|
||||
/**
|
||||
* The statuses a trunk's move puts back to pending on one open pull request's head, from what the forge holds on it
|
||||
* now: none when it has no merge check yet, when a check is pending (on its way, or already out of date — its
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package main
|
||||
|
||||
// Over a stand-in tree holding novox's own files of 2026-10-14 23:25 (read through node-reads for
|
||||
// Over a stand-in tree holding the files of a real machine of 2026-10-14 23:25 (read through node-reads for
|
||||
// novox/hq issue 746): the mode, the per-size controls, meminfo's huge page lines, and fail2ban's and
|
||||
// journald's memory.stat and io.stat, cut to the lines the tool reads.
|
||||
|
||||
@@ -26,7 +26,7 @@ func tree(t *testing.T, files map[string]string) *Machine {
|
||||
return &Machine{Root: root}
|
||||
}
|
||||
|
||||
func novox(t *testing.T) *Machine {
|
||||
func capturedMachine(t *testing.T) *Machine {
|
||||
thp := "sys/kernel/mm/transparent_hugepage/"
|
||||
cg := "sys/fs/cgroup/system.slice/"
|
||||
files := map[string]string{
|
||||
@@ -53,8 +53,8 @@ func novox(t *testing.T) *Machine {
|
||||
return tree(t, files)
|
||||
}
|
||||
|
||||
func TestNovoxBeforeTheChange(t *testing.T) {
|
||||
a, err := novox(t).HugePages(10)
|
||||
func TestTheCapturedMachineBeforeTheChange(t *testing.T) {
|
||||
a, err := capturedMachine(t).HugePages(10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -86,7 +86,7 @@ func TestNovoxBeforeTheChange(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestUnitsAreBoundedAndNoneIsSaid(t *testing.T) {
|
||||
a, err := novox(t).HugePages(1)
|
||||
a, err := capturedMachine(t).HugePages(1)
|
||||
if err != nil || len(a.Units) != 1 {
|
||||
t.Fatalf("%v %+v", err, a)
|
||||
}
|
||||
|
||||
@@ -129,7 +129,7 @@ func (b bed) file(t *testing.T, p string) string {
|
||||
}
|
||||
|
||||
func TestApplySetsTheModeRotatesTheJournalOnceAndRestoreGivesBackWhatWasFound(t *testing.T) {
|
||||
// novox's file on 2026-10-14, read through node-reads.
|
||||
// a real machine's file on 2026-10-14, read through node-reads.
|
||||
b := newBed(t, "madvise", "[always] madvise never\n")
|
||||
out, err := b.run(t, "apply")
|
||||
if err != nil || !strings.Contains(out, "always -> madvise") {
|
||||
|
||||
@@ -107,8 +107,9 @@ func TestAutomxIsAGoBundleForAlpineMountedAtOptMeshAndRestartedWithIt(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
// Clause 3: an upstream artifact pinned by digest, the one route-proxy's trust artifact runs; never an image
|
||||
// built from a Dockerfile (clause 11), and no Dockerfile or base to build one on.
|
||||
// Clause 3: an upstream artifact pinned by digest, the one route-proxy's trust artifact runs. automx's own design
|
||||
// builds no image from a Dockerfile and has no Dockerfile or base to build one on; the catalogue as such does not
|
||||
// refuse the image kind (novox/hq ADR 0487 clause 13).
|
||||
func TestAutomxRunsTheMirroredAlpineImageAndBuildsNoImage(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
c := m.resource("automx")
|
||||
@@ -121,7 +122,7 @@ func TestAutomxRunsTheMirroredAlpineImageAndBuildsNoImage(t *testing.T) {
|
||||
}
|
||||
for _, a := range m.Build["artifacts"].([]any) {
|
||||
if a.(map[string]any)["kind"] == "image" {
|
||||
t.Errorf("artifact %v is an image built from a Dockerfile (ADR 0452)", a)
|
||||
t.Errorf("artifact %v is an image built from a Dockerfile, which automx's design (ADR 0480) does not use", a)
|
||||
}
|
||||
}
|
||||
if _, has := m.Build["on"]; has {
|
||||
|
||||
@@ -40,6 +40,12 @@ with `GOTOOLCHAIN=local`, `GOFLAGS=-mod=mod` and the module's cache directory. E
|
||||
environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. Workspaces older than a day
|
||||
are removed.
|
||||
|
||||
A pin on a pull request's head (issue 1148): when `go mod download` fails on a `git.novox.be/novox/*` pseudo-version, the
|
||||
fetch looks for that commit among the forge's `refs/pull/*/head`, places the repository at it under the workspace's
|
||||
`pinned/beside/` as a plain directory, and hands it to Go (the fetch's commands, and the build with `GOPROXY=off`) through a
|
||||
temporary `-modfile` in the workspace's `pins/` that replaces the module by that directory. A pin Go can fetch is left as
|
||||
it is, a pin on no ref fails naming the module and the commit, and a directory that vendors is never resolved.
|
||||
|
||||
The account's ssh files (`files/forge`, `files/known_hosts`, `files/gitconfig`) are filled on the node by the
|
||||
provisioner's client from the module's entry of `forge-account` (the manifest says `moved`, ADR 0466 §4, ADR 0400 §5).
|
||||
The forge writes that entry only once the module has run and reported its key (ADR 0400 §2), so until then the client
|
||||
@@ -59,14 +65,16 @@ integration-tagged test runs the real go on the fixtures.
|
||||
|
||||
## The integration level
|
||||
|
||||
`build_go_test` with `level: integration` runs `go test -tags integration` against a store and a bus started for the
|
||||
run alone (novox/hq ADR 0470). The store is a postgres of the installed package: `initdb` and `pg_ctl` into
|
||||
`build_go_test` with `level: integration` runs `go test -tags integration,isolation` (isolation: the bubblewrap proofs,
|
||||
which a build node can run) against a store and a bus started for the run alone (novox/hq ADR 0470). The store is a postgres of the installed package: `initdb` and `pg_ctl` into
|
||||
`<work>/<workspace>/run-<id>/pg`, as the module's account, listening on a unix socket there and on no TCP address, with
|
||||
durability off; the tests get `MESH_TEST_POSTGRES` (the socket directory percent-encoded in the URL, because the
|
||||
controller's tests cut the URL at its last slash). The bus is a nats-server from the Go library at the version this
|
||||
module's `go.mod` names, run by this same binary started as `mesh-build-go bus <dir>` (`bus.go`: JetStream on, a
|
||||
loopback port of its own, its address on the first line, stopped when its input closes); the tests get `MESH_TEST_NATS`.
|
||||
Both run inside the step, are stopped, and the run's directory is removed whatever the tests did.
|
||||
Both run inside the step, are stopped, and the run's directory is removed whatever the tests did. The store, the bus and
|
||||
the tests run together in one bubblewrap (ADR 0448, issue 1309): this binary again as `mesh-build-go level <spec>`, which
|
||||
prints the level's result as one line of JSON, so the tests reach the level's own store and bus and nothing of the machine.
|
||||
|
||||
Setup (compiling the tests, the store, the bus) is judged within 30 s and the tests' running time (the wall time of
|
||||
`go test`) within 60 s, apart; a level over either says `over_bound` and names the bound, with the slowest tests for the running
|
||||
@@ -81,7 +89,9 @@ is said until a source exists.
|
||||
one digest of everything those tests read (ADR 0445 §5): the files, by content and under their path in the tree, of the
|
||||
package and every package its test binary reaches (`go list -test -deps`) that lies in the module's own tree; the version
|
||||
and checksum of every module outside it; the package's own `testdata/` files; and the level, the build tags and, at the
|
||||
integration level, the postgres major and the nats-server version. The Go version is not in it: the toolchain enters a
|
||||
integration level, the postgres major and the nats-server version; and, when the fetch placed siblings beside the
|
||||
workspace (`beside`), each one's repository and commit, which the answer echoes as `beside` (the checker keeps a verdict only
|
||||
for the siblings it was judged with). The text's form is `DigestVersion` 3. The Go version is not in it: the toolchain enters a
|
||||
fingerprint once, as the version the build nodes agree on (ADR 0477 §3), and the standard library's packages are that
|
||||
version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors
|
||||
is listed and tested from its vendor directory (`-mod=vendor`).
|
||||
@@ -95,11 +105,16 @@ that vendors), `go build -ldflags "-s -w -X main.builtFor=<system>" -o .mesh-bui
|
||||
module's directory, by the installed go (`GOTOOLCHAIN=local`) from the module cache the fetch filled and no network
|
||||
(`GOPROXY=off`). The binary is named as the build agent names it. The output directory is packed with mesh-sdk's `Pack`,
|
||||
as the build agent packs; two build nodes on the same toolchain give the same digest for the same tree (ADR 0477 §6). The step ends with the artifact, the binary, the digest, the size and
|
||||
the path of the archive, which stays in the workspace's `artifacts` directory: publishing it is the builder's, in a later
|
||||
release. An artifact that is neither a Go bundle nor an archive fails the step naming it.
|
||||
the reference of the archive: once the build ended, this process pushes it to the artifact store (novox/hq ADR 0498 §1 §2,
|
||||
mesh-sdk's `buildstep.PushBlob`) under `<module>/<artifact>` by its digest, a candidate's as a trunk build's, and answers
|
||||
`artifact-store://<module>/<artifact>/blobs/<digest>`; a push that fails fails the step `not stored: <artifact>: <the
|
||||
store's words>`. The packed bytes never stay on the node. The store's address is `${dir:state}/registry`, placed by the
|
||||
manifest, which requires `artifact-store`. The go build runs shut off in bubblewrap (ADR 0448, `isolate.go`): no
|
||||
network, none of the account, the caches beneath a throwaway layer unless the step is told `publish` (a trunk build). An artifact that is neither a Go
|
||||
bundle nor an archive fails the step naming it.
|
||||
|
||||
The same tool packs an archive artifact of any module's tree, whatever its language (ADR 0478 §1, issue 1589): the
|
||||
directory named by `from`, or a file pinned by `url` and `sha256`, packed with the same `Pack` as the build agent's `pack`
|
||||
(`testdata/captured/README` says how its digest was captured). A pinned file is fetched, checked against its sha256 (a
|
||||
mismatch fails the step naming both digests) and laid out as the artifact says (`file`, or unpacked from `within`; gzip and
|
||||
bzip2, not yet xz); it runs no command and needs no Go.
|
||||
mismatch fails the step naming both digests) and laid out as the artifact says (`file`, or unpacked from `within`; gzip, bzip2
|
||||
and xz, read in Go as the build agent reads it, issue 1590); it runs no command and needs no Go.
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package main
|
||||
|
||||
// What this module keeps on the bus, announced from its code (novox/hq ADR 0495 §2 §5; issue 1790): every state it
|
||||
// keeps, each readable by others, since another module reads each today (the builder, the checker and the indexer read toolchains; the builder and the checker read steps; gitea reads keys). A state of its own that it
|
||||
// did not name here is refused at the call (stdio.Keeps).
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// Kept are the states the module keeps.
|
||||
var Kept = []announce.State{
|
||||
{Name: "toolchains", Readable: true},
|
||||
{Name: "keys", Readable: true},
|
||||
{Name: "steps", Readable: true},
|
||||
}
|
||||
|
||||
// keepStates announces Kept as this module's (stdio.Keeps): main calls it before it serves.
|
||||
func keepStates() error { return stdio.Keeps(Kept...) }
|
||||
@@ -0,0 +1,63 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announcetest"
|
||||
)
|
||||
|
||||
// 1790 (ADR 0495 §5): served as the runtime serves it, the module announces on the bus every state it keeps, as its
|
||||
// code declares them. The runtime's part, publishing what the bundle asks to publish, is played by relay.
|
||||
func TestTheModuleAnnouncesEveryStateItKeeps(t *testing.T) {
|
||||
const module = "mesh-build-go"
|
||||
announcetest.Check(t, module, Kept, func(url string) func() {
|
||||
t.Setenv("MESH_MODULE", module)
|
||||
t.Setenv("MESH_NODE", "shanks")
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := keepStates(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inR, inW := io.Pipe()
|
||||
outR, outW := io.Pipe()
|
||||
go func() { _ = stdio.ServeOn(module, nil, inR, outW) }()
|
||||
go relay(t, module, nc, outR, inW)
|
||||
return func() { _ = inW.Close(); _ = outW.Close(); nc.Close() }
|
||||
})
|
||||
}
|
||||
|
||||
// relay is the runtime's side of the channel: each mesh/publish the bundle asks is published on its module's event
|
||||
// subject, mesh.mod.<module>.event.<key>, and answered.
|
||||
func relay(t *testing.T, module string, nc *nats.Conn, from io.Reader, to io.Writer) {
|
||||
sc := bufio.NewScanner(from)
|
||||
sc.Buffer(make([]byte, 1<<20), 1<<24)
|
||||
enc := json.NewEncoder(to)
|
||||
for sc.Scan() {
|
||||
var m struct {
|
||||
ID json.RawMessage `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params struct {
|
||||
Key string `json:"key"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
} `json:"params"`
|
||||
}
|
||||
if json.Unmarshal(sc.Bytes(), &m) != nil || m.Method != "mesh/publish" {
|
||||
continue
|
||||
}
|
||||
if err := nc.Publish("mesh.mod."+module+".event."+m.Params.Key, m.Params.Body); err != nil {
|
||||
t.Errorf("publishing %s: %v", m.Params.Key, err)
|
||||
}
|
||||
_ = nc.Flush()
|
||||
_ = enc.Encode(map[string]any{"id": m.ID, "result": map[string]any{}})
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/therootcompany/xz"
|
||||
)
|
||||
|
||||
// maxUpstream is the largest pinned file fetched and the largest it may unpack to, as the build agent's and the
|
||||
@@ -42,9 +43,9 @@ func (m *Module) httpClient() *http.Client {
|
||||
|
||||
// buildArchive packs one archive artifact of the module in dir: a pinned file when it names a url, else the directory
|
||||
// `from` of the module's tree.
|
||||
func (m *Module) buildArchive(ctx context.Context, workspace, module, dir string, art catalogue.Artifact) (any, error) {
|
||||
func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir string, art catalogue.Artifact) (any, error) {
|
||||
if art.URL != "" {
|
||||
return m.pinnedArchive(ctx, workspace, module, art)
|
||||
return m.pinnedArchive(ctx, workspace, module, name, art)
|
||||
}
|
||||
from := strings.Trim(path.Clean("/"+filepath.ToSlash(art.From)), "/")
|
||||
if strings.TrimSpace(art.From) == "" {
|
||||
@@ -76,12 +77,12 @@ func (m *Module) buildArchive(ctx context.Context, workspace, module, dir string
|
||||
if !within(realDir, realRoot) {
|
||||
return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module: it resolves to %s, and the module is %s", art.Name, art.From, realRoot, realDir)
|
||||
}
|
||||
return m.packInto(workspace, module, art.Name, root)
|
||||
return m.packInto(ctx, workspace, module, name, art.Name, root)
|
||||
}
|
||||
|
||||
// pinnedArchive fetches the file an artifact is pinned to, checks it against the pin, lays it out as the artifact says
|
||||
// (kept whole as `file`, or unpacked from `within`) and packs it.
|
||||
func (m *Module) pinnedArchive(ctx context.Context, workspace, module string, art catalogue.Artifact) (any, error) {
|
||||
func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name string, art catalogue.Artifact) (any, error) {
|
||||
file, err := m.pinnedFile(ctx, workspace, art)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", art.Name, err)
|
||||
@@ -110,7 +111,7 @@ func (m *Module) pinnedArchive(ctx context.Context, workspace, module string, ar
|
||||
} else if err := unpackUpstream(file, laid, art.Within); err != nil {
|
||||
return nil, fmt.Errorf("%s: %s (%s): %w", art.Name, art.URL, art.Version, err)
|
||||
}
|
||||
return m.packInto(workspace, module, art.Name, laid)
|
||||
return m.packInto(ctx, workspace, module, name, art.Name, laid)
|
||||
}
|
||||
|
||||
// pinnedFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes are still the
|
||||
@@ -190,10 +191,13 @@ func sumOf(file string) (string, error) {
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// unpackUpstream unpacks a tar archive compressed with gzip or bzip2 into a directory: of its entries only those within
|
||||
// `within` (all, when empty), with that prefix taken off. Only files and directories, and nothing outside: the
|
||||
// node-engine unpacks nothing else. An xz archive is refused by name; the build agent reads it, this module does not
|
||||
// yet (it is a later issue, not a quiet difference in the archive).
|
||||
// xzDictMax is the largest xz dictionary read (the build agent's): xz's own presets use at most 64 MiB (-9); a stream
|
||||
// asking for more is refused rather than given the memory.
|
||||
const xzDictMax = 64 << 20
|
||||
|
||||
// unpackUpstream unpacks a tar archive compressed with gzip, bzip2 or xz into a directory: of its entries only those
|
||||
// within `within` (all, when empty), with that prefix taken off. Only files and directories, and nothing outside: the
|
||||
// node-engine unpacks nothing else.
|
||||
func unpackUpstream(file, into, within string) error {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
@@ -214,9 +218,22 @@ func unpackUpstream(file, into, within string) error {
|
||||
case bytes.HasPrefix(magic, []byte("BZh")):
|
||||
stream = bzip2.NewReader(buffered)
|
||||
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
|
||||
return errors.New("it is an xz archive, which this build module does not read yet; the build agent does")
|
||||
// As the build agent reads it (mesh-controller internal/builder/upstream_file.go; issue 1590): in Go
|
||||
// (github.com/therootcompany/xz, a port of xz-embedded), never by a tool the node may not carry. xzDictMax is
|
||||
// a real maximum: a block whose LZMA2 dictionary is larger is refused before its dictionary is allocated. The
|
||||
// reader keeps a filter decoder for every block of a stream, so the file's framing is walked first and a file
|
||||
// of more blocks than xzBlocksMax is refused before anything is decoded (xz_framing.go). What the stream
|
||||
// unpacks to is bounded by untar (maxUpstream, maxEntries).
|
||||
if err := xzWithinLimits(file); err != nil {
|
||||
return err
|
||||
}
|
||||
zipped, err := xz.NewReader(buffered, xzDictMax)
|
||||
if err != nil {
|
||||
return fmt.Errorf("it is not an xz stream this build module reads (a dictionary of %d MiB at most): %v", xzDictMax>>20, err)
|
||||
}
|
||||
stream = zipped
|
||||
default:
|
||||
return errors.New("it is not a tar archive compressed with gzip or bzip2; a single file is kept whole by naming it in `file`")
|
||||
return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept whole by naming it in `file`")
|
||||
}
|
||||
return untar(stream, into, within)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
@@ -40,16 +41,11 @@ const (
|
||||
capturedWithGo = "go1.27.1"
|
||||
)
|
||||
|
||||
// sameAsBuildAgent says the archive at res.Path holds the tar stream the build agent's pack wrote, and, under the Go the
|
||||
// sameAsBuildAgent says the archive the build pushed holds the tar stream the build agent's pack wrote, and, under the Go the
|
||||
// digests were captured with, has its digest.
|
||||
func sameAsBuildAgent(t *testing.T, res BuildResult, stream, digest string) {
|
||||
t.Helper()
|
||||
f, err := os.Open(res.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
z, err := gzip.NewReader(f)
|
||||
z, err := gzip.NewReader(bytes.NewReader(archiveOf(t, res)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -157,8 +153,8 @@ func TestAnArchiveOfADirectoryHasTheBuildAgentsDigest(t *testing.T) {
|
||||
if res.Artifact != "files" || res.Size == 0 || res.Binary != "" {
|
||||
t.Errorf("result = %+v", res)
|
||||
}
|
||||
if info, err := os.Stat(res.Path); err != nil || info.Size() != int64(res.Size) {
|
||||
t.Errorf("the archive at %q: %v", res.Path, err)
|
||||
if len(archiveOf(t, res)) != res.Size {
|
||||
t.Errorf("the archive pushed is not %d bytes", res.Size)
|
||||
}
|
||||
if len(*calls) != 0 {
|
||||
t.Errorf("an archive ran %d commands", len(*calls))
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The lead's decision L5 (the checker keeps a verdict of a core repository): the fetch writes the siblings it placed
|
||||
// beside the workspace, and the digest folds each one's repository and commit into every package's digest and echoes
|
||||
// them, so a verdict is kept only for the siblings it was judged with.
|
||||
func TestTheSiblingsCommitsAreInEveryDigestAndEchoed(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, _ := digestFixture(t)
|
||||
alone := digests(t, m, "unit")
|
||||
if len(alone.Beside) != 0 {
|
||||
t.Errorf("a workspace without siblings echoes %v", alone.Beside)
|
||||
}
|
||||
write := func(placed []Sibling) {
|
||||
raw, _ := json.Marshal(placed)
|
||||
if err := os.WriteFile(filepath.Join(m.Work, "ws1", besideRecord), raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
other := strings.Repeat("b", 40)
|
||||
write([]Sibling{{Repository: "novox/mesh-sdk", Commit: commitSHA, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}})
|
||||
with := digests(t, m, "unit")
|
||||
want := []string{"novox/mesh-catalog " + commitSHA, "novox/mesh-sdk " + commitSHA}
|
||||
if !slices.Equal(with.Beside, want) {
|
||||
t.Errorf("echoed %v, want %v", with.Beside, want)
|
||||
}
|
||||
write([]Sibling{{Repository: "novox/mesh-sdk", Commit: other, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}})
|
||||
moved := digests(t, m, "unit")
|
||||
for p, d := range alone.Packages {
|
||||
if with.Packages[p] == d {
|
||||
t.Errorf("%s: the siblings left the digest as it was without them", p)
|
||||
}
|
||||
if moved.Packages[p] == with.Packages[p] {
|
||||
t.Errorf("%s: a sibling's new commit left the digest as it was", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The fetch writes the siblings it placed, which the digest reads.
|
||||
func TestTheFetchRecordsTheSiblingsItPlaced(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n"}
|
||||
m, states := fetchModule(t, f)
|
||||
args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{
|
||||
map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}}}
|
||||
if _, err := fetchTool(t, m)(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res FetchResult
|
||||
json.Unmarshal(st.Result, &res)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(m.Work, res.Workspace, besideRecord))
|
||||
var placed []Sibling
|
||||
if err != nil || json.Unmarshal(raw, &placed) != nil || len(placed) != 1 || placed[0].Repository != "novox/mesh-sdk" || placed[0].Commit != commitSHA {
|
||||
t.Errorf("the record is %s (%v)", raw, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The checker's agreement: the integration level runs and digests under the tags integration and isolation, so the
|
||||
// bubblewrap proofs tagged isolation run in every check on a build node.
|
||||
func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, _ := digestFixture(t)
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, m.Run, &seen)
|
||||
digests(t, m, LevelIntegration)
|
||||
for _, s := range seen {
|
||||
if !strings.Contains(s.command, " -tags integration,isolation ") {
|
||||
t.Errorf("the integration digest listed %q", s.command)
|
||||
}
|
||||
}
|
||||
if DigestVersion != "3" {
|
||||
t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion)
|
||||
}
|
||||
}
|
||||
@@ -3,8 +3,8 @@ package main
|
||||
// The build step (novox/hq ADR 0405 §3 §7, ADR 0453 §7; issue 1487): a Go bundle artifact of a module of a fetched
|
||||
// workspace, built with today's recipe — the build agent's, mesh-controller's internal/builder/toolchain.go (the Go
|
||||
// entry) and builder.go (compile, binaryName, pack) — by the installed Go and no network, packed with mesh-sdk's Pack
|
||||
// so that its digest is the build agent's for the same tree. The archive is left in the workspace; publishing it is
|
||||
// the builder's, in a later release.
|
||||
// so that its digest is the build agent's for the same tree, and pushed to the artifact store by this process once the
|
||||
// build ended (ADR 0498 §1 §2, push.go).
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The Go recipe of the build agent (toolchain.go, the entry of language go): static, trimmed of its own paths and of
|
||||
@@ -33,8 +34,9 @@ type BuildResult struct {
|
||||
Binary string `json:"binary"`
|
||||
Digest string `json:"digest"`
|
||||
Size int `json:"size"`
|
||||
// Path is the archive, in the workspace's directory of the work directory.
|
||||
Path string `json:"path"`
|
||||
// Reference is where the archive is in the artifact store (novox/hq ADR 0498 §1):
|
||||
// artifact-store://<module>/<artifact>/blobs/<digest>.
|
||||
Reference string `json:"reference"`
|
||||
}
|
||||
|
||||
func (m *Module) buildTool() StepTool {
|
||||
@@ -49,6 +51,7 @@ func (m *Module) buildTool() StepTool {
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
|
||||
"module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"},
|
||||
"artifact": map[string]any{"type": "string", "description": "the artifact's name in the module's manifest"},
|
||||
"publish": map[string]any{"type": "boolean", "description": "true for a merged trunk commit alone: the build writes the shared Go caches; without it (a candidate) they are read beneath a throwaway layer"},
|
||||
},
|
||||
"required": []string{"workspace", "module", "artifact"},
|
||||
},
|
||||
@@ -76,6 +79,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
name, _ := args["artifact"].(string)
|
||||
publish, _ := args["publish"].(bool)
|
||||
if !workspaceOK.MatchString(workspace) {
|
||||
return nil, fmt.Errorf("%q is not a workspace id", workspace)
|
||||
}
|
||||
@@ -106,7 +110,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", "))
|
||||
}
|
||||
if art.Kind == "archive" {
|
||||
return m.buildArchive(ctx, workspace, module, dir, *art)
|
||||
return m.buildArchive(ctx, workspace, module, manifest.Module, dir, *art)
|
||||
}
|
||||
if art.Kind != "bundle" || art.Language != "go" {
|
||||
return nil, fmt.Errorf("the artifact %s of %s is a %s of language %q: this module builds Go bundles and archives", name, manifest.Module, art.Kind, art.Language)
|
||||
@@ -133,13 +137,19 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
linker += " -X " + recipeSystemStamp + "=" + system
|
||||
}
|
||||
cmd := []string{"-C", dir, "build", "-ldflags", linker, "-o", filepath.Join(out, binary), "./" + strings.Trim(art.From, "./")}
|
||||
if r := m.Run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" {
|
||||
// Shut off (ADR 0448): the shared caches are written by a trunk build alone (publish), a candidate's go build writes
|
||||
// beneath a throwaway layer.
|
||||
run, err := m.stepRunner(workspace, publish)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r := run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("go build of %s: %s", art.Name, causeLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
if err := m.insideModule(workspace, dir, out); err != nil {
|
||||
return nil, fmt.Errorf("the output directory of %s: %w", art.Name, err)
|
||||
}
|
||||
res, err := m.packInto(workspace, module, art.Name, out)
|
||||
res, err := m.packInto(ctx, workspace, module, manifest.Module, art.Name, out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -147,22 +157,20 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// packInto packs a directory with mesh-sdk's Pack, as the build agent packs every artifact, and leaves the archive in
|
||||
// the workspace's artifacts directory.
|
||||
func (m *Module) packInto(workspace, module, artifact, root string) (BuildResult, error) {
|
||||
// packInto packs a directory with mesh-sdk's Pack, as the build agent packs every artifact, and pushes the archive to
|
||||
// the artifact store from this process, after the build step ended (novox/hq ADR 0498 §1 §2): a candidate's as a trunk
|
||||
// build's, under <module>/<artifact> by its digest. The step answers the reference; a push that fails fails the step
|
||||
// `not stored:`. The packed bytes never stay on the node: they are held in this process until the push answered (the
|
||||
// lead's decision L9).
|
||||
func (m *Module) packInto(ctx context.Context, workspace, module, name, artifact, root string) (BuildResult, error) {
|
||||
archive, digest, err := buildstep.Pack(root)
|
||||
if err != nil {
|
||||
return BuildResult{}, fmt.Errorf("packing %s: %w", artifact, err)
|
||||
}
|
||||
artifacts := filepath.Join(m.Work, workspace, "artifacts")
|
||||
if err := os.MkdirAll(artifacts, 0o700); err != nil {
|
||||
return BuildResult{}, err
|
||||
if err := m.push(ctx, name+"/"+artifact, digest, archive); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("%s%s: %v", delivery.NotStored, artifact, err)
|
||||
}
|
||||
path := filepath.Join(artifacts, strings.ReplaceAll(module, "/", "_")+"."+artifact+".tar.gz")
|
||||
if err := os.WriteFile(path, archive, 0o600); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
return BuildResult{Artifact: artifact, Digest: digest, Size: len(archive), Path: path}, nil
|
||||
return BuildResult{Artifact: artifact, Digest: digest, Size: len(archive), Reference: buildstep.BlobReference(name, artifact, digest)}, nil
|
||||
}
|
||||
|
||||
// buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the installed go
|
||||
@@ -172,6 +180,8 @@ func (m *Module) buildEnv(dir string) []string {
|
||||
mod := "-mod=mod"
|
||||
if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil {
|
||||
mod = "-mod=vendor"
|
||||
} else if modfile, ok := m.pinModfile(dir); ok {
|
||||
mod += " -modfile=" + modfile // a pin on a pull request's head the fetch placed beside the workspace (pins.go)
|
||||
}
|
||||
return []string{
|
||||
"CGO_ENABLED=0",
|
||||
|
||||
@@ -41,7 +41,7 @@ func buildModule(t *testing.T, manifest, content string) (*Module, *keptStates,
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(manifest), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
*calls = append(*calls, cmdRecord{env: env, line: name + " " + strings.Join(args, " ")})
|
||||
for i, a := range args {
|
||||
if a == "-o" {
|
||||
@@ -51,7 +51,8 @@ func buildModule(t *testing.T, manifest, content string) (*Module, *keptStates,
|
||||
}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
}, &[]shutOff{}) // the compile runs shut off (1309)
|
||||
withStore(t, m) // the archive is pushed to a store of the test's own (ADR 0498)
|
||||
return m, states, calls
|
||||
}
|
||||
|
||||
@@ -143,13 +144,10 @@ func TestTheArchiveHoldsOneExecutableAndTwoBuildsAreOneDigest(t *testing.T) {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
first, _ := runBuild(t, m, states, "tools")
|
||||
second, _ := runBuild(t, m, states, "tools")
|
||||
if first.Digest != second.Digest || first.Path != second.Path {
|
||||
if first.Digest != second.Digest || first.Reference != second.Reference {
|
||||
t.Errorf("two builds: %+v and %+v", first, second)
|
||||
}
|
||||
raw, err := os.ReadFile(first.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := archiveOf(t, first)
|
||||
sum := sha256.Sum256(raw)
|
||||
if first.Digest != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
t.Errorf("the digest %s is not the archive's", first.Digest)
|
||||
|
||||
@@ -20,7 +20,13 @@ import (
|
||||
)
|
||||
|
||||
// DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind.
|
||||
const DigestVersion = "2" // 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3)
|
||||
// 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3). 3: the siblings'
|
||||
// commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement).
|
||||
const DigestVersion = "3"
|
||||
|
||||
// integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation
|
||||
// for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run.
|
||||
var integrationTags = []string{"integration", "isolation"}
|
||||
|
||||
// goPackage is the part of `go list -json` a digest reads.
|
||||
type goPackage struct {
|
||||
@@ -71,9 +77,12 @@ func (p *goPackage) hasTests() bool { return len(p.TestGoFiles)+len(p.XTestGoFil
|
||||
// DigestResult is what a digest step answers: one digest per package that has tests at the level, and the packages
|
||||
// that have none.
|
||||
type DigestResult struct {
|
||||
Level string `json:"level"`
|
||||
Packages map[string]string `json:"packages"`
|
||||
WithoutTests []string `json:"without_tests,omitempty"`
|
||||
Level string `json:"level"`
|
||||
Packages map[string]string `json:"packages"`
|
||||
// Beside echoes the siblings folded into every digest, "<owner/repo> <commit>" sorted: the checker keeps a verdict
|
||||
// only when they are the siblings it gave the fetch.
|
||||
Beside []string `json:"beside,omitempty"`
|
||||
WithoutTests []string `json:"without_tests,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Module) digestTool() StepTool {
|
||||
@@ -113,20 +122,27 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
|
||||
if _, err := m.checkPostgres(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tags = []string{"integration"}
|
||||
tags = integrationTags
|
||||
header = append(header, "postgres "+strconv.Itoa(m.Recipe.PostgresMajor), "nats-server "+m.Recipe.NatsServer)
|
||||
default:
|
||||
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
|
||||
}
|
||||
header = append(header, "tags "+strings.Join(tags, ","))
|
||||
beside, err := m.besideLines(workspace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, b := range beside {
|
||||
header = append(header, "beside "+b)
|
||||
}
|
||||
|
||||
dirs, err := m.goModDirs(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := DigestResult{Level: level, Packages: map[string]string{}}
|
||||
res := DigestResult{Level: level, Packages: map[string]string{}, Beside: beside}
|
||||
for _, d := range dirs {
|
||||
pkgs, err := m.listPackages(ctx, tree, d, tags)
|
||||
pkgs, err := m.listPackages(ctx, workspace, tree, d, tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -139,14 +155,18 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
|
||||
}
|
||||
|
||||
// listPackages is `go list -test -deps -json` of the directory's packages, by import path.
|
||||
func (m *Module) listPackages(ctx context.Context, tree, dir string, tags []string) (map[string]*goPackage, error) {
|
||||
func (m *Module) listPackages(ctx context.Context, workspace, tree, dir string, tags []string) (map[string]*goPackage, error) {
|
||||
abs := filepath.Join(tree, dir)
|
||||
args := []string{"-C", abs, "list", "-test", "-deps", "-json"}
|
||||
if len(tags) > 0 {
|
||||
args = append(args, "-tags", strings.Join(tags, ","))
|
||||
}
|
||||
args = append(args, "./...")
|
||||
r := m.Run(ctx, m.goEnvFor(abs), "go", args...)
|
||||
run, err := m.stepRunner(workspace, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r := run(ctx, m.goEnvFor(abs), "go", args...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("go list in %s: %s", dir, causeLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ func digestFixture(t *testing.T) (*Module, string) {
|
||||
m := moduleWith(newStates(), "go1.27.1")
|
||||
m.Home, m.Work, m.Cache = t.TempDir(), work, t.TempDir()
|
||||
real := StepRunner(m.Home)
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch name {
|
||||
case "git":
|
||||
return forgekey.Ran{Stdout: "go.mod\n"}
|
||||
@@ -38,7 +38,7 @@ func digestFixture(t *testing.T) (*Module, string) {
|
||||
return forgekey.Ran{Stdout: "postgres (PostgreSQL) " + strconv.Itoa(m.Recipe.PostgresMajor) + ".1\n"}
|
||||
}
|
||||
return real(ctx, env, name, args...)
|
||||
}
|
||||
}, &[]shutOff{}) // go list runs shut off (1309); here the real go, unshut, reads the fixture
|
||||
return m, tree
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
|
||||
// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the
|
||||
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
|
||||
const ForgeAlias = "mesh-forge"
|
||||
@@ -37,6 +39,18 @@ type FetchResult struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Commit string `json:"commit"`
|
||||
Modules []string `json:"modules"`
|
||||
// The merge preview (ADR 0484 clause 3), only when the call gave a base: the trunk commit, the merge commit of the
|
||||
// head onto it, the paths the merge changes, and the tree of each module directory (a directory holding a
|
||||
// module.json) in the merge. A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the
|
||||
// files, and the workspace is left at the trunk commit.
|
||||
Base string `json:"base,omitempty"`
|
||||
Merge string `json:"merge,omitempty"`
|
||||
Conflicts []string `json:"conflicts,omitempty"`
|
||||
Changed []string `json:"changed,omitempty"`
|
||||
Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"`
|
||||
// Beside are the sibling repositories placed as plain directories under the workspace's beside directory, which
|
||||
// build_go_test hands to the tests as MESH_CHECK_BESIDE (ADR 0484 clause 4).
|
||||
Beside []Sibling `json:"beside,omitempty"`
|
||||
}
|
||||
|
||||
// fetchTool is the fetch step tool.
|
||||
@@ -52,6 +66,14 @@ func (m *Module) fetchTool() StepTool {
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
||||
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
||||
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
|
||||
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
|
||||
"dir": buildstep.FetchRequestProperties()["dir"],
|
||||
"paths": buildstep.FetchRequestProperties()["paths"],
|
||||
"beside": map[string]any{"type": "array", "description": "optional sibling repositories the tests read, each placed as a plain directory (no symlink) under the workspace's beside/<name>, which build_go_test gives the tests as MESH_CHECK_BESIDE",
|
||||
"items": map[string]any{"type": "object", "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
||||
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
||||
}, "required": []string{"repository", "commit"}}},
|
||||
},
|
||||
"required": []string{"repository", "commit"},
|
||||
},
|
||||
@@ -61,11 +83,16 @@ func (m *Module) fetchTool() StepTool {
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
commit, _ := args["commit"].(string)
|
||||
ref, _ := args["ref"].(string)
|
||||
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
|
||||
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
|
||||
// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
|
||||
// named paths, the workspace is narrowed to them (buildstep.Narrow).
|
||||
req, err := buildstep.ParseFetchRequest(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
||||
siblings, err := siblingsArg(args["beside"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
id := buildstep.NewID()
|
||||
@@ -74,26 +101,79 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
|
||||
if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref); err != nil {
|
||||
remote := m.remote(repository)
|
||||
if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.FetchOptions...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := FetchResult{Workspace: id, Commit: commit}
|
||||
if base != "" {
|
||||
if err := m.merge(ctx, &res, tree, remote, base, commit); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if res.Beside, err = m.placeSiblings(ctx, root, siblings); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if (base == "" || res.Merge != "") && len(req.Paths) > 0 {
|
||||
if err := buildstep.Narrow(tree, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Said beside the tree, so the steps that list the tree's go.mod files leave out those narrowed away.
|
||||
if err := os.WriteFile(filepath.Join(root, narrowedMark), nil, 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
dirs, err := m.goModDirs(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.Modules = dirs
|
||||
if base != "" && res.Merge == "" {
|
||||
return res, nil // a conflict: the tree is at the trunk, nothing of the head is built or tested
|
||||
}
|
||||
for _, d := range dirs {
|
||||
if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil {
|
||||
continue // a module that vendors resolves nothing
|
||||
}
|
||||
for _, verb := range []string{"download", "verify"} {
|
||||
r := m.Run(ctx, m.goEnv(), "go", "-C", filepath.Join(tree, d), "mod", verb)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("go mod %s in %s: %s", verb, d, causeLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr))
|
||||
}
|
||||
if err := m.downloadModules(ctx, root, filepath.Join(tree, d), d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return FetchResult{Workspace: id, Commit: commit, Modules: dirs}, nil
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// downloadModules downloads and verifies the Go modules of the directory dir (d in the tree). A pin of a forge module
|
||||
// that Go cannot fetch is looked for among the forge's pull request heads and handed to Go by a modfile (pins.go); a
|
||||
// failure that names no such pin is the first failure.
|
||||
func (m *Module) downloadModules(ctx context.Context, root, dir, d string) error {
|
||||
for {
|
||||
failure := m.downloadAndVerify(ctx, dir, d)
|
||||
if failure == nil {
|
||||
return nil
|
||||
}
|
||||
progress, err := m.resolvePins(ctx, root, dir, failure.Error())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !progress {
|
||||
return failure
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error {
|
||||
for _, verb := range []string{"download", "verify"} {
|
||||
cmd, env := []string{"-C", dir, "mod", verb}, m.goEnvFor(dir)
|
||||
if modfile, ok := m.pinModfile(dir); ok && verb == "verify" {
|
||||
// go mod verify takes no -modfile: it is run on the modfile itself, which names the placed repositories.
|
||||
cmd, env = []string{"-C", filepath.Dir(modfile), "mod", verb}, m.goEnv()
|
||||
}
|
||||
r := m.Run(ctx, env, "go", cmd...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("go mod %s in %s: %s", verb, d, causeLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// goModDirs are the directories of the tree that hold a go.mod, relative to it, in order.
|
||||
@@ -104,7 +184,12 @@ func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
|
||||
}
|
||||
dirs := []string{}
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
if l = strings.TrimSpace(l); l != "" && !inTestdata(l) {
|
||||
if narrowed(tree) {
|
||||
if _, err := os.Stat(filepath.Join(tree, filepath.FromSlash(l))); err != nil {
|
||||
continue // outside a narrowed workspace's paths (buildstep.Narrow): no module of this build
|
||||
}
|
||||
}
|
||||
dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
|
||||
}
|
||||
}
|
||||
@@ -112,6 +197,17 @@ func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
|
||||
return dirs, nil
|
||||
}
|
||||
|
||||
// inTestdata says a path lies under a directory named testdata, whose go.mod files are fixtures Go ignores, not modules
|
||||
// of the tree (issue 1810).
|
||||
func inTestdata(path string) bool {
|
||||
for _, seg := range strings.Split(filepath.ToSlash(path), "/") {
|
||||
if seg == "testdata" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// goEnv is what a go command of a step is given beyond the base environment: the installed go alone (GOTOOLCHAIN=local, ADR 0477 §1: no toolchain is ever downloaded),
|
||||
// modules resolved by their go.mod and go.sum, the module's own caches, the mesh's own modules fetched directly (by
|
||||
// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory
|
||||
@@ -134,6 +230,8 @@ func (m *Module) goEnvFor(dir string) []string {
|
||||
env := m.goEnv()
|
||||
if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil {
|
||||
env = append(env, "GOFLAGS=-mod=vendor") // the last value of a name is the one used
|
||||
} else if modfile, ok := m.pinModfile(dir); ok {
|
||||
env = append(env, "GOFLAGS=-mod=mod -modfile="+modfile) // a pin on a pull request's head (pins.go)
|
||||
}
|
||||
return env
|
||||
}
|
||||
@@ -183,3 +281,12 @@ func firstLine(s string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// narrowedMark is the file beside a workspace's tree saying the fetch narrowed it to a module's named paths.
|
||||
const narrowedMark = "narrowed"
|
||||
|
||||
// narrowed says whether the fetch narrowed the workspace whose tree this is.
|
||||
func narrowed(tree string) bool {
|
||||
_, err := os.Stat(filepath.Join(filepath.Dir(tree), narrowedMark))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
@@ -30,7 +30,8 @@ type fakeStep struct {
|
||||
gomods string
|
||||
gate chan struct{} // when set, the first git command waits for it
|
||||
goRule func(line string) forgekey.Ran
|
||||
vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout
|
||||
vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout
|
||||
rule func(line string) (forgekey.Ran, bool) // answers a command before the defaults do
|
||||
}
|
||||
|
||||
func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
@@ -41,6 +42,11 @@ func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...s
|
||||
if f.gate != nil && strings.Contains(line, " init ") {
|
||||
<-f.gate
|
||||
}
|
||||
if f.rule != nil {
|
||||
if r, ok := f.rule(line); ok {
|
||||
return r
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case name == "git" && strings.Contains(line, " rev-parse HEAD"):
|
||||
return forgekey.Ran{Stdout: commitSHA + "\n"}
|
||||
|
||||
@@ -143,8 +143,9 @@ func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) {
|
||||
// template in the files/ archive (novox/hq ADR 0349 §2).
|
||||
func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) {
|
||||
m := readForgeManifest(t)
|
||||
if len(m.Requires) != 2 || m.Requires[0] != "forge-account" || m.Requires[1] != "go-toolchain" {
|
||||
t.Errorf("requires = %v, want [forge-account go-toolchain]", m.Requires)
|
||||
// artifact-store: the module pushes a done build's archive itself (ADR 0498 §2).
|
||||
if len(m.Requires) != 3 || m.Requires[0] != "forge-account" || m.Requires[1] != "go-toolchain" || m.Requires[2] != "artifact-store" {
|
||||
t.Errorf("requires = %v, want [forge-account go-toolchain artifact-store]", m.Requires)
|
||||
}
|
||||
keeps := false
|
||||
for _, s := range m.State {
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// 1810/AC1: the go.mod files under a testdata directory are fixtures Go itself ignores, not modules of the tree, so
|
||||
// the build modules neither test nor digest them (a repository holding this module held one whose test reads
|
||||
// MESH_CHECK_BESIDE, which failed every check of it).
|
||||
func TestGoModDirsLeaveOutTheFixturesUnderTestdata(t *testing.T) {
|
||||
m := &Module{}
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
return forgekey.Ran{Stdout: "checks/deps/go.mod\ngo.mod\nmodules/x/cmd/x/testdata/repos/beside/go.mod\nmodules/x/go.mod\ntestdata/go.mod\n"}
|
||||
}
|
||||
got, err := m.goModDirs(context.Background(), t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{".", "checks/deps", "modules/x"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("go.mod directories = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
@@ -33,11 +34,22 @@ func (m *Module) integrationBounds() Bounds {
|
||||
return b
|
||||
}
|
||||
|
||||
func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string) (res *TestResult, err error) {
|
||||
func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string, pattern string) (res *TestResult, err error) {
|
||||
major, err := m.checkPostgres(ctx)
|
||||
if err != nil {
|
||||
return nil, err // a *buildstep.Refusal: this level only
|
||||
}
|
||||
if !m.Inside {
|
||||
// The store, the bus and the tests run together in one shut-off process (level.go), so the tests reach the
|
||||
// level's own store and bus and nothing else.
|
||||
return m.integrationShutOff(ctx, workspace, packages, pattern)
|
||||
}
|
||||
return m.integrationHere(ctx, workspace, tree, packages, pattern, major)
|
||||
}
|
||||
|
||||
// integrationHere is the level itself, in the shut-off process: the store and the bus started for the run, the tests
|
||||
// compiled and run against them, both stopped and the run's directory removed however it ends.
|
||||
func (m *Module) integrationHere(ctx context.Context, workspace, tree string, packages []string, pattern string, major int) (res *TestResult, err error) {
|
||||
run := filepath.Join(m.Work, workspace, "run-"+buildstep.NewID())
|
||||
if err := os.MkdirAll(run, 0o700); err != nil {
|
||||
return nil, err
|
||||
@@ -60,16 +72,17 @@ func (m *Module) integration(ctx context.Context, workspace, tree string, packag
|
||||
return nil, fmt.Errorf("the integration level's bus: %w", err)
|
||||
}
|
||||
defer stopBus()
|
||||
env := []string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}
|
||||
env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...)
|
||||
// Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run
|
||||
// below finds them built. Whether they compile is for the run to say; its failure is not read here.
|
||||
for _, d := range dirs {
|
||||
m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", "integration", "-run", "^$", "-count=1", "./...")
|
||||
m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1", "./...")
|
||||
}
|
||||
setup := m.clock().Sub(began)
|
||||
|
||||
return m.runLevel(ctx, tree, levelOpts{
|
||||
level: LevelIntegration, tags: []string{"integration"}, packages: packages, bounds: m.integrationBounds(),
|
||||
workspace: workspace, run: pattern,
|
||||
level: LevelIntegration, tags: integrationTags, packages: packages, bounds: m.integrationBounds(),
|
||||
env: env,
|
||||
setup: setup, wall: true,
|
||||
store: "postgres " + strconv.Itoa(major), bus: "nats-server " + m.Recipe.NatsServer,
|
||||
|
||||
@@ -107,6 +107,9 @@ func integModule(t *testing.T, f *integFake) (*Module, *keptStates) {
|
||||
m := moduleWith(states, "go1.27.1")
|
||||
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
|
||||
m.Run, m.Now, m.StartBus = f.run, f.now, f.startBus
|
||||
// These tests judge the level as it runs in its shut-off process (level.go, 1309-S5): the store, the bus and the
|
||||
// tests side by side. That the step starts that one process shut off is isolate_test.go's.
|
||||
m.Inside = true
|
||||
if err := os.MkdirAll(filepath.Join(m.Work, "ws1", "tree"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -177,7 +180,7 @@ func TestTheStoreListensOnASocketUnderTheRunAndNowhereElse(t *testing.T) {
|
||||
test = c
|
||||
}
|
||||
}
|
||||
if !strings.Contains(test.line, "-tags integration") {
|
||||
if !strings.Contains(test.line, "-tags integration,isolation") { // the bubblewrap proofs run too (the checker's agreement)
|
||||
t.Errorf("the run does not ask for the tag: %s", test.line)
|
||||
}
|
||||
env := map[string]string{}
|
||||
@@ -277,7 +280,7 @@ func TestAFailedOrPanickedRunStopsTheServersAndRemovesTheRunDirectory(t *testing
|
||||
m, _ = integModule(t, g)
|
||||
func() {
|
||||
defer func() { recover() }()
|
||||
m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil)
|
||||
m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil, "")
|
||||
}()
|
||||
checkCleaned(g, m)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
// The run steps shut off (novox/hq ADR 0448; issue 1309, lead decision L2 of R4e): every command that runs a change's
|
||||
// code (go test, go build, go list) runs through mesh-sdk's buildstep.Isolated, in bubblewrap: no network, none of the
|
||||
// account (its home, its ssh key, its bus login, the runtime directory), the workspace read-write, the shared Go caches
|
||||
// read-only under a throwaway writable layer, or read-write for a trunk build alone (publish). The fetch step (git, go
|
||||
// mod download) holds the key and the network and runs none of the change's code, so it runs as before.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// BubblewrapLeast is the oldest bubblewrap a run step is shut off with: --overlay-src and --tmp-overlay, which keep the
|
||||
// shared caches unchanged by a candidate, came in 0.10.
|
||||
const BubblewrapLeast = "0.10"
|
||||
|
||||
// caches are the shared warm caches of the module: Go's module cache and its build cache.
|
||||
func (m *Module) caches() []string {
|
||||
return []string{filepath.Join(m.Cache, "gomod"), filepath.Join(m.Cache, "gobuild")}
|
||||
}
|
||||
|
||||
// isolation is how a run step of the workspace is shut off: the workspace (its tree, its siblings, its run directories,
|
||||
// its packed artifacts) read-write, the caches beneath a throwaway layer unless write, and an environment of a PATH and
|
||||
// a locale alone (the step's HOME is its own, inside its /tmp).
|
||||
func (m *Module) isolation(workspace string, write bool) buildstep.Isolation {
|
||||
path := os.Getenv("PATH")
|
||||
if path == "" {
|
||||
path = defaultPath
|
||||
}
|
||||
return buildstep.Isolation{
|
||||
Workspace: filepath.Join(m.Work, workspace),
|
||||
Caches: m.caches(),
|
||||
WriteCaches: write,
|
||||
Env: []string{"PATH=" + path, "LC_ALL=C"},
|
||||
}
|
||||
}
|
||||
|
||||
// stepRunner is the runner of a run step of the workspace: buildstep.Isolated over m.Run, so every command it runs is
|
||||
// shut off; write is true for a trunk build alone (the shared caches are written by trunk builds only). Inside is the
|
||||
// integration level's own process, already running shut off (level.go): it runs its commands as they are.
|
||||
func (m *Module) stepRunner(workspace string, write bool) (forgekey.Runner, error) {
|
||||
if m.Inside {
|
||||
return m.Run, nil
|
||||
}
|
||||
for _, c := range m.caches() {
|
||||
// bubblewrap binds only a directory that is there; the fetch makes them, a step on a fresh node may come first.
|
||||
if err := os.MkdirAll(c, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("the shared cache %s: %w", c, err)
|
||||
}
|
||||
}
|
||||
return buildstep.Isolated(m.Run, m.isolation(workspace, write)), nil
|
||||
}
|
||||
|
||||
// ProbeBubblewrap reads the installed bubblewrap's version (`bwrap --version`: "bubblewrap 0.13.0").
|
||||
func ProbeBubblewrap(ctx context.Context) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, ProbeTimeout)
|
||||
defer cancel()
|
||||
r := forgekey.ExecRunnerClean(0)(ctx, []string{"PATH=" + defaultPath}, buildstep.Bubblewrap, "--version")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("`%s --version` failed: %s", buildstep.Bubblewrap, firstLine(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
return parseBubblewrapVersion(r.Stdout)
|
||||
}
|
||||
|
||||
var bubblewrapVersion = regexp.MustCompile(`^bubblewrap (\d+)\.(\d+)(\.\d+)?\s*$`)
|
||||
|
||||
func parseBubblewrapVersion(out string) (string, error) {
|
||||
f := bubblewrapVersion.FindStringSubmatch(strings.TrimSpace(out))
|
||||
if f == nil {
|
||||
return "", fmt.Errorf("`%s --version` answered %q", buildstep.Bubblewrap, strings.TrimSpace(out))
|
||||
}
|
||||
return f[1] + "." + f[2] + f[3], nil
|
||||
}
|
||||
|
||||
// bubblewrapProblem says why the installed bubblewrap cannot shut a run step off, or "" when it can: it is missing, or
|
||||
// older than BubblewrapLeast.
|
||||
func bubblewrapProblem(version string, err error) string {
|
||||
if err != nil {
|
||||
return "bubblewrap cannot be run, and every build and test step runs in it (ADR 0448): " + err.Error()
|
||||
}
|
||||
if !atLeast(version, BubblewrapLeast) {
|
||||
return fmt.Sprintf("bubblewrap %s installed, and a build or test step needs %s or later (ADR 0448)", version, BubblewrapLeast)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// atLeast says whether the dotted version v is least or later.
|
||||
func atLeast(v, least string) bool {
|
||||
a, b := strings.Split(v, "."), strings.Split(least, ".")
|
||||
for i := 0; i < len(b); i++ {
|
||||
var x, y int
|
||||
if i < len(a) {
|
||||
x, _ = strconv.Atoi(a[i])
|
||||
}
|
||||
y, _ = strconv.Atoi(b[i])
|
||||
if x != y {
|
||||
return x > y
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// checkBubblewrap refuses a step when bubblewrap cannot shut it off; nil Bubblewrap probes nothing (a module that only
|
||||
// guards).
|
||||
func (m *Module) checkBubblewrap(ctx context.Context) error {
|
||||
if m.Bubblewrap == nil {
|
||||
return nil
|
||||
}
|
||||
v, err := m.Bubblewrap(ctx)
|
||||
if why := bubblewrapProblem(v, err); why != "" {
|
||||
return &buildstep.Refusal{Text: why, Want: BubblewrapLeast + " or later", Have: v}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// shutOff is one command a step ran in bubblewrap: the bubblewrap words before "--", what it bound read-write, read-only
|
||||
// and beneath a throwaway layer, and the command with its environment.
|
||||
type shutOff struct {
|
||||
flags []string
|
||||
binds, roBinds, overlays []string
|
||||
env []string
|
||||
command string
|
||||
chdir string
|
||||
}
|
||||
|
||||
// bwrapArity is how many words each bubblewrap option the SDK writes takes.
|
||||
var bwrapArity = map[string]int{"--unshare-all": 0, "--die-with-parent": 0, "--new-session": 0, "--clearenv": 0,
|
||||
"--proc": 1, "--dev": 1, "--tmpfs": 1, "--dir": 1, "--chdir": 1, "--overlay-src": 1, "--tmp-overlay": 1,
|
||||
"--ro-bind": 2, "--ro-bind-try": 2, "--bind": 2, "--symlink": 2, "--setenv": 2}
|
||||
|
||||
// throughBubblewrap is the fake runner of a step's tests: a bubblewrap command is read back (recorded in seen) and its
|
||||
// command handed to run with the environment bubblewrap was told to set, the way it runs in the step; any other command
|
||||
// (the fetch's git, postgres --version) goes to run as it is.
|
||||
func throughBubblewrap(t *testing.T, run forgekey.Runner, seen *[]shutOff) forgekey.Runner {
|
||||
t.Helper()
|
||||
var mu sync.Mutex
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if name != buildstep.Bubblewrap {
|
||||
return run(ctx, env, name, args...)
|
||||
}
|
||||
var s shutOff
|
||||
i := 0
|
||||
for ; i < len(args) && args[i] != "--"; i++ {
|
||||
n, ok := bwrapArity[args[i]]
|
||||
if !ok || i+n >= len(args) {
|
||||
t.Errorf("bubblewrap was given %q, which the test does not know: %v", args[i], args)
|
||||
return forgekey.Ran{Status: 1, Err: "unknown bubblewrap option"}
|
||||
}
|
||||
s.flags = append(s.flags, args[i])
|
||||
switch args[i] {
|
||||
case "--bind":
|
||||
s.binds = append(s.binds, args[i+1])
|
||||
case "--ro-bind", "--ro-bind-try":
|
||||
s.roBinds = append(s.roBinds, args[i+1])
|
||||
case "--overlay-src":
|
||||
s.overlays = append(s.overlays, args[i+1])
|
||||
case "--setenv":
|
||||
s.env = append(s.env, args[i+1]+"="+args[i+2])
|
||||
case "--chdir":
|
||||
s.chdir = args[i+1]
|
||||
}
|
||||
i += n
|
||||
}
|
||||
if i+1 >= len(args) {
|
||||
t.Errorf("bubblewrap was given no command: %v", args)
|
||||
return forgekey.Ran{Status: 1, Err: "no command"}
|
||||
}
|
||||
s.command = args[i+1] + " " + strings.Join(args[i+2:], " ")
|
||||
mu.Lock()
|
||||
*seen = append(*seen, s)
|
||||
mu.Unlock()
|
||||
return run(ctx, s.env, args[i+1], args[i+2:]...)
|
||||
}
|
||||
}
|
||||
|
||||
// checkShutOff fails the test unless s ran with no network and none of the account: every namespace unshared, the
|
||||
// environment cleared, nothing of the home or the state bound, the caches beneath a throwaway layer unless write.
|
||||
func checkShutOff(t *testing.T, m *Module, s shutOff, write bool) {
|
||||
t.Helper()
|
||||
for _, f := range []string{"--unshare-all", "--clearenv", "--die-with-parent"} {
|
||||
if !slices.Contains(s.flags, f) {
|
||||
t.Errorf("%s ran without %s: %v", s.command, f, s.flags)
|
||||
}
|
||||
}
|
||||
for _, b := range append(append([]string{}, s.binds...), s.roBinds...) {
|
||||
if m.Home != "" && (b == m.Home || strings.HasPrefix(b, m.Home+"/")) {
|
||||
t.Errorf("%s was given the account's home: %s", s.command, b)
|
||||
}
|
||||
if strings.HasPrefix(b, "/run") {
|
||||
t.Errorf("%s was given the runtime directory: %s", s.command, b)
|
||||
}
|
||||
}
|
||||
for _, e := range s.env {
|
||||
if strings.HasPrefix(e, "HOME="+m.Home) && m.Home != "" {
|
||||
t.Errorf("%s was told the account's home: %s", s.command, e)
|
||||
}
|
||||
}
|
||||
caches := m.caches()
|
||||
for _, c := range caches {
|
||||
bound, layered := slices.Contains(s.binds, c), slices.Contains(s.overlays, c)
|
||||
switch {
|
||||
case write && (!bound || layered):
|
||||
t.Errorf("a trunk build's %s does not write the shared cache %s (bound %v, layered %v)", s.command, c, bound, layered)
|
||||
case !write && (bound || !layered):
|
||||
t.Errorf("%s may write the shared cache %s (bound %v, layered %v): only a trunk build does", s.command, c, bound, layered)
|
||||
}
|
||||
}
|
||||
if want := filepath.Join(m.Work, "ws1"); !slices.Contains(s.binds, want) {
|
||||
t.Errorf("%s was not given its workspace %s read-write: %v", s.command, want, s.binds)
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S2: every go test of the unit level runs shut off in bubblewrap: no network, none of the account, the caches
|
||||
// beneath a throwaway layer.
|
||||
func TestTheUnitLevelRunsEveryGoTestShutOff(t *testing.T) {
|
||||
m, states, ws, calls := workspaceFor(t, "failing")
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, m.Run, &seen)
|
||||
if _, st := runUnit(t, m, states, ws); st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
ran := 0
|
||||
for _, c := range *calls {
|
||||
if strings.HasPrefix(c.line, "go ") {
|
||||
ran++
|
||||
}
|
||||
}
|
||||
if ran == 0 || len(seen) != ran {
|
||||
t.Fatalf("%d go commands ran, %d of them shut off: %v", ran, len(seen), *calls)
|
||||
}
|
||||
for _, s := range seen {
|
||||
checkShutOff(t, m, s, false)
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S2: a try's tests: build_go_test's run pattern reaches go test as one -run word; a pattern that is a flag is refused.
|
||||
func TestTheRunPatternReachesGoTestAndAFlagIsRefused(t *testing.T) {
|
||||
m, states, ws, calls := workspaceFor(t, "failing")
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, m.Run, &seen)
|
||||
run := stepTool(t, m, "build_go_test")
|
||||
if _, err := run(map[string]any{"workspace": ws, "level": "unit", "run": "TestA|TestB"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
states.waitEnd(t)
|
||||
found := false
|
||||
for _, c := range *calls {
|
||||
found = found || strings.Contains(c.line, " -run TestA|TestB ")
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no go test was given -run TestA|TestB: %v", *calls)
|
||||
}
|
||||
for _, bad := range []string{"-exec=/bin/sh", "a b", "(unclosed"} {
|
||||
if runPatternOK(bad) {
|
||||
t.Errorf("the run pattern %q was taken", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S3 and S4: a candidate's go build runs shut off with the caches beneath a throwaway layer; a trunk build
|
||||
// (publish) writes them.
|
||||
func TestABuildRunsShutOffAndOnlyATrunkBuildWritesTheCaches(t *testing.T) {
|
||||
for _, publish := range []bool{false, true} {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "binary")
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, m.Run, &seen)
|
||||
run := stepTool(t, m, "build_go_build")
|
||||
args := map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "tools"}
|
||||
if publish {
|
||||
args["publish"] = true
|
||||
}
|
||||
if _, err := run(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepPassed {
|
||||
t.Fatalf("publish %v: the step ended %s: %s", publish, st.State, st.Reason)
|
||||
}
|
||||
if len(seen) != 1 || !strings.HasPrefix(seen[0].command, "go ") || !strings.Contains(seen[0].command, " build ") {
|
||||
t.Fatalf("publish %v: shut off %v, want the one go build", publish, seen)
|
||||
}
|
||||
checkShutOff(t, m, seen[0], publish)
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S3: the digest's go list runs shut off.
|
||||
func TestTheDigestListsPackagesShutOff(t *testing.T) {
|
||||
m, _ := digestFixture(t)
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, m.Run, &seen)
|
||||
digests(t, m, LevelUnit)
|
||||
if len(seen) == 0 {
|
||||
t.Fatal("the digest ran no command shut off")
|
||||
}
|
||||
for _, s := range seen {
|
||||
if !strings.HasPrefix(s.command, "go ") || !strings.Contains(s.command, " list ") {
|
||||
t.Errorf("the digest shut off %q, want go list", s.command)
|
||||
}
|
||||
checkShutOff(t, m, s, false)
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S5: the integration level runs its store, its bus and its tests in ONE shut-off process (this program again,
|
||||
// `level <spec>`), so they reach each other and nothing else; its answer is the level's result. Nothing of the level
|
||||
// runs outside but reading postgres's version.
|
||||
func TestTheIntegrationLevelRunsInOneShutOffProcess(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "failing")
|
||||
m.Executable = "/opt/bundle/mesh-build-go"
|
||||
want := TestResult{Level: LevelIntegration, Passed: true, Tests: 3, Packages: []PackageResult{}, Store: "postgres 18"}
|
||||
var outside []string
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch {
|
||||
case name == "postgres" && len(args) == 1 && args[0] == "--version":
|
||||
return forgekey.Ran{Stdout: "postgres (PostgreSQL) 18.6\n"}
|
||||
case name == m.Executable && len(args) == 2 && args[0] == levelArg:
|
||||
var spec levelSpec
|
||||
if err := json.Unmarshal([]byte(args[1]), &spec); err != nil || spec.Workspace != ws || spec.Work != m.Work {
|
||||
t.Errorf("the level's spec %s: %v", args[1], err)
|
||||
}
|
||||
raw, _ := json.Marshal(levelAnswer{Result: &want})
|
||||
return forgekey.Ran{Stdout: string(raw) + "\n"}
|
||||
}
|
||||
outside = append(outside, name+" "+strings.Join(args, " "))
|
||||
return forgekey.Ran{Status: 1, Err: "not expected here"}
|
||||
}, &seen)
|
||||
run := stepTool(t, m, "build_go_test")
|
||||
if _, err := run(map[string]any{"workspace": ws, "level": LevelIntegration}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
var got TestResult
|
||||
if err := json.Unmarshal(st.Result, &got); err != nil || got.Tests != 3 || !got.Passed || got.Store != "postgres 18" {
|
||||
t.Errorf("the result %s (%v), want the shut-off process's", st.Result, err)
|
||||
}
|
||||
if len(outside) > 0 {
|
||||
t.Errorf("ran outside the shut-off process: %v", outside)
|
||||
}
|
||||
if len(seen) != 1 {
|
||||
t.Fatalf("shut off %d processes, want one: %v", len(seen), seen)
|
||||
}
|
||||
checkShutOff(t, m, seen[0], false)
|
||||
if !slices.Contains(seen[0].roBinds, m.Executable) || slices.Contains(seen[0].roBinds, filepath.Dir(m.Executable)) {
|
||||
t.Errorf("the program is bound %v; want the program alone, read-only", seen[0].roBinds)
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S1: a step is refused, and the module is unhealthy, while bubblewrap is missing or older than 0.10, naming it.
|
||||
func TestAnOldOrMissingBubblewrapRefusesTheStepsAndIsUnhealthy(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
version string
|
||||
err error
|
||||
ok bool
|
||||
}{{"0.13.0", nil, true}, {"0.10.0", nil, true}, {"0.9.0", nil, false}, {"", errors.New("bwrap: not found"), false}} {
|
||||
m := moduleWith(newStates(), "go1.27.1")
|
||||
m.Bubblewrap = func(context.Context) (string, error) { return c.version, c.err }
|
||||
err := m.Check(context.Background())
|
||||
h := m.Health(context.Background())
|
||||
raw, _ := json.Marshal(h)
|
||||
if c.ok {
|
||||
if err != nil || strings.Contains(string(raw), "unhealthy") {
|
||||
t.Errorf("bubblewrap %q: refused %v, health %s", c.version, err, raw)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var ref *buildstep.Refusal
|
||||
if !errors.As(err, &ref) || !strings.Contains(err.Error(), "bubblewrap") {
|
||||
t.Errorf("bubblewrap %q %v: Check answered %v, want a refusal naming bubblewrap", c.version, c.err, err)
|
||||
}
|
||||
if !strings.Contains(string(raw), "unhealthy") || !strings.Contains(string(raw), "bubblewrap") {
|
||||
t.Errorf("bubblewrap %q %v: health %s, want unhealthy naming bubblewrap", c.version, c.err, raw)
|
||||
}
|
||||
}
|
||||
for out, want := range map[string]string{"bubblewrap 0.13.0\n": "0.13.0", "bubblewrap 0.10": "0.10"} {
|
||||
if got, err := parseBubblewrapVersion(out); err != nil || got != want {
|
||||
t.Errorf("%q read as %q, %v", out, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stepTool is the Run of the named tool.
|
||||
func stepTool(t *testing.T, m *Module, name string) func(map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == name {
|
||||
return tool.Run
|
||||
}
|
||||
}
|
||||
t.Fatalf("no tool %s", name)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1309-S6 (ADR 0448's check): a test of the change, run by the unit level, reads no key and no login of the account,
|
||||
// resolves no name, reaches no socket of the machine (a TCP port on its loopback, a unix socket) and leaves the shared
|
||||
// caches unchanged. The probe is testdata/repos/probe.
|
||||
func TestARunStepReachesNoKeyLoginNameOrSocketAndLeavesTheCachesUnchanged(t *testing.T) {
|
||||
needTools(t, "the shut-off unit level", "go", "git", "bwrap")
|
||||
tree := repository(t, "probe")
|
||||
work, err := os.MkdirTemp("", "w")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(work) })
|
||||
if err := os.MkdirAll(filepath.Join(work, "ws1"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(tree, filepath.Join(work, "ws1", "tree")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
home := t.TempDir()
|
||||
key, login := filepath.Join(home, ".ssh", "mesh_ed25519"), filepath.Join(home, "broker")
|
||||
for _, f := range []string{key, login} {
|
||||
if err := os.MkdirAll(filepath.Dir(f), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(f, []byte("secret"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
tcp, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tcp.Close()
|
||||
sockDir := t.TempDir()
|
||||
unix, err := net.Listen("unix", filepath.Join(sockDir, "s"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer unix.Close()
|
||||
m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()}
|
||||
m.Run = StepRunner(home)
|
||||
for _, c := range m.caches() {
|
||||
if err := os.MkdirAll(c, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
res, err := m.runLevel(context.Background(), filepath.Join(work, "ws1", "tree"), levelOpts{workspace: "ws1", level: LevelUnit,
|
||||
bounds: Bounds{Run: UnitBound * 100}, env: []string{"PROBE_KEY=" + key, "PROBE_LOGIN=" + login,
|
||||
"PROBE_TCP=" + tcp.Addr().String(), "PROBE_SOCKET=" + filepath.Join(sockDir, "s"), "PROBE_CACHE=" + m.caches()[1]}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !res.Passed {
|
||||
t.Fatalf("the probe did not run: %+v", res)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(work, "ws1", "tree", "seen.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("the probe wrote nothing: %v", err)
|
||||
}
|
||||
var seen map[string]bool
|
||||
if err := json.Unmarshal(raw, &seen); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for what, reached := range seen {
|
||||
if reached && what != "wroteCache" {
|
||||
t.Errorf("the run step reached %s: %s", what, raw)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.caches()[1], "written-by-a-candidate")); err == nil {
|
||||
t.Errorf("a candidate's step changed the shared cache")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
// The integration level shut off (novox/hq ADR 0448, ADR 0470; issue 1309 S5): its store, its bus and its tests run in
|
||||
// one bubblewrap of their own, so the tests reach the level's postgres (a socket in the run's directory) and its bus (a
|
||||
// loopback port of the step's own network) and nothing of the machine. The process in it is this binary started again
|
||||
// as `mesh-build-go level <spec>`: it runs the level as integrationHere does and prints its answer as one line of JSON.
|
||||
// The tools bundle never runs a level in its own process, and a level never serves tools.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// levelArg is the first argument that makes this binary an integration level's process.
|
||||
const levelArg = "level"
|
||||
|
||||
// levelSpec is what the shut-off process is asked: the workspace, the packages and the -run pattern, the level's bounds.
|
||||
type levelSpec struct {
|
||||
Work string `json:"work"`
|
||||
Cache string `json:"cache"`
|
||||
Workspace string `json:"workspace"`
|
||||
Packages []string `json:"packages,omitempty"`
|
||||
Run string `json:"run,omitempty"`
|
||||
Setup time.Duration `json:"setup"`
|
||||
Bound time.Duration `json:"bound"`
|
||||
}
|
||||
|
||||
// levelAnswer is the one line the process prints: the level's result, or why it could not run.
|
||||
type levelAnswer struct {
|
||||
Result *TestResult `json:"result,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// executable is this program's path, bound read-only into the step: Executable when a test set it.
|
||||
func (m *Module) executable() (string, error) {
|
||||
if m.Executable != "" {
|
||||
return m.Executable, nil
|
||||
}
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.EvalSymlinks(exe)
|
||||
}
|
||||
|
||||
// integrationShutOff runs the integration level in one shut-off process and reads its answer.
|
||||
func (m *Module) integrationShutOff(ctx context.Context, workspace string, packages []string, pattern string) (*TestResult, error) {
|
||||
exe, err := m.executable()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the integration level's program: %w", err)
|
||||
}
|
||||
b := m.integrationBounds()
|
||||
spec, err := json.Marshal(levelSpec{Work: m.Work, Cache: m.Cache, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range m.caches() {
|
||||
if err := os.MkdirAll(c, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("the shared cache %s: %w", c, err)
|
||||
}
|
||||
}
|
||||
iso := m.isolation(workspace, false)
|
||||
iso.ReadOnly = append(iso.ReadOnly, exe) // the program alone, not the directory it lies in
|
||||
r := buildstep.Isolated(m.Run, iso)(ctx, nil, exe, levelArg, string(spec))
|
||||
var answer levelAnswer
|
||||
line := lastLine(r.Stdout)
|
||||
if err := json.Unmarshal([]byte(line), &answer); err != nil || (answer.Result == nil && answer.Error == "") {
|
||||
return nil, fmt.Errorf("the integration level's shut-off process answered nothing it says (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return nil, fmt.Errorf("%s", answer.Error)
|
||||
}
|
||||
return answer.Result, nil
|
||||
}
|
||||
|
||||
func lastLine(s string) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
return strings.TrimSpace(lines[len(lines)-1])
|
||||
}
|
||||
|
||||
// runLevelProcess is the shut-off process: args is the spec. It prints its answer and returns the exit status, 0 when it
|
||||
// printed one.
|
||||
func runLevelProcess(args []string) int {
|
||||
say := func(a levelAnswer) int {
|
||||
raw, _ := json.Marshal(a)
|
||||
fmt.Println(string(raw))
|
||||
return 0
|
||||
}
|
||||
if len(args) != 1 {
|
||||
fmt.Fprintln(os.Stderr, "usage: mesh-build-go level <spec>")
|
||||
return 2
|
||||
}
|
||||
var spec levelSpec
|
||||
if err := json.Unmarshal([]byte(args[0]), &spec); err != nil {
|
||||
return say(levelAnswer{Error: "the level's spec: " + err.Error()})
|
||||
}
|
||||
if !workspaceOK.MatchString(spec.Workspace) {
|
||||
return say(levelAnswer{Error: fmt.Sprintf("%q is not a workspace id", spec.Workspace)})
|
||||
}
|
||||
m := &Module{Inside: true, Recipe: Current(), Work: spec.Work, Cache: spec.Cache, Home: os.Getenv("HOME"),
|
||||
IntegrationBounds: Bounds{Setup: spec.Setup, Run: spec.Bound}, Say: func(string, ...any) {}}
|
||||
m.Run = StepRunner(m.Home)
|
||||
ctx := context.Background()
|
||||
major, err := m.checkPostgres(ctx)
|
||||
if err != nil {
|
||||
return say(levelAnswer{Error: err.Error()})
|
||||
}
|
||||
tree := filepath.Join(spec.Work, spec.Workspace, "tree")
|
||||
res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major)
|
||||
if err != nil {
|
||||
return say(levelAnswer{Error: err.Error()})
|
||||
}
|
||||
return say(levelAnswer{Result: res})
|
||||
}
|
||||
@@ -29,6 +29,9 @@ func main() {
|
||||
if len(os.Args) > 1 && os.Args[1] == busArg {
|
||||
os.Exit(runBus(os.Args[2:]))
|
||||
}
|
||||
if len(os.Args) > 1 && os.Args[1] == levelArg {
|
||||
os.Exit(runLevelProcess(os.Args[2:]))
|
||||
}
|
||||
ctx := context.Background()
|
||||
m := &Module{
|
||||
Node: strings.TrimSpace(os.Getenv("MESH_NODE")),
|
||||
@@ -47,12 +50,19 @@ func main() {
|
||||
m.Home = u.HomeDir
|
||||
}
|
||||
m.Run = StepRunner(m.Home)
|
||||
m.RegistryFile = m.Home + "/registry"
|
||||
m.Probe = ProbeGo(m.Home, m.Cache)
|
||||
m.Bubblewrap = ProbeBubblewrap
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
if m.Node != "" {
|
||||
go m.Start(ctx)
|
||||
}
|
||||
go startKeeper(ctx)
|
||||
// Every state it keeps, said on the bus before it serves (ADR 0495 §2; issue 1790).
|
||||
if err := keepStates(); err != nil {
|
||||
logf("announcing the states it keeps: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -64,7 +64,7 @@ func TestManifestIsValidAndDeclaresTheAccountTheCapabilityAndThePackages(t *test
|
||||
}
|
||||
}
|
||||
slices.Sort(packages)
|
||||
if want := []string{"git", "openssh", "postgresql"}; !slices.Equal(packages, want) {
|
||||
if want := []string{"bubblewrap", "git", "openssh", "postgresql"}; !slices.Equal(packages, want) {
|
||||
t.Errorf("packages = %v, want %v", packages, want)
|
||||
}
|
||||
}
|
||||
@@ -144,7 +144,34 @@ func TestGiteaReadsTheBuildModulesKeysAndTheCatalogueAcceptsThemTogether(t *test
|
||||
t.Errorf("gitea reads %v, lacks %s", gitea.Reads, want)
|
||||
}
|
||||
}
|
||||
if problems := catalogue.StateReadsNothingDeclares(all); len(problems) > 0 {
|
||||
t.Errorf("a read names a state its owner does not keep: %v", problems)
|
||||
}
|
||||
|
||||
// 1309-S1: every run step runs in bubblewrap, so the module declares its package (the lead's decision: each build module
|
||||
// that runs a step through it declares it), beside the user-namespaces capability bubblewrap needs.
|
||||
func TestTheModuleDeclaresBubblewrapAndTheCapabilityItNeeds(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
if r := resource(t, m, "bubblewrap"); r["type"] != "package" || r["package"] != "bubblewrap" {
|
||||
t.Errorf("the bubblewrap resource is %v, want the package bubblewrap", r)
|
||||
}
|
||||
if !slices.Contains(m.Capabilities, "user-namespaces") {
|
||||
t.Errorf("capabilities %v lack user-namespaces", m.Capabilities)
|
||||
}
|
||||
}
|
||||
|
||||
// 1587-S1 (ADR 0498 §2): the module requires the artifact store and is told its address in a file of its home, which no
|
||||
// run step is given (a run step has no network either, ADR 0448).
|
||||
func TestTheModuleRequiresTheArtifactStoreAndIsToldItsAddress(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
if !slices.Contains(m.Requires, "artifact-store") {
|
||||
t.Errorf("requires %v lack artifact-store", m.Requires)
|
||||
}
|
||||
r := resource(t, m, "registry")
|
||||
// The validator reads the template (files/registry) into the resource's content.
|
||||
if content, _ := r["content"].(string); r["type"] != "file" || r["path"] != "${dir:state}/registry" || !strings.Contains(content, "${seat:mesh-artifact-store:reach}") {
|
||||
t.Errorf("the registry resource is %v", r)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(moduleDir, "files", "registry"))
|
||||
if err != nil || strings.TrimSpace(string(raw)) != "${seat:mesh-artifact-store:reach}" {
|
||||
t.Errorf("files/registry is %q (%v), want the store seat's reach", raw, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,10 @@ type Module struct {
|
||||
// both outside the account's home (ADR 0466 §1); Home is the account's home.
|
||||
Run forgekey.Runner
|
||||
Home, Work, Cache string
|
||||
// RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of
|
||||
// the SDK's fetch and merge. A test points them at a local repository (buildstep.AllowFileRemote).
|
||||
RemoteBase string
|
||||
FetchOptions []buildstep.Option
|
||||
// HTTP fetches the files an archive artifact is pinned to (url and sha256); a client of its own with a fifteen-minute
|
||||
// timeout when nil. A test serves its own.
|
||||
HTTP *http.Client
|
||||
@@ -46,6 +50,18 @@ type Module struct {
|
||||
IntegrationBounds Bounds
|
||||
// Now is the clock the levels' times are read from; time.Now when nil.
|
||||
Now func() time.Time
|
||||
// Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is
|
||||
// unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing.
|
||||
Bubblewrap func(ctx context.Context) (string, error)
|
||||
// Inside is the integration level's own process, started shut off in bubblewrap by a test step (level.go): its
|
||||
// commands run as they are, never shut off a second time.
|
||||
Inside bool
|
||||
// RegistryFile holds the artifact store's address (host:port), placed by the manifest; Push puts a blob there
|
||||
// (buildstep.PushBlob when nil; a test's fake store).
|
||||
RegistryFile string
|
||||
Push func(ctx context.Context, registry, repository, digest string, body []byte) error
|
||||
// Executable is this program's path, which the integration level starts again shut off; os.Executable when empty.
|
||||
Executable string
|
||||
// StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own
|
||||
// program, StartBus in bus.go, when nil).
|
||||
StartBus func(ctx context.Context, dir string) (url string, stop func(), err error)
|
||||
@@ -65,7 +81,10 @@ func (m *Module) guard(installed string, probeErr error) buildstep.Guard {
|
||||
// CheckRound.
|
||||
func (m *Module) Check(ctx context.Context) error {
|
||||
installed, err := m.Probe(ctx)
|
||||
return m.guard(installed, err).Check(ctx)
|
||||
if err := m.guard(installed, err).Check(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.checkBubblewrap(ctx)
|
||||
}
|
||||
|
||||
// Start checks once, as the module starts, and then every CheckRound until ctx ends.
|
||||
@@ -91,6 +110,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
|
||||
if err := m.guard(installed, perr).Check(ctx); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
if err := m.checkBubblewrap(ctx); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("%s installed, the version the module builds with (recipe %s)", installed, m.Recipe.Version()))
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0499 §4 (issue 1758): a fetch asked with the module's directory and its named paths leaves the workspace holding
|
||||
// only those, the manifest and the lock files; the rest of the commit is gone before any run step; a request naming
|
||||
// paths without a directory is refused before anything is fetched.
|
||||
func TestAFetchWithNamedPathsNarrowsTheWorkspace(t *testing.T) {
|
||||
f := newForge(t)
|
||||
_, head := f.repo("novox/narrow", map[string]string{
|
||||
"README.md": "the repository\n",
|
||||
"lib/go.mod": "module fixture/lib\n\ngo 1.25.0\n",
|
||||
"lib/module.json": `{"module":"lib","version":"1"}`,
|
||||
"lib/cmd/x/x.go": "package main\n\nfunc main() {}\n",
|
||||
"lib/docs/a.md": "not named\n",
|
||||
"other/go.mod": "module fixture/other\n\ngo 1.25.0\n",
|
||||
"other/o.go": "package other\n",
|
||||
})
|
||||
m, states := f.module(t)
|
||||
res, st := fetched(t, m, states, map[string]any{"repository": "novox/narrow", "commit": head, "dir": "lib", "paths": []any{"cmd"}})
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the fetch ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
tree := filepath.Join(m.Work, res.Workspace, "tree")
|
||||
for _, kept := range []string{"lib/go.mod", "lib/module.json", "lib/cmd/x/x.go"} {
|
||||
if _, err := os.Stat(filepath.Join(tree, kept)); err != nil {
|
||||
t.Errorf("%s was narrowed away: %v", kept, err)
|
||||
}
|
||||
}
|
||||
for _, gone := range []string{"README.md", "lib/docs/a.md", "other/o.go"} {
|
||||
if _, err := os.Stat(filepath.Join(tree, gone)); err == nil {
|
||||
t.Errorf("%s is still in the workspace", gone)
|
||||
}
|
||||
}
|
||||
if strings.Join(res.Modules, ",") != "lib" {
|
||||
t.Errorf("the fetch answered the modules %v, want lib alone", res.Modules)
|
||||
}
|
||||
_, st = fetched(t, m, states, map[string]any{"repository": "novox/narrow", "commit": head, "paths": []any{"cmd"}})
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "no module directory") {
|
||||
t.Errorf("paths without a directory: %s %q", st.State, st.Reason)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package main
|
||||
|
||||
// Pins on a pull request's head (novox/hq ADR 0482 §2, ADR 0500 §2; issue 1148): a group's Go members pin the head of
|
||||
// the group's mesh-sdk pull request, a commit that only refs/pull/<n>/head of the forge carries until the merge. Go
|
||||
// cannot fetch it by its pseudo-version, so a fetch that meets "unknown revision" for such a pin finds the commit among
|
||||
// the forge's pull request heads, places that repository beside the workspace as a plain directory, and hands it to
|
||||
// Go through a temporary -modfile, a copy of the directory's go.mod that replaces the module by that directory. The
|
||||
// fetch's go commands and the build use the same modfile, and the build still runs with GOPROXY=off. A pin Go can
|
||||
// fetch is not touched, and a directory that vendors is never resolved.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const forgePrefix = "git.novox.be/novox/"
|
||||
|
||||
var pseudoVersion = regexp.MustCompile(`[-.][0-9]{14}-([0-9a-f]{12})$`)
|
||||
|
||||
// pin is a requirement of a go.mod on a commit of a repository of the forge, by a pseudo-version.
|
||||
type pin struct {
|
||||
module string // git.novox.be/novox/<repository>[/<directory of the module in it>]
|
||||
repo string // <repository>
|
||||
sub string // the module's directory in the repository, "" for its root
|
||||
short string // the 12 digits of the commit the pseudo-version ends in
|
||||
}
|
||||
|
||||
// forgePins are the pseudo-version requirements of a go.mod on the forge's own modules.
|
||||
func forgePins(gomod string) []pin {
|
||||
var pins []pin
|
||||
for _, line := range strings.Split(gomod, "\n") {
|
||||
if i := strings.Index(line, "//"); i >= 0 {
|
||||
line = line[:i]
|
||||
}
|
||||
f := strings.Fields(line)
|
||||
if len(f) > 0 && f[0] == "require" {
|
||||
f = f[1:]
|
||||
}
|
||||
if len(f) != 2 || !strings.HasPrefix(f[0], forgePrefix) {
|
||||
continue
|
||||
}
|
||||
v := pseudoVersion.FindStringSubmatch(f[1])
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(strings.TrimPrefix(f[0], forgePrefix), "/", 2)
|
||||
p := pin{module: f[0], repo: parts[0], short: v[1]}
|
||||
if len(parts) == 2 {
|
||||
p.sub = parts[1]
|
||||
}
|
||||
pins = append(pins, p)
|
||||
}
|
||||
return pins
|
||||
}
|
||||
|
||||
// pinsDir is where the temporary modfile of a module directory of a workspace lies: the workspace's pins directory,
|
||||
// by the directory's path in the tree. dir is outside any workspace of the work directory, and the answer empty, when
|
||||
// it does not lie in <work>/<workspace>/tree.
|
||||
func (m *Module) pinsDir(dir string) string {
|
||||
rel, err := filepath.Rel(m.Work, dir)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(filepath.ToSlash(rel), "/")
|
||||
if len(parts) < 2 || parts[0] == ".." || parts[1] != "tree" {
|
||||
return ""
|
||||
}
|
||||
name := strings.Join(parts[2:], "_")
|
||||
if name == "" {
|
||||
name = "root"
|
||||
}
|
||||
return filepath.Join(m.Work, parts[0], "pins", name)
|
||||
}
|
||||
|
||||
// pinModfile is the modfile handed to Go for dir, and whether there is one: only a directory a fetch resolved a pin
|
||||
// for has it.
|
||||
func (m *Module) pinModfile(dir string) (string, bool) {
|
||||
p := m.pinsDir(dir)
|
||||
if p == "" {
|
||||
return "", false
|
||||
}
|
||||
p = filepath.Join(p, "go.mod")
|
||||
_, err := os.Stat(p)
|
||||
return p, err == nil
|
||||
}
|
||||
|
||||
// resolvePins resolves the pins of dir that a go command failed on: those whose commit the failure names. It answers
|
||||
// false when there is none, and the failure is then the caller's.
|
||||
func (m *Module) resolvePins(ctx context.Context, root, dir, failure string) (bool, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "go.mod"))
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
modfile, _ := m.pinModfile(dir)
|
||||
done := ""
|
||||
if modfile != "" {
|
||||
if b, err := os.ReadFile(modfile); err == nil {
|
||||
done = string(b)
|
||||
}
|
||||
}
|
||||
progress := false
|
||||
for _, p := range forgePins(string(raw)) {
|
||||
if !strings.Contains(failure, p.short) || strings.Contains(done, "replace "+p.module+" =>") {
|
||||
continue
|
||||
}
|
||||
where, err := m.placePin(ctx, root, p, failure)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := m.replaceInModfile(dir, p.module, where); err != nil {
|
||||
return false, err
|
||||
}
|
||||
progress = true
|
||||
}
|
||||
return progress, nil
|
||||
}
|
||||
|
||||
// placePin finds the commit of a pin among the pull request heads of the forge, places the repository at it beside
|
||||
// the workspace (the way a fetch places a sibling) and answers the directory of the module in it.
|
||||
func (m *Module) placePin(ctx context.Context, root string, p pin, failure string) (string, error) {
|
||||
repository := "novox/" + p.repo
|
||||
r := m.Run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-c", "credential.helper=", "ls-remote", m.remote(repository), "refs/pull/*/head")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("the pin of %s at commit %s: listing the pull requests of %s: %s", p.module, p.short, repository, firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
commit, ref := "", ""
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if sha, name, ok := strings.Cut(strings.TrimSpace(l), "\t"); ok && commitOK.MatchString(sha) && strings.HasPrefix(sha, p.short) && strings.HasSuffix(name, "/head") {
|
||||
commit, ref = sha, name
|
||||
break
|
||||
}
|
||||
}
|
||||
if commit == "" {
|
||||
return "", fmt.Errorf("the pin of %s at commit %s is on no pull request head of %s on the forge; go said: %s", p.module, p.short, repository, causeLine(failure))
|
||||
}
|
||||
// One placement per repository and commit, shared by every go.mod directory of the tree that pins it.
|
||||
at := filepath.Join(root, "pinned", p.repo+"@"+commit)
|
||||
where := filepath.Join(at, "beside", p.repo, filepath.FromSlash(p.sub))
|
||||
if _, err := os.Stat(filepath.Join(at, "beside", p.repo)); err == nil {
|
||||
return where, nil
|
||||
}
|
||||
if _, err := m.placeSiblings(ctx, at, []siblingArg{{repository: repository, commit: commit, ref: ref}}); err != nil {
|
||||
return "", fmt.Errorf("the pin of %s at commit %s: %w", p.module, p.short, err)
|
||||
}
|
||||
return where, nil
|
||||
}
|
||||
|
||||
// replaceInModfile adds `replace module => where` to the temporary modfile of dir, made from the directory's go.mod and
|
||||
// go.sum the first time; Go reads the sums beside the modfile.
|
||||
func (m *Module) replaceInModfile(dir, module, where string) error {
|
||||
pins := m.pinsDir(dir)
|
||||
if pins == "" {
|
||||
return fmt.Errorf("the directory %s lies outside a workspace", dir)
|
||||
}
|
||||
if err := os.MkdirAll(pins, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
modfile := filepath.Join(pins, "go.mod")
|
||||
body, err := os.ReadFile(modfile)
|
||||
if err != nil {
|
||||
if body, err = os.ReadFile(filepath.Join(dir, "go.mod")); err != nil {
|
||||
return err
|
||||
}
|
||||
sum, _ := os.ReadFile(filepath.Join(dir, "go.sum"))
|
||||
if err := os.WriteFile(filepath.Join(pins, "go.sum"), sum, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
body = append(body, []byte("\nreplace "+module+" => "+where+"\n")...)
|
||||
return os.WriteFile(modfile, body, 0o600)
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
// A pin on a pull request's head, on real git repositories and the real go (issue 1148); pins_test.go proves the
|
||||
// commands with a fake runner.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1148/AC2: given a forge with a mesh-sdk-like pull request whose branch is deleted (only refs/pull/7/head carries its
|
||||
// commit) and a Go member pinning that head by a pseudo-version, when build_go_fetch then build_go_build run on the
|
||||
// member, then both pass, the build with GOPROXY=off.
|
||||
func TestRealFetchAndBuildOfAMemberPinningAPullHead(t *testing.T) {
|
||||
// Not skipped when a tool is missing: a pass of the check must mean this test ran (the check prints no test log).
|
||||
for _, tool := range []string{"go", "git"} {
|
||||
if _, err := exec.LookPath(tool); err != nil {
|
||||
t.Fatalf("no %s where this check runs: the pin on a pull request's head cannot be proven", tool)
|
||||
}
|
||||
}
|
||||
f := newForge(t)
|
||||
work, fork := f.repo("novox/dep", map[string]string{"README": "dep\n"})
|
||||
head := f.commit("novox/dep", work, fork, map[string]string{
|
||||
"go/go.mod": "module git.novox.be/novox/dep/go\n\ngo 1.25.0\n",
|
||||
"go/dep.go": "package dep\n\n// Name is what the member prints.\nconst Name = \"from the pull request\"\n",
|
||||
}, "refs/pull/7/head")
|
||||
git(t, work, "checkout", "-q", "main") // no branch of the repository carries the head
|
||||
member := map[string]string{
|
||||
"go.mod": "module fixture/member\n\ngo 1.25.0\n\nrequire git.novox.be/novox/dep/go v0.0.0-20261022120000-" + head[:12] + "\n",
|
||||
"cmd/prog/main.go": "package main\n\nimport (\n\t\"fmt\"\n\n\tdep \"git.novox.be/novox/dep/go\"\n)\n\n" +
|
||||
"func main() { fmt.Println(dep.Name) }\n",
|
||||
"module.json": `{"module":"fixture-member","version":"1","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/prog","binary":"prog-bin","loads":["prog-bin"]}]}}`,
|
||||
}
|
||||
_, memberHead := f.repo("novox/member", member)
|
||||
m, states := f.module(t)
|
||||
withStore(t, m) // the build pushes its archive (ADR 0498)
|
||||
|
||||
res, st := fetched(t, m, states, map[string]any{"repository": "novox/member", "commit": memberHead})
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the fetch ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.Work, res.Workspace, "pinned", "dep@"+head, "beside", "dep", "go", "dep.go")); err != nil {
|
||||
t.Errorf("the pull request's repository was not placed beside the workspace: %v", err)
|
||||
}
|
||||
// The build runs shut off in bubblewrap (1309), which a build node has and build-agent's check container has not:
|
||||
// there the build half is not run, said loudly; the fetch half above, the pin's own part (issue 1148), always is.
|
||||
needTools(t, "the shut-off build of a member pinning a pull head", "bwrap")
|
||||
built, err := m.build(context.Background(), map[string]any{"workspace": res.Workspace, "module": ".", "artifact": "tools"})
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("the build: %w", err))
|
||||
}
|
||||
if b := built.(BuildResult); b.Digest == "" || b.Binary != "prog-bin" {
|
||||
t.Errorf("the build answered %+v", b)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// pinShort is the 12 digits of commitSHA a pseudo-version ends in.
|
||||
const pinShort = "0123456789ab"
|
||||
|
||||
const pinnedGoMod = "module fixture/member\n\ngo 1.26\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go v0.1.15-0.20261022120000-" + pinShort + "\n)\n"
|
||||
|
||||
// pinHarness runs a fetch of a tree whose only go.mod is gomod (a vendoring one when vendor is set) against a runner
|
||||
// that stands for the forge and for go: a pull request's head is listed by ls-remote when pulls says so, and go mod
|
||||
// download of a pseudo-version that no ref carries fails with "unknown revision" unless the command was handed a
|
||||
// -modfile.
|
||||
type pinHarness struct {
|
||||
t *testing.T
|
||||
m *Module
|
||||
f *fakeStep
|
||||
pulls string // what git ls-remote answers
|
||||
onTrunk bool // the pin is fetchable as it is: go mod download never says unknown revision
|
||||
cmds []cmdRecord
|
||||
}
|
||||
|
||||
func newPinHarness(t *testing.T, gomod string, vendor bool, pulls string) (*pinHarness, *keptStates) {
|
||||
f := &fakeStep{gomods: "go.mod\n"}
|
||||
m, states := fetchModule(t, f)
|
||||
h := &pinHarness{t: t, m: m, f: f, pulls: pulls}
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
h.cmds = append(h.cmds, cmdRecord{env: env, line: line})
|
||||
switch {
|
||||
case name == "git" && strings.Contains(line, " checkout ") && !strings.Contains(line, "pinned"):
|
||||
for i, a := range args {
|
||||
if a == "-C" {
|
||||
os.MkdirAll(args[i+1], 0o755)
|
||||
for _, g := range strings.Fields(f.gomods) {
|
||||
d := filepath.Join(args[i+1], filepath.Dir(g))
|
||||
os.MkdirAll(d, 0o755)
|
||||
os.WriteFile(filepath.Join(d, "go.mod"), []byte(gomod), 0o644)
|
||||
os.WriteFile(filepath.Join(d, "go.sum"), []byte("sum\n"), 0o644)
|
||||
}
|
||||
if vendor {
|
||||
os.MkdirAll(filepath.Join(args[i+1], "vendor"), 0o755)
|
||||
os.WriteFile(filepath.Join(args[i+1], "vendor", "modules.txt"), nil, 0o644)
|
||||
}
|
||||
}
|
||||
}
|
||||
case name == "git" && strings.Contains(line, "ls-remote"):
|
||||
return forgekey.Ran{Stdout: h.pulls}
|
||||
case name == "go" && strings.Contains(line, " mod download"):
|
||||
if !h.onTrunk && !strings.Contains(strings.Join(env, " "), "-modfile=") {
|
||||
return forgekey.Ran{Status: 1, Stderr: "go: git.novox.be/novox/mesh-sdk/go@v0.1.15-0.20261022120000-" + pinShort + ": invalid version: unknown revision " + pinShort + "\n"}
|
||||
}
|
||||
}
|
||||
return f.run(ctx, env, name, args...)
|
||||
}
|
||||
return h, states
|
||||
}
|
||||
|
||||
func (h *pinHarness) fetch(states *keptStates) (FetchResult, delivery.Step) {
|
||||
h.t.Helper()
|
||||
if _, err := fetchTool(h.t, h.m)(fetchArgs); err != nil {
|
||||
h.t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(h.t)
|
||||
var res FetchResult
|
||||
if st.State == delivery.StepPassed {
|
||||
json.Unmarshal(st.Result, &res)
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
|
||||
func (h *pinHarness) ran(sub string) bool {
|
||||
for _, c := range h.cmds {
|
||||
if strings.Contains(c.line, sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const pullList = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\trefs/pull/6/head\n" + commitSHA + "\trefs/pull/7/head\n"
|
||||
|
||||
// 1148/AC1: a go.mod requiring mesh-sdk at a pseudo-version only a pull request's head carries is fetched: the
|
||||
// repository is placed beside the workspace at that commit, go mod download and verify succeed through a temporary
|
||||
// -modfile replacing the module by that directory, and the build is handed the same -modfile with GOPROXY=off.
|
||||
func TestAPinOnAPullHeadIsPlacedBesideAndHandedToGoByAModfile(t *testing.T) {
|
||||
h, states := newPinHarness(t, pinnedGoMod, false, pullList)
|
||||
res, st := h.fetch(states)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
placed := filepath.Join(h.m.Work, res.Workspace, "pinned", "mesh-sdk@"+commitSHA, "beside", "mesh-sdk")
|
||||
if !h.ran("-C " + placed + " checkout -q --detach " + commitSHA) {
|
||||
t.Errorf("mesh-sdk was not placed at %s at the commit:\n%s", placed, strings.Join(h.f.lines(), "\n"))
|
||||
}
|
||||
if !h.ran("refs/pull/*/head") {
|
||||
t.Errorf("the forge's pull request heads were not listed")
|
||||
}
|
||||
dir := filepath.Join(h.m.Work, res.Workspace, "tree")
|
||||
flags := ""
|
||||
for _, e := range h.m.goEnvFor(dir) {
|
||||
if strings.HasPrefix(e, "GOFLAGS=") {
|
||||
flags = e
|
||||
}
|
||||
}
|
||||
_, modfile, ok := strings.Cut(flags, "-modfile=")
|
||||
if !ok {
|
||||
t.Fatalf("go is not handed a -modfile: %v", h.m.goEnvFor(dir))
|
||||
}
|
||||
raw, err := os.ReadFile(strings.Fields(modfile)[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := "replace git.novox.be/novox/mesh-sdk/go => " + filepath.Join(placed, "go"); !strings.Contains(string(raw), want) || !strings.Contains(string(raw), "module fixture/member") {
|
||||
t.Errorf("the modfile lacks %q:\n%s", want, raw)
|
||||
}
|
||||
build := strings.Join(h.m.buildEnv(dir), " ")
|
||||
if !strings.Contains(build, "-modfile="+strings.Fields(modfile)[0]) || !strings.Contains(build, "GOPROXY=off") {
|
||||
t.Errorf("the build is not handed the modfile with GOPROXY=off: %s", build)
|
||||
}
|
||||
if !h.ran(" mod verify") {
|
||||
t.Errorf("go mod verify did not run")
|
||||
}
|
||||
}
|
||||
|
||||
// 1148/AC3: a pin go downloads as it is (on the trunk or a tag) behaves as today: no listing of pull requests, no
|
||||
// repository placed, no -modfile in the fetch or the build.
|
||||
func TestAPinGoCanDownloadIsLeftAsToday(t *testing.T) {
|
||||
h, states := newPinHarness(t, pinnedGoMod, false, pullList)
|
||||
h.onTrunk = true
|
||||
res, st := h.fetch(states)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if h.ran("ls-remote") || h.ran("pinned") {
|
||||
t.Errorf("a pin on the trunk made the fetch look for a pull request or place a repository:\n%s", strings.Join(h.f.lines(), "\n"))
|
||||
}
|
||||
dir := filepath.Join(h.m.Work, res.Workspace, "tree")
|
||||
for _, e := range append(h.m.goEnvFor(dir), h.m.buildEnv(dir)...) {
|
||||
if strings.Contains(e, "-modfile") {
|
||||
t.Errorf("a -modfile is handed to go: %s", e)
|
||||
}
|
||||
}
|
||||
if !h.ran(" mod download") || !h.ran(" mod verify") {
|
||||
t.Errorf("go mod download and verify did not run as before")
|
||||
}
|
||||
}
|
||||
|
||||
// 1148/AC4: a pin that no ref of the forge carries ends the step failed, naming the module and the commit.
|
||||
func TestAPinOnNoRefFailsNamingTheModuleAndTheCommit(t *testing.T) {
|
||||
h, states := newPinHarness(t, pinnedGoMod, false, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\trefs/pull/6/head\n")
|
||||
_, st := h.fetch(states)
|
||||
if st.State != delivery.StepFailed {
|
||||
t.Fatalf("the step ended %s, want failed", st.State)
|
||||
}
|
||||
if !strings.Contains(st.Reason, "no pull request head") || !strings.Contains(st.Reason, "unknown revision") ||
|
||||
!strings.Contains(st.Reason, "git.novox.be/novox/mesh-sdk/go") || !strings.Contains(st.Reason, pinShort) {
|
||||
t.Errorf("the reason does not name the module and the commit: %q", st.Reason)
|
||||
}
|
||||
if !h.ran("ls-remote") {
|
||||
t.Errorf("the forge's pull request heads were never looked at")
|
||||
}
|
||||
}
|
||||
|
||||
// 1148/AC5: a directory that vendors resolves nothing: no go command, no listing, no repository placed.
|
||||
func TestAVendoringDirectoryResolvesNothing(t *testing.T) {
|
||||
h, states := newPinHarness(t, pinnedGoMod, true, pullList)
|
||||
_, st := h.fetch(states)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if h.ran("go -C") || h.ran("ls-remote") || h.ran("pinned") {
|
||||
t.Errorf("a vendoring directory was resolved:\n%s", strings.Join(h.f.lines(), "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// 1148/AC1 (a repository with several go.mod directories, as mesh-catalog has): two directories pinning the same pull
|
||||
// head share one placement and both are handed a modfile; the fetch does not fail placing it twice.
|
||||
func TestTwoDirectoriesPinningOneHeadShareOnePlacement(t *testing.T) {
|
||||
h, states := newPinHarness(t, pinnedGoMod, false, pullList)
|
||||
h.f.gomods = "go.mod\nsub/go.mod\n"
|
||||
res, st := h.fetch(states)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
checkouts := 0
|
||||
for _, c := range h.cmds {
|
||||
if strings.Contains(c.line, "pinned") && strings.Contains(c.line, " checkout ") {
|
||||
checkouts++
|
||||
}
|
||||
}
|
||||
if checkouts != 1 {
|
||||
t.Errorf("the repository was placed %d times, want once:\n%s", checkouts, strings.Join(h.f.lines(), "\n"))
|
||||
}
|
||||
for _, d := range []string{"tree", "tree/sub"} {
|
||||
if _, ok := h.m.pinModfile(filepath.Join(h.m.Work, res.Workspace, d)); !ok {
|
||||
t.Errorf("%s was not handed a modfile", d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1148/AC1: two directories pinning two different commits of one repository get two placements, each its own.
|
||||
func TestTwoCommitsOfOneRepositoryGetTwoPlacements(t *testing.T) {
|
||||
other := "fedcba9876543210fedcba9876543210fedcba98"
|
||||
h, states := newPinHarness(t, pinnedGoMod, false, commitSHA+"\trefs/pull/7/head\n"+other+"\trefs/pull/8/head\n")
|
||||
h.f.gomods = "go.mod\nsub/go.mod\n"
|
||||
base := h.m.Run
|
||||
h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if name == "git" && strings.Contains(strings.Join(args, " "), "pinned/mesh-sdk@"+other+"/") && strings.Contains(strings.Join(args, " "), " rev-parse HEAD") {
|
||||
return forgekey.Ran{Stdout: other + "\n"}
|
||||
}
|
||||
// the second directory pins the other commit
|
||||
if name == "git" && strings.Contains(strings.Join(args, " "), " checkout ") && !strings.Contains(strings.Join(args, " "), "pinned") {
|
||||
r := base(ctx, env, name, args...)
|
||||
for i, a := range args {
|
||||
if a == "-C" {
|
||||
os.WriteFile(filepath.Join(args[i+1], "sub", "go.mod"), []byte(strings.ReplaceAll(pinnedGoMod, pinShort, other[:12])), 0o644)
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
if name == "go" && strings.Contains(strings.Join(args, " "), "/sub mod download") && !strings.Contains(strings.Join(env, " "), "-modfile=") {
|
||||
return forgekey.Ran{Status: 1, Stderr: "go: git.novox.be/novox/mesh-sdk/go@v0.1.15-0.20261022120000-" + other[:12] + ": invalid version: unknown revision " + other[:12] + "\n"}
|
||||
}
|
||||
return base(ctx, env, name, args...)
|
||||
}
|
||||
_, st := h.fetch(states)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
for _, c := range []string{commitSHA, other} {
|
||||
if !h.ran("pinned/mesh-sdk@" + c) {
|
||||
t.Errorf("no placement of mesh-sdk at %s", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package main
|
||||
|
||||
// The merge preview and the siblings of a fetch (novox/hq ADR 0484 clauses 3 and 4; issues 1675 and 1677). The merge
|
||||
// itself is the SDK's (buildstep.MergeInto), the one every build module's fetch step makes; what is here is the answer
|
||||
// built from it and the placing of the sibling repositories the tests read.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// Sibling is a repository placed beside the fetched tree: its name (the directory under beside/) and its commit.
|
||||
type Sibling struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
Dir string `json:"dir"`
|
||||
}
|
||||
|
||||
// remote is the address a repository is fetched from: the forge over ssh as the account, or RemoteBase when a test set it.
|
||||
func (m *Module) remote(repository string) string {
|
||||
base := m.RemoteBase
|
||||
if base == "" {
|
||||
base = "ssh://" + ForgeAlias + "/"
|
||||
}
|
||||
return base + repository + ".git"
|
||||
}
|
||||
|
||||
// merge makes the merge preview of the head onto base and writes the answer into res.
|
||||
func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base, commit string) error {
|
||||
mr, err := buildstep.MergeInto(ctx, m.Run, tree, remote, base, commit, m.FetchOptions...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res.Base = mr.Base
|
||||
res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed
|
||||
if res.Merge == "" {
|
||||
return nil
|
||||
}
|
||||
res.Trees, err = buildstep.ModuleTrees(ctx, m.Run, nil, tree, res.Merge)
|
||||
return err
|
||||
}
|
||||
|
||||
// ref is the pull request head carrying the commit, when only it does (a pin on a pull request, issue 1148).
|
||||
type siblingArg struct{ repository, commit, ref string }
|
||||
|
||||
func siblingsArg(v any) ([]siblingArg, error) {
|
||||
if v == nil {
|
||||
return nil, nil
|
||||
}
|
||||
list, ok := v.([]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("beside is a list of {repository, commit}")
|
||||
}
|
||||
var out []siblingArg
|
||||
seen := map[string]bool{}
|
||||
for _, e := range list {
|
||||
o, _ := e.(map[string]any)
|
||||
repository, _ := o["repository"].(string)
|
||||
commit, _ := o["commit"].(string)
|
||||
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
|
||||
return nil, fmt.Errorf("beside: %q is not a repository, <owner>/<name>", repository)
|
||||
}
|
||||
if !commitOK.MatchString(commit) {
|
||||
return nil, fmt.Errorf("beside: %q is not a full commit (40 hex digits)", commit)
|
||||
}
|
||||
name := repository[strings.Index(repository, "/")+1:]
|
||||
if seen[name] {
|
||||
return nil, fmt.Errorf("beside: two repositories are named %s", name)
|
||||
}
|
||||
seen[name] = true
|
||||
out = append(out, siblingArg{repository: repository, commit: commit})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// BesideDir is where a workspace's siblings are placed, and what MESH_CHECK_BESIDE names.
|
||||
func (m *Module) besideDir(workspace string) string {
|
||||
return filepath.Join(m.Work, workspace, "beside")
|
||||
}
|
||||
|
||||
// placeSiblings fetches each sibling into beside/<name> of the workspace root, as a plain directory. A symlink anywhere
|
||||
// in a placed tree fails the step naming it: the mesh creates none, and a test must not be led out of its sibling.
|
||||
func (m *Module) placeSiblings(ctx context.Context, root string, siblings []siblingArg) ([]Sibling, error) {
|
||||
var placed []Sibling
|
||||
for _, s := range siblings {
|
||||
name := s.repository[strings.Index(s.repository, "/")+1:]
|
||||
dir := filepath.Join(root, "beside", name)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, s.ref, m.FetchOptions...); err != nil {
|
||||
return nil, fmt.Errorf("beside %s: %w", s.repository, err)
|
||||
}
|
||||
if link := firstSymlink(dir); link != "" {
|
||||
return nil, fmt.Errorf("beside %s holds a symlink, %s", s.repository, link)
|
||||
}
|
||||
placed = append(placed, Sibling{Repository: s.repository, Commit: s.commit, Dir: name})
|
||||
}
|
||||
sort.Slice(placed, func(i, j int) bool { return placed[i].Dir < placed[j].Dir })
|
||||
if len(placed) > 0 {
|
||||
// What the digest folds in (the lead's decision L5): the siblings this workspace's tests read, by commit.
|
||||
raw, err := json.Marshal(placed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, besideRecord), raw, 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return placed, nil
|
||||
}
|
||||
|
||||
// besideRecord is the file in a workspace's root naming the siblings the fetch placed beside it.
|
||||
const besideRecord = "beside.json"
|
||||
|
||||
// besideLines are the siblings a workspace's fetch placed, as the digest folds them in and echoes them: "<owner/repo>
|
||||
// <commit>", sorted; none when it placed none.
|
||||
func (m *Module) besideLines(workspace string) ([]string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(m.Work, workspace, besideRecord))
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var placed []Sibling
|
||||
if err := json.Unmarshal(raw, &placed); err != nil {
|
||||
return nil, fmt.Errorf("the workspace's record of its siblings: %w", err)
|
||||
}
|
||||
var out []string
|
||||
for _, s := range placed {
|
||||
if !repositoryOK.MatchString(s.Repository) || !commitOK.MatchString(s.Commit) {
|
||||
return nil, fmt.Errorf("the workspace's record of its siblings names %q at %q", s.Repository, s.Commit)
|
||||
}
|
||||
out = append(out, s.Repository+" "+s.Commit)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func firstSymlink(dir string) string {
|
||||
var found string
|
||||
filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
|
||||
if err == nil && d.Type()&fs.ModeSymlink != 0 && found == "" {
|
||||
found, _ = filepath.Rel(dir, p)
|
||||
return fs.SkipAll
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return found
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
// The fetch step's merge and siblings on real git repositories and the real go (issues 1675 and 1677): they start
|
||||
// processes, so they carry the tag; preview_test.go proves the commands with a fake runner.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func git(t *testing.T, dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"}, args...)...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func put(t *testing.T, dir, name, body string) {
|
||||
t.Helper()
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// forge is a directory of bare repositories <owner>/<name>.git standing in for the forge, and a module fetching from it.
|
||||
type forge struct {
|
||||
t *testing.T
|
||||
root string
|
||||
}
|
||||
|
||||
func newForge(t *testing.T) *forge { return &forge{t: t, root: t.TempDir()} }
|
||||
|
||||
// repo makes the repository owner/name with the files on main and answers its work directory and the commit.
|
||||
func (f *forge) repo(name string, files map[string]string) (work, commit string) {
|
||||
bare := filepath.Join(f.root, "forge", name+".git")
|
||||
work = filepath.Join(f.root, "work", name)
|
||||
os.MkdirAll(bare, 0o755)
|
||||
os.MkdirAll(work, 0o755)
|
||||
git(f.t, bare, "init", "-q", "--bare")
|
||||
git(f.t, work, "init", "-q", "-b", "main")
|
||||
for n, c := range files {
|
||||
put(f.t, work, n, c)
|
||||
}
|
||||
git(f.t, work, "add", ".")
|
||||
git(f.t, work, "commit", "-q", "-m", "one")
|
||||
git(f.t, work, "push", "-q", bare, "HEAD:refs/heads/main")
|
||||
return work, git(f.t, work, "rev-parse", "HEAD")
|
||||
}
|
||||
|
||||
// commit makes a commit off `from` in the repository's work directory and publishes it as the pull request's head.
|
||||
func (f *forge) commit(name, work, from string, files map[string]string, ref string) string {
|
||||
git(f.t, work, "checkout", "-q", "-B", "b"+ref[len(ref)-3:], from)
|
||||
for n, c := range files {
|
||||
put(f.t, work, n, c)
|
||||
}
|
||||
git(f.t, work, "add", ".")
|
||||
git(f.t, work, "commit", "-q", "-m", "change")
|
||||
sha := git(f.t, work, "rev-parse", "HEAD")
|
||||
git(f.t, work, "push", "-q", "-f", filepath.Join(f.root, "forge", name+".git"), "HEAD:"+ref)
|
||||
return sha
|
||||
}
|
||||
|
||||
func (f *forge) module(t *testing.T) (*Module, *keptStates) {
|
||||
m, states := fetchModule(t, &fakeStep{})
|
||||
m.Run = StepRunner(m.Home)
|
||||
m.RemoteBase = "file://" + filepath.Join(f.root, "forge") + "/"
|
||||
m.FetchOptions = []buildstep.Option{buildstep.AllowFileRemote()}
|
||||
return m, states
|
||||
}
|
||||
|
||||
// waitLong is waitEnd for a step that runs the real go, whose first build in a cold cache takes longer than 3 s.
|
||||
func waitLong(t *testing.T, s *keptStates) delivery.Step {
|
||||
t.Helper()
|
||||
select {
|
||||
case key := <-s.ended:
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v := s.steps[key]
|
||||
return v[len(v)-1]
|
||||
case <-time.After(2 * time.Minute):
|
||||
t.Fatal("the step never ended on its key")
|
||||
return delivery.Step{}
|
||||
}
|
||||
}
|
||||
|
||||
func fetched(t *testing.T, m *Module, states *keptStates, args map[string]any) (FetchResult, delivery.Step) {
|
||||
t.Helper()
|
||||
if _, err := fetchTool(t, m)(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res FetchResult
|
||||
if st.State == delivery.StepPassed {
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
|
||||
var baseFiles = map[string]string{
|
||||
"go.mod": "module fixture/m\n\ngo 1.26\n",
|
||||
"a.txt": "one\ntwo\nthree\n",
|
||||
"modules/m/module.json": "{\"name\":\"m\"}\n",
|
||||
"modules/m/go.sum": "",
|
||||
"modules/m/m.txt": "m\n",
|
||||
"modules/other/other.txt": "o\n",
|
||||
}
|
||||
|
||||
// 1675/AC1 and AC2 on real git: a clean merge answers the merge commit, the changed paths (the trunk's change is not
|
||||
// among them) and the tree of each module directory, which equals git's own; a conflict is a passed step naming the files,
|
||||
// leaving the workspace at the trunk commit; and no base is today's fetch.
|
||||
func TestRealFetchMergesConflictsAndPlainlyFetches(t *testing.T) {
|
||||
f := newForge(t)
|
||||
work, fork := f.repo("o/r", baseFiles)
|
||||
trunk := f.commit("o/r", work, fork, map[string]string{"modules/other/other.txt": "trunk\n", "a.txt": "one\ntwo\nTRUNK\n"}, "refs/heads/main")
|
||||
clean := f.commit("o/r", work, fork, map[string]string{"modules/m/m.txt": "head\n"}, "refs/pull/1/head")
|
||||
clash := f.commit("o/r", work, fork, map[string]string{"a.txt": "one\ntwo\nHEAD\n"}, "refs/pull/2/head")
|
||||
m, states := f.module(t)
|
||||
|
||||
res, st := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head", "base": trunk})
|
||||
if st.State != delivery.StepPassed || res.Merge == "" || res.Base != trunk || res.Commit != clean || len(res.Conflicts) != 0 {
|
||||
t.Fatalf("clean: %s %s", st.State, st.Result)
|
||||
}
|
||||
if strings.Join(res.Changed, ",") != "modules/m/m.txt" {
|
||||
t.Errorf("changed = %v, want the head's own change alone", res.Changed)
|
||||
}
|
||||
tree := res.Trees["modules/m"]
|
||||
if want := git(t, filepath.Join(m.Work, res.Workspace, "tree"), "rev-parse", res.Merge+":modules/m"); tree.Tree != want {
|
||||
t.Errorf("tree = %s, want %s", tree.Tree, want)
|
||||
}
|
||||
if tree.Manifest != sha256Hex([]byte("{\"name\":\"m\"}\n")) || tree.Locks["go.sum"] != sha256Hex(nil) {
|
||||
t.Errorf("manifest and locks = %+v", tree)
|
||||
}
|
||||
if got := git(t, filepath.Join(m.Work, res.Workspace, "tree"), "rev-parse", "HEAD"); got != res.Merge {
|
||||
t.Errorf("the workspace is at %s, want the merge %s", got, res.Merge)
|
||||
}
|
||||
|
||||
res, st = fetched(t, m, states, map[string]any{"repository": "o/r", "commit": clash, "ref": "refs/pull/2/head", "base": trunk})
|
||||
if st.State != delivery.StepPassed || res.Merge != "" || strings.Join(res.Conflicts, ",") != "a.txt" {
|
||||
t.Fatalf("conflict: %s %s", st.State, st.Result)
|
||||
}
|
||||
if got := git(t, filepath.Join(m.Work, res.Workspace, "tree"), "rev-parse", "HEAD"); got != trunk {
|
||||
t.Errorf("after a conflict the workspace is at %s, want the trunk %s", got, trunk)
|
||||
}
|
||||
|
||||
res, st = fetched(t, m, states, map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head"})
|
||||
if st.State != delivery.StepPassed || res.Merge != "" || res.Base != "" || res.Trees != nil || len(res.Modules) != 1 {
|
||||
t.Errorf("no base: %s %s", st.State, st.Result)
|
||||
}
|
||||
}
|
||||
|
||||
// 1677/AC1 on the real go: the unit level of a repository whose test reads MESH_CHECK_BESIDE fails loudly without the
|
||||
// sibling and passes with it; the sibling is a plain directory with no symlink in it; and a sibling that holds a
|
||||
// symlink fails the fetch naming it.
|
||||
func TestRealTestsReadTheSiblingBesideTheTree(t *testing.T) {
|
||||
needTools(t, "the shut-off unit level", "go", "git", "bwrap") // the tests run in bubblewrap (1309)
|
||||
f := newForge(t)
|
||||
files := map[string]string{}
|
||||
for _, n := range []string{"go.mod", "beside.go", "beside_test.go"} {
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "repos", "beside", n))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files[n] = string(raw)
|
||||
}
|
||||
_, head := f.repo("o/r", files)
|
||||
_, sibling := f.repo("o/sibling", map[string]string{"data.txt": "from the sibling\n"})
|
||||
m, states := f.module(t)
|
||||
|
||||
without, _ := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": head})
|
||||
if _, err := m.testToolRun(t, without.Workspace); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var res TestResult
|
||||
st := waitLong(t, states)
|
||||
json.Unmarshal(st.Result, &res)
|
||||
if st.State != delivery.StepPassed || res.Passed || len(res.Failed) != 1 || !strings.Contains(res.Failed[0].Output, "MESH_CHECK_BESIDE is not set") {
|
||||
t.Fatalf("without the sibling: %s %s", st.State, st.Result)
|
||||
}
|
||||
|
||||
with, st2 := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": head,
|
||||
"beside": []any{map[string]any{"repository": "o/sibling", "commit": sibling}}})
|
||||
if st2.State != delivery.StepPassed || len(with.Beside) != 1 {
|
||||
t.Fatalf("with the sibling: %s %s", st2.State, st2.Result)
|
||||
}
|
||||
m.testToolRun(t, with.Workspace)
|
||||
res = TestResult{}
|
||||
st = waitLong(t, states)
|
||||
json.Unmarshal(st.Result, &res)
|
||||
if !res.Passed {
|
||||
t.Errorf("with the sibling the level failed: %s", st.Result)
|
||||
}
|
||||
if link := firstSymlink(filepath.Join(m.Work, with.Workspace, "beside")); link != "" {
|
||||
t.Errorf("the placed tree holds a symlink, %s", link)
|
||||
}
|
||||
|
||||
_, bad := f.repo("o/linked", map[string]string{"data.txt": "x\n"})
|
||||
work := filepath.Join(f.root, "work", "o/linked")
|
||||
if err := os.Symlink("/etc", filepath.Join(work, "etc-link")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
git(t, work, "add", ".")
|
||||
git(t, work, "commit", "-q", "-m", "link")
|
||||
bad = git(t, work, "rev-parse", "HEAD")
|
||||
git(t, work, "push", "-q", "-f", filepath.Join(f.root, "forge", "o/linked.git"), "HEAD:refs/heads/main")
|
||||
_, st3 := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": head,
|
||||
"beside": []any{map[string]any{"repository": "o/linked", "commit": bad}}})
|
||||
if st3.State != delivery.StepFailed || !strings.Contains(st3.Reason, "etc-link") {
|
||||
t.Errorf("a sibling with a symlink: %s %s", st3.State, st3.Reason)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
const (
|
||||
baseSHA = "abcdef0123456789abcdef0123456789abcdef01"
|
||||
mergeSHA = "fedcba9876543210fedcba9876543210fedcba98"
|
||||
treeSHA = "1111111111111111111111111111111111111111"
|
||||
)
|
||||
|
||||
// previewRules answers the merge's commands as git does: the merge commit, the changed paths, the manifests, the trees.
|
||||
func previewRules(conflict bool) func(string) (forgekey.Ran, bool) {
|
||||
merged := false
|
||||
return func(line string) (forgekey.Ran, bool) {
|
||||
switch {
|
||||
case strings.Contains(line, "--is-shallow-repository"):
|
||||
return forgekey.Ran{Stdout: "true\n"}, true
|
||||
case strings.Contains(line, " merge --no-ff"):
|
||||
if conflict {
|
||||
return forgekey.Ran{Status: 1, Stderr: "CONFLICT (content): Merge conflict in a.go\n"}, true
|
||||
}
|
||||
merged = true
|
||||
return forgekey.Ran{}, true
|
||||
case strings.Contains(line, "diff --name-only --diff-filter=U"):
|
||||
return forgekey.Ran{Stdout: "a.go\nsub/b.go\n"}, true
|
||||
case strings.Contains(line, "diff --name-only --no-renames"):
|
||||
return forgekey.Ran{Stdout: "a.go\nmodules/m/main.go\n"}, true
|
||||
case strings.Contains(line, " rev-parse HEAD") && merged:
|
||||
return forgekey.Ran{Stdout: mergeSHA + "\n"}, true
|
||||
case strings.Contains(line, " ls-files ") && strings.Contains(line, "module.json"):
|
||||
return forgekey.Ran{Stdout: "modules/m/module.json\n"}, true
|
||||
case strings.Contains(line, " rev-parse "+mergeSHA+":modules/m"):
|
||||
return forgekey.Ran{Stdout: treeSHA + "\n"}, true
|
||||
}
|
||||
return forgekey.Ran{}, false
|
||||
}
|
||||
}
|
||||
|
||||
// 1675/AC1: with a base the fetch merges the head onto it with the SDK's merge (the identity and flags of the check
|
||||
// merge, hooks off), and answers base, merge commit, changed paths and, per module directory, tree, manifest hash and lock hashes.
|
||||
func TestFetchWithABaseMergesAndAnswersThePreview(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n", rule: previewRules(false)}
|
||||
m, states := fetchModule(t, f)
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
// the workspace's module.json and go.sum are on disk once the fake checks the head out
|
||||
for i, a := range args {
|
||||
if a == "-C" && len(args) > i+1 && strings.Contains(strings.Join(args, " "), " checkout ") {
|
||||
os.MkdirAll(filepath.Join(args[i+1], "modules", "m"), 0o755)
|
||||
os.WriteFile(filepath.Join(args[i+1], "modules", "m", "module.json"), []byte("{}\n"), 0o644)
|
||||
os.WriteFile(filepath.Join(args[i+1], "modules", "m", "go.sum"), []byte("sum\n"), 0o644)
|
||||
}
|
||||
}
|
||||
return f.run(ctx, env, name, args...)
|
||||
}
|
||||
args := map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "base": baseSHA}
|
||||
if _, err := fetchTool(t, m)(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
var res FetchResult
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Base != baseSHA || res.Merge != mergeSHA || res.Commit != commitSHA || len(res.Conflicts) != 0 ||
|
||||
strings.Join(res.Changed, ",") != "a.go,modules/m/main.go" {
|
||||
t.Errorf("result = %s", st.Result)
|
||||
}
|
||||
mt, ok := res.Trees["modules/m"]
|
||||
if !ok || mt.Tree != treeSHA || mt.Manifest != sha256Hex([]byte("{}\n")) || mt.Locks["go.sum"] != sha256Hex([]byte("sum\n")) {
|
||||
t.Errorf("trees = %+v", res.Trees)
|
||||
}
|
||||
lines := strings.Join(f.lines(), "\n")
|
||||
for _, want := range []string{"fetch --no-tags --unshallow origin " + baseSHA, "checkout -q --detach " + baseSHA,
|
||||
"core.hooksPath=/dev/null", "merge --no-ff --no-edit --no-verify"} {
|
||||
if !strings.Contains(lines, want) {
|
||||
t.Errorf("git was not asked %q:\n%s", want, lines)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1675/AC2: a conflict is an answer: the step ends passed, the files are named, no merge commit, nothing downloaded.
|
||||
func TestFetchWithAConflictEndsPassedAndNamesTheFiles(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n", rule: previewRules(true)}
|
||||
m, states := fetchModule(t, f)
|
||||
args := map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "base": baseSHA}
|
||||
if _, err := fetchTool(t, m)(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("a conflict ended the step %s: %s", st.State, st.Reason)
|
||||
}
|
||||
var res FetchResult
|
||||
json.Unmarshal(st.Result, &res)
|
||||
if res.Merge != "" || res.Base != baseSHA || strings.Join(res.Conflicts, ",") != "a.go,sub/b.go" || len(res.Changed) != 0 {
|
||||
t.Errorf("result = %s", st.Result)
|
||||
}
|
||||
lines := strings.Join(f.lines(), "\n")
|
||||
if !strings.Contains(lines, "merge --abort") || strings.Contains(lines, "mod download") {
|
||||
t.Errorf("the merge was not undone, or modules were downloaded:\n%s", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// 1675/AC3: no base is today's fetch: no merge, no extra command, and the answer carries none of the preview.
|
||||
func TestFetchWithoutABaseIsTodaysFetch(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n"}
|
||||
m, states := fetchModule(t, f)
|
||||
fetchTool(t, m)(fetchArgs)
|
||||
st := states.waitEnd(t)
|
||||
lines := strings.Join(f.lines(), "\n")
|
||||
if strings.Contains(lines, "merge") || strings.Contains(lines, "unshallow") {
|
||||
t.Errorf("a fetch without a base merged:\n%s", lines)
|
||||
}
|
||||
for _, key := range []string{"base", "merge", "conflicts", "changed", "trees", "beside"} {
|
||||
if strings.Contains(string(st.Result), `"`+key+`"`) {
|
||||
t.Errorf("the answer carries %q without a base: %s", key, st.Result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestABaseThatIsNotACommitIsRefusedBeforeAnythingRuns(t *testing.T) {
|
||||
f := &fakeStep{}
|
||||
m, states := fetchModule(t, f)
|
||||
fetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "base": "main"})
|
||||
if st := states.waitEnd(t); st.State != delivery.StepFailed || len(f.lines()) != 0 {
|
||||
t.Errorf("the step ended %s after %d commands", st.State, len(f.lines()))
|
||||
}
|
||||
}
|
||||
|
||||
// 1677/AC1: each sibling is fetched into beside/<name> of the workspace, and the tests of the workspace are given
|
||||
// MESH_CHECK_BESIDE naming that directory, at the unit level; a workspace without siblings is given none.
|
||||
func TestSiblingsArePlacedBesideAndTheUnitLevelIsGivenTheirDirectory(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n"}
|
||||
m, states := fetchModule(t, f)
|
||||
args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{
|
||||
map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA},
|
||||
map[string]any{"repository": "novox/mesh-host", "commit": commitSHA}}}
|
||||
if _, err := fetchTool(t, m)(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res FetchResult
|
||||
json.Unmarshal(st.Result, &res)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if len(res.Beside) != 2 || res.Beside[0].Dir != "mesh-catalog" || res.Beside[1].Dir != "mesh-host" {
|
||||
t.Fatalf("result = %s", st.Result)
|
||||
}
|
||||
lines := strings.Join(f.lines(), "\n")
|
||||
for _, d := range []string{"beside/mesh-catalog", "beside/mesh-host"} {
|
||||
if !strings.Contains(lines, filepath.Join(m.Work, res.Workspace, d)+" init") && !strings.Contains(lines, "-C "+filepath.Join(m.Work, res.Workspace, d)) {
|
||||
t.Errorf("nothing was fetched into %s:\n%s", d, lines)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(lines, "remote add origin ssh://mesh-forge/novox/mesh-catalog.git") {
|
||||
t.Errorf("the sibling was not fetched over ssh:\n%s", lines)
|
||||
}
|
||||
|
||||
calls := &[]cmdRecord{}
|
||||
m.Run = throughBubblewrap(t, capturedRunner(t, "failing", calls), &[]shutOff{})
|
||||
if err := os.MkdirAll(filepath.Join(m.Work, res.Workspace, "tree"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.testToolRun(t, res.Workspace); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
states.waitEnd(t)
|
||||
want := "MESH_CHECK_BESIDE=" + filepath.Join(m.Work, res.Workspace, "beside")
|
||||
if !envHas(*calls, "go", want) {
|
||||
t.Errorf("no go command was given %s", want)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) testToolRun(t *testing.T, ws string) (any, error) {
|
||||
t.Helper()
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == "build_go_test" {
|
||||
return tool.Run(map[string]any{"workspace": ws, "level": "unit"})
|
||||
}
|
||||
}
|
||||
t.Fatal("no tool build_go_test")
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func envHas(calls []cmdRecord, name, want string) bool {
|
||||
for _, c := range calls {
|
||||
if strings.HasPrefix(c.line, name+" ") {
|
||||
for _, e := range c.env {
|
||||
if e == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestAWorkspaceWithoutSiblingsIsGivenNoBesideVariable(t *testing.T) {
|
||||
m, states, ws, calls := workspaceFor(t, "failing")
|
||||
runUnit(t, m, states, ws)
|
||||
for _, c := range *calls {
|
||||
for _, e := range c.env {
|
||||
if strings.HasPrefix(e, "MESH_CHECK_BESIDE=") {
|
||||
t.Errorf("%s was given %s", c.line, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSiblingsThatAreNotRepositoriesOrCommitsAreRefused(t *testing.T) {
|
||||
for name, beside := range map[string]any{
|
||||
"not a list": "x",
|
||||
"bad repository": []any{map[string]any{"repository": "../x/y", "commit": baseSHA}},
|
||||
"bad commit": []any{map[string]any{"repository": "novox/a", "commit": "main"}},
|
||||
"twice": []any{map[string]any{"repository": "novox/a", "commit": baseSHA}, map[string]any{"repository": "other/a", "commit": baseSHA}},
|
||||
} {
|
||||
f := &fakeStep{}
|
||||
m, states := fetchModule(t, f)
|
||||
fetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "beside": beside})
|
||||
if st := states.waitEnd(t); st.State != delivery.StepFailed || len(f.lines()) != 0 {
|
||||
t.Errorf("%s: the step ended %s after %d commands", name, st.State, len(f.lines()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1288/AC5 (the checker's contract): an import path runs only in the go.mod directory whose module holds it; a
|
||||
// relative pattern runs in every directory; no pattern is ./... everywhere.
|
||||
func TestPackagesRunOnlyInTheModuleThatHoldsThem(t *testing.T) {
|
||||
tree := t.TempDir()
|
||||
for dir, mod := range map[string]string{".": "example.org/root", "tools": "example.org/root/tools", "other": "example.org/other"} {
|
||||
os.MkdirAll(filepath.Join(tree, dir), 0o755)
|
||||
os.WriteFile(filepath.Join(tree, dir, "go.mod"), []byte("module "+mod+"\n\ngo 1.26\n"), 0o644)
|
||||
}
|
||||
dirs := []string{".", "other", "tools"}
|
||||
got, err := packagesByDir(tree, dirs, []string{"example.org/root/a", "example.org/root/tools/b", "example.org/other"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{".": "example.org/root/a", "tools": "example.org/root/tools/b", "other": "example.org/other"}
|
||||
for d, w := range want {
|
||||
if strings.Join(got[d], ",") != w {
|
||||
t.Errorf("%s runs %v, want %s", d, got[d], w)
|
||||
}
|
||||
}
|
||||
got, _ = packagesByDir(tree, dirs, []string{"./x/..."})
|
||||
if len(got["."]) != 1 || len(got["tools"]) != 1 || len(got["other"]) != 1 {
|
||||
t.Errorf("a relative pattern = %v", got)
|
||||
}
|
||||
got, _ = packagesByDir(tree, dirs, nil)
|
||||
if got["tools"][0] != "./..." {
|
||||
t.Errorf("no pattern = %v", got)
|
||||
}
|
||||
if _, err := packagesByDir(tree, dirs, []string{"example.org/nowhere/x"}); err == nil || !strings.Contains(err.Error(), "example.org/nowhere/x") {
|
||||
t.Errorf("a package no module holds: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
|
||||
|
||||
// 1677/AC (integration level): the integration level's go test is given MESH_CHECK_BESIDE when the workspace has siblings
|
||||
// beside it, with the store's and the bus's addresses, and is given none when it has not.
|
||||
func TestTheIntegrationLevelIsGivenTheBesideDirectoryOnlyWhenSiblingsWerePlaced(t *testing.T) {
|
||||
for _, siblings := range []bool{true, false} {
|
||||
f := newIntegFake(t, "slow")
|
||||
m, states := integModule(t, f)
|
||||
beside := filepath.Join(m.Work, "ws1", "beside")
|
||||
if siblings {
|
||||
if err := os.MkdirAll(filepath.Join(beside, "mesh-catalog"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, st := runLevelTool(t, m, states, "integration"); st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
tested := 0
|
||||
for _, c := range f.calls {
|
||||
if !strings.HasPrefix(c.line, "go ") || !strings.Contains(c.line, "-tags integration") {
|
||||
continue
|
||||
}
|
||||
tested++
|
||||
got := ""
|
||||
hasNATS := false
|
||||
for _, e := range c.env {
|
||||
if strings.HasPrefix(e, "MESH_CHECK_BESIDE=") {
|
||||
got = strings.TrimPrefix(e, "MESH_CHECK_BESIDE=")
|
||||
}
|
||||
hasNATS = hasNATS || strings.HasPrefix(e, "MESH_TEST_NATS=")
|
||||
}
|
||||
want := ""
|
||||
if siblings {
|
||||
want = beside
|
||||
}
|
||||
if got != want || !hasNATS {
|
||||
t.Errorf("siblings %v: %s was given MESH_CHECK_BESIDE=%q (want %q), MESH_TEST_NATS present %v", siblings, c.line, got, want, hasNATS)
|
||||
}
|
||||
}
|
||||
if tested == 0 {
|
||||
t.Errorf("siblings %v: no integration go test ran", siblings)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
// The push of a done build's archive (novox/hq ADR 0498 §1 §2; issue 1587): from this module's own process, after the
|
||||
// build step ended, with mesh-sdk's one client (buildstep.PushBlob), to the artifact store whose address the manifest
|
||||
// binds into RegistryFile. No build step reaches the store: a run step has no network (ADR 0448).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// registry is the artifact store's address, host:port, as the manifest bound it into RegistryFile.
|
||||
func (m *Module) registry() (string, error) {
|
||||
raw, err := os.ReadFile(m.RegistryFile)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the artifact store's address cannot be read: %w", err)
|
||||
}
|
||||
addr := strings.TrimSpace(string(raw))
|
||||
if addr == "" || strings.Contains(addr, "${") || strings.HasPrefix(addr, ":") || strings.HasSuffix(addr, ":") {
|
||||
return "", fmt.Errorf("the artifact store's address in %s is %q, not host:port", m.RegistryFile, addr)
|
||||
}
|
||||
return addr, nil
|
||||
}
|
||||
|
||||
// push puts body into the artifact store as a blob of repository (<module>/<artifact>) by its digest: Push when a test
|
||||
// set it, else buildstep.PushBlob.
|
||||
func (m *Module) push(ctx context.Context, repository, digest string, body []byte) error {
|
||||
addr, err := m.registry()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m.Push != nil {
|
||||
return m.Push(ctx, addr, repository, digest, body)
|
||||
}
|
||||
return buildstep.PushBlob(ctx, &http.Client{Timeout: 10 * time.Minute}, addr, repository, digest, body)
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back.
|
||||
var pushed sync.Map
|
||||
|
||||
// fakeStore makes m push to a store of the test's own: its address in RegistryFile, each blob kept by repository and
|
||||
// digest; fail, when set, is the store's refusal.
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte // "<repository>@<digest>"
|
||||
fail error
|
||||
addr string
|
||||
}
|
||||
|
||||
func withStore(t *testing.T, m *Module) *fakeStore {
|
||||
t.Helper()
|
||||
s := &fakeStore{blobs: map[string][]byte{}}
|
||||
m.RegistryFile = filepath.Join(t.TempDir(), "registry")
|
||||
if err := os.WriteFile(m.RegistryFile, []byte("store.test:5000\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Push = func(ctx context.Context, registry, repository, digest string, body []byte) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.addr = registry
|
||||
if s.fail != nil {
|
||||
return s.fail
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if "sha256:"+hex.EncodeToString(sum[:]) != digest {
|
||||
t.Errorf("pushed %s with bytes of another digest", digest)
|
||||
}
|
||||
s.blobs[repository+"@"+digest] = append([]byte(nil), body...)
|
||||
pushed.Store(digest, s.blobs[repository+"@"+digest])
|
||||
return nil
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// archiveOf is the archive a build pushed, by its digest.
|
||||
func archiveOf(t *testing.T, res BuildResult) []byte {
|
||||
t.Helper()
|
||||
raw, ok := pushed.Load(res.Digest)
|
||||
if !ok {
|
||||
t.Fatalf("nothing was pushed under %s", res.Digest)
|
||||
}
|
||||
return raw.([]byte)
|
||||
}
|
||||
|
||||
// 1587-S2 (ADR 0498 §1): a done build answers a reference of the stated form whose tail is the build's digest, and the
|
||||
// store holds the archive there; a bundle and an archive alike.
|
||||
func TestADoneBuildAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
store := withStore(t, m)
|
||||
res, st := runBuild(t, m, states, "tools")
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if want := "artifact-store://demo/tools/blobs/" + res.Digest; res.Reference != want {
|
||||
t.Errorf("reference %q, want %q", res.Reference, want)
|
||||
}
|
||||
if store.addr != "store.test:5000" || store.blobs["demo/tools@"+res.Digest] == nil {
|
||||
t.Errorf("the store at %q holds %d blobs, not demo/tools@%s", store.addr, len(store.blobs), res.Digest)
|
||||
}
|
||||
a, states2, _ := archiveModule(t, "")
|
||||
withStore(t, a)
|
||||
arch, st := runBuild(t, a, states2, "files")
|
||||
if st.State != delivery.StepPassed || arch.Reference != "artifact-store://demo/files/blobs/"+arch.Digest {
|
||||
t.Errorf("the archive artifact: %s %q, reference %q", st.State, st.Reason, arch.Reference)
|
||||
}
|
||||
}
|
||||
|
||||
// 1587-S3 (ADR 0498 §2): a push the store refuses fails the step `not stored: <artifact>: <the store's words>`, and so
|
||||
// does a store whose address the manifest did not place.
|
||||
func TestARefusedPushFailsTheStepNotStored(t *testing.T) {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
store := withStore(t, m)
|
||||
store.fail = errors.New("store.test:5000 refused the blob: 500 Internal Server Error disk full")
|
||||
if _, st := runBuild(t, m, states, "tools"); st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, delivery.NotStored+"tools: ") || !strings.Contains(st.Reason, "disk full") {
|
||||
t.Errorf("the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
m.RegistryFile = filepath.Join(t.TempDir(), "absent")
|
||||
if _, st := runBuild(t, m, states, "tools"); st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, delivery.NotStored+"tools: ") {
|
||||
t.Errorf("no address: the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// 1587-S4 (the lead's decision L9): the packed file is not left on the node once its push answered, a candidate's
|
||||
// included: the workspace holds no archive, pushed or refused.
|
||||
func TestThePackedFileLeavesTheNodeOnceItsPushAnswered(t *testing.T) {
|
||||
for _, fail := range []error{nil, errors.New("refused")} {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
withStore(t, m).fail = fail
|
||||
runBuild(t, m, states, "tools")
|
||||
var archives []string
|
||||
filepath.WalkDir(m.Work, func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil && strings.HasSuffix(p, ".tar.gz") {
|
||||
archives = append(archives, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if len(archives) != 0 {
|
||||
t.Errorf("push answered %v: archives left on the node: %v", fail, archives)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,7 @@ const PostgresMajor = 18
|
||||
|
||||
// RecipeSteps is the recipe's own version: changed whenever the steps of fetch, test or build change in a way
|
||||
// that can change what they produce (ADR 0405 §7). It is one of the three values the recipe version hashes.
|
||||
const RecipeSteps = "1"
|
||||
const RecipeSteps = "2"
|
||||
|
||||
// natsModule is the library the integration level's bus is started from (ADR 0470 §2).
|
||||
const natsModule = "github.com/nats-io/nats-server/v2"
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/distribution/distribution/v3/configuration"
|
||||
"github.com/distribution/distribution/v3/registry/handlers"
|
||||
_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// realStore starts the artifact store's own program (the distribution registry, modules/distribution) inside this
|
||||
// test, with its in-memory storage, on a loopback port, and points m at it with the real push client. It is imported
|
||||
// by this integration test alone: the module's program does not carry it.
|
||||
func realStore(t *testing.T, m *Module) string {
|
||||
t.Helper()
|
||||
config := &configuration.Configuration{Storage: configuration.Storage{"inmemory": configuration.Parameters{}}}
|
||||
config.HTTP.Secret = "a throwaway store"
|
||||
config.Log.Level = "error"
|
||||
config.Log.AccessLog.Disabled = true
|
||||
srv := httptest.NewServer(handlers.NewApp(context.Background(), config))
|
||||
t.Cleanup(srv.Close)
|
||||
addr := strings.TrimPrefix(srv.URL, "http://")
|
||||
m.RegistryFile = filepath.Join(t.TempDir(), "registry")
|
||||
if err := os.WriteFile(m.RegistryFile, []byte(addr+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Push = nil
|
||||
return addr
|
||||
}
|
||||
|
||||
// resolve reads a done artifact's reference from the store, as the controller will: artifact-store://<module>/
|
||||
// <artifact>/blobs/<digest> is <store>/v2/<module>/<artifact>/blobs/<digest>.
|
||||
func resolve(t *testing.T, addr string, a delivery.BuiltArtifact) []byte {
|
||||
t.Helper()
|
||||
path, ok := strings.CutPrefix(a.Reference, "artifact-store://")
|
||||
if !ok {
|
||||
t.Fatalf("%s's reference %q is not the artifact store's", a.Name, a.Reference)
|
||||
}
|
||||
resp, err := http.Get("http://" + addr + "/v2/" + path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("the store answered %s for %s", resp.Status, a.Reference)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// 1587-S9 (ADR 0498's check): a done build's every artifact names a reference whose tail is its digest, and that
|
||||
// reference resolves in a real artifact store to bytes of that digest. The build is the real go's, shut off, of the
|
||||
// fixture testdata/repos/build, with an archive beside its bundle.
|
||||
func TestADoneBuildsArtifactsResolveInARealArtifactStore(t *testing.T) {
|
||||
needTools(t, "the build", "go", "bwrap")
|
||||
work := t.TempDir()
|
||||
tree := filepath.Join(work, "ws1", "tree")
|
||||
if err := os.MkdirAll(tree, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out, err := exec.Command("cp", "-r", filepath.Join("testdata", "repos", "build")+"/.", tree).CombinedOutput(); err != nil {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
manifest := `{"module":"fixture-build","version":"1","build":{"artifacts":[` +
|
||||
`{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/prog","binary":"prog-bin","loads":["prog-bin"]},` +
|
||||
`{"name":"files","kind":"archive","from":"cmd"}]}}`
|
||||
if err := os.WriteFile(filepath.Join(tree, "module.json"), []byte(manifest), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
home := t.TempDir()
|
||||
m := &Module{Home: home, Work: work, Cache: t.TempDir()}
|
||||
m.Run = StepRunner(home)
|
||||
addr := realStore(t, m)
|
||||
done := delivery.BuilderBuild{V: 1, State: delivery.BuildDone, Manifest: strings.Repeat("0", 64)}
|
||||
for _, a := range []struct{ name, kind string }{{"tools", "bundle"}, {"files", "archive"}} {
|
||||
out, err := m.build(context.Background(), map[string]any{"workspace": "ws1", "module": ".", "artifact": a.name})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", a.name, err)
|
||||
}
|
||||
r := out.(BuildResult)
|
||||
done.Artifacts = append(done.Artifacts, delivery.BuiltArtifact{Name: a.name, Kind: a.kind, Digest: r.Digest, Reference: r.Reference})
|
||||
}
|
||||
if _, err := delivery.EncodeBuilderBuild(done); err != nil {
|
||||
t.Errorf("the done key the builder would write is refused: %v", err)
|
||||
}
|
||||
if ok, missing := done.Referenced(); !ok {
|
||||
t.Errorf("the done key is not deliverable, missing %v", missing)
|
||||
}
|
||||
for _, a := range done.Artifacts {
|
||||
if !strings.HasSuffix(a.Reference, "/blobs/"+a.Digest) {
|
||||
t.Errorf("%s: the reference %q does not end in its digest %s", a.Name, a.Reference, a.Digest)
|
||||
}
|
||||
sum := sha256.Sum256(resolve(t, addr, a))
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != a.Digest {
|
||||
t.Errorf("%s resolves to %s, its digest is %s", a.Name, got, a.Digest)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,7 @@ func (m *Module) testTool() StepTool {
|
||||
"level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
|
||||
"packages": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "package patterns of each module, ./... when none"},
|
||||
"run": map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"},
|
||||
},
|
||||
"required": []string{"workspace", "level"},
|
||||
},
|
||||
@@ -56,6 +57,10 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pattern, _ := args["run"].(string)
|
||||
if !runPatternOK(pattern) {
|
||||
return nil, fmt.Errorf("%q is not a go test -run pattern", pattern)
|
||||
}
|
||||
if !workspaceOK.MatchString(workspace) {
|
||||
return nil, fmt.Errorf("%q is not a workspace id", workspace)
|
||||
}
|
||||
@@ -65,13 +70,23 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
||||
}
|
||||
switch level {
|
||||
case LevelUnit:
|
||||
return m.runLevel(ctx, tree, levelOpts{level: LevelUnit, packages: packages, bounds: Bounds{Run: m.unitBound()}})
|
||||
return m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)})
|
||||
case LevelIntegration:
|
||||
return m.integration(ctx, workspace, tree, packages)
|
||||
return m.integration(ctx, workspace, tree, packages, pattern)
|
||||
}
|
||||
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
|
||||
}
|
||||
|
||||
// besideEnv is MESH_CHECK_BESIDE, naming the directory build_go_fetch placed the workspace's siblings in (ADR 0484
|
||||
// clause 4), when it placed any; the tests that read other repositories look there (mesh-sdk go/internal/beside).
|
||||
func (m *Module) besideEnv(workspace string) []string {
|
||||
dir := m.besideDir(workspace)
|
||||
if st, err := os.Stat(dir); err != nil || !st.IsDir() {
|
||||
return nil
|
||||
}
|
||||
return []string{"MESH_CHECK_BESIDE=" + dir}
|
||||
}
|
||||
|
||||
func packagesArg(v any) ([]string, error) {
|
||||
if v == nil {
|
||||
return nil, nil
|
||||
@@ -93,6 +108,10 @@ func packagesArg(v any) ([]string, error) {
|
||||
|
||||
// levelOpts are what a level asks of runLevel.
|
||||
type levelOpts struct {
|
||||
// workspace is the workspace's id: its commands run shut off in it (stepRunner).
|
||||
workspace string
|
||||
// run is go test's -run pattern, "" for every test.
|
||||
run string
|
||||
level string
|
||||
tags []string
|
||||
packages []string
|
||||
@@ -115,18 +134,29 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
|
||||
return nil, err
|
||||
}
|
||||
res := &TestResult{Level: o.level, Packages: []PackageResult{}, Setup: o.setup.Seconds(), Store: o.store, Bus: o.bus}
|
||||
packages := o.packages
|
||||
if len(packages) == 0 {
|
||||
packages = []string{"./..."}
|
||||
byDir, err := packagesByDir(tree, dirs, o.packages)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
run, err := m.stepRunner(o.workspace, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
began := m.clock()
|
||||
for _, d := range dirs {
|
||||
packages := byDir[d]
|
||||
if len(packages) == 0 {
|
||||
continue // no package asked for lies in this module
|
||||
}
|
||||
args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout}
|
||||
if len(o.tags) > 0 {
|
||||
args = append(args, "-tags", strings.Join(o.tags, ","))
|
||||
}
|
||||
if o.run != "" {
|
||||
args = append(args, "-run", o.run)
|
||||
}
|
||||
args = append(args, packages...)
|
||||
r := m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
|
||||
r := run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
|
||||
rd := newReader(res, d)
|
||||
noise := rd.read(r.Stdout)
|
||||
rd.finish()
|
||||
@@ -141,3 +171,69 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
|
||||
res.judge(o.bounds)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// packagesByDir says which patterns run in which go.mod directory. A relative pattern (./...) runs in every directory, as
|
||||
// it always did; an import path, as build_go_digest keys its packages, runs only in the directory whose module holds it
|
||||
// (the longest module path that is the path or its prefix), as another module's path would fail the step. No pattern
|
||||
// means ./... everywhere. An import path no module of the tree holds is an error that names it.
|
||||
func packagesByDir(tree string, dirs, patterns []string) (map[string][]string, error) {
|
||||
out := map[string][]string{}
|
||||
if len(patterns) == 0 {
|
||||
patterns = []string{"./..."}
|
||||
}
|
||||
modules := map[string]string{}
|
||||
for _, d := range dirs {
|
||||
if raw, err := os.ReadFile(filepath.Join(tree, d, "go.mod")); err == nil {
|
||||
modules[d] = modulePath(string(raw))
|
||||
}
|
||||
}
|
||||
for _, p := range patterns {
|
||||
if strings.HasPrefix(p, "./") || p == "." {
|
||||
for _, d := range dirs {
|
||||
out[d] = append(out[d], p)
|
||||
}
|
||||
continue
|
||||
}
|
||||
owner, best := "", -1
|
||||
for _, d := range dirs {
|
||||
mp := modules[d]
|
||||
if mp != "" && (p == mp || strings.HasPrefix(p, mp+"/")) && len(mp) > best {
|
||||
owner, best = d, len(mp)
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
return nil, fmt.Errorf("no module of the workspace holds the package %s", p)
|
||||
}
|
||||
out[owner] = append(out[owner], p)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// modulePath is the module path a go.mod declares, "" when it declares none.
|
||||
func modulePath(gomod string) string {
|
||||
for _, l := range strings.Split(gomod, "\n") {
|
||||
if f := strings.Fields(l); len(f) == 2 && f[0] == "module" {
|
||||
return strings.Trim(f[1], `"`)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// runPatternOK says whether a -run pattern is one go test takes as a single argument: no flag, no space, no control
|
||||
// character, and short ("" is every test).
|
||||
func runPatternOK(p string) bool {
|
||||
if len(p) > 512 || strings.HasPrefix(p, "-") {
|
||||
return false
|
||||
}
|
||||
for _, r := range p {
|
||||
if r <= ' ' || r == 0x7f {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if p != "" {
|
||||
if _, err := regexp.Compile(strings.Split(p, "/")[0]); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -21,6 +21,9 @@ func TestMain(m *testing.M) {
|
||||
if len(os.Args) > 1 && os.Args[1] == busArg {
|
||||
os.Exit(runBus(os.Args[2:]))
|
||||
}
|
||||
if len(os.Args) > 1 && os.Args[1] == levelArg {
|
||||
os.Exit(runLevelProcess(os.Args[2:])) // and the integration level's shut-off process (level.go)
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
@@ -54,7 +57,7 @@ func repository(t *testing.T, name string) string {
|
||||
// test_test.go read the output captured from them, and this one proves the capture is what the real go says today. It
|
||||
// starts go test, so it carries the tag.
|
||||
func TestUnitLevelRunsTheRealGoOnTheFixtureRepositories(t *testing.T) {
|
||||
needTools(t, "the unit level on fixtures", "go", "git")
|
||||
needTools(t, "the unit level on fixtures", "go", "git", "bwrap")
|
||||
want := map[string]struct {
|
||||
passed, noTests bool
|
||||
failed string
|
||||
@@ -66,11 +69,18 @@ func TestUnitLevelRunsTheRealGoOnTheFixtureRepositories(t *testing.T) {
|
||||
"broken": {failed: "fixture/broken"},
|
||||
}
|
||||
for name, w := range want {
|
||||
tree := repository(t, name)
|
||||
home := t.TempDir()
|
||||
m := &Module{Home: home, Work: t.TempDir(), Cache: t.TempDir()}
|
||||
// The tree lies in a workspace of the work directory, as a fetch leaves it: the level runs shut off in it (1309).
|
||||
tree := filepath.Join(m.Work, "ws1", "tree")
|
||||
if err := os.MkdirAll(filepath.Dir(tree), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(repository(t, name), tree); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Run = StepRunner(home)
|
||||
res, err := m.runLevel(context.Background(), tree, levelOpts{level: LevelUnit, bounds: Bounds{Run: UnitBound}})
|
||||
res, err := m.runLevel(context.Background(), tree, levelOpts{workspace: "ws1", level: LevelUnit, bounds: Bounds{Run: UnitBound}})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
@@ -96,7 +106,7 @@ func processesNamed(text string) []string {
|
||||
// integration level passes, and afterwards no postgres or bus process of the run is left and its directory is gone.
|
||||
// It starts postgres and a bus, so it runs in the merge check and in a try, never on the way.
|
||||
func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing.T) {
|
||||
needTools(t, "the integration level", "go", "git", "initdb", "pg_ctl", "postgres")
|
||||
needTools(t, "the integration level", "go", "git", "initdb", "pg_ctl", "postgres", "bwrap")
|
||||
tree := repository(t, "store")
|
||||
// A short work directory: a unix socket's path is bounded at 108 bytes.
|
||||
work, err := os.MkdirTemp("/tmp", "w")
|
||||
@@ -113,7 +123,7 @@ func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing.
|
||||
home := t.TempDir()
|
||||
m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()}
|
||||
m.Run = StepRunner(home)
|
||||
res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil)
|
||||
res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("the integration level: %v", err)
|
||||
}
|
||||
@@ -134,7 +144,7 @@ func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing.
|
||||
// 1487/AC2: given a fixture Go module, when it is built twice, both digests are equal and the archive holds one
|
||||
// executable named as binaryName says. It runs the real go, so it carries the tag.
|
||||
func TestBuildTwiceGivesOneDigestAndOneExecutable(t *testing.T) {
|
||||
needTools(t, "the build", "go")
|
||||
needTools(t, "the build", "go", "bwrap")
|
||||
work := t.TempDir()
|
||||
tree := filepath.Join(work, "ws1", "tree")
|
||||
if err := os.MkdirAll(tree, 0o755); err != nil {
|
||||
@@ -151,6 +161,7 @@ func TestBuildTwiceGivesOneDigestAndOneExecutable(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
m := &Module{Home: home, Work: work, Cache: t.TempDir()}
|
||||
m.Run = StepRunner(home)
|
||||
withStore(t, m)
|
||||
args := map[string]any{"workspace": "ws1", "module": ".", "artifact": "tools"}
|
||||
first, err := m.build(context.Background(), args)
|
||||
if err != nil {
|
||||
@@ -164,11 +175,7 @@ func TestBuildTwiceGivesOneDigestAndOneExecutable(t *testing.T) {
|
||||
if a.Digest != b.Digest || a.Binary != "prog-bin" {
|
||||
t.Errorf("two builds of one tree: %+v and %+v", a, b)
|
||||
}
|
||||
raw, err := os.ReadFile(a.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||
zr, err := gzip.NewReader(bytes.NewReader(archiveOf(t, a)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ func workspaceFor(t *testing.T, captured string) (*Module, *keptStates, string,
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.27.1")
|
||||
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
|
||||
m.Run = capturedRunner(t, captured, calls)
|
||||
m.Run = throughBubblewrap(t, capturedRunner(t, captured, calls), &[]shutOff{}) // the step's go runs shut off (1309)
|
||||
if err := os.MkdirAll(filepath.Join(m.Work, "ws1", "tree"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -185,12 +185,12 @@ func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t *
|
||||
// A go test that said nothing a test says (it could not run) ends the step failed, not the level.
|
||||
func TestAGoTestThatCouldNotRunEndsTheStepFailed(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "empty")
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if name == "go" {
|
||||
return forgekey.Ran{Status: 1, Stderr: "go: cannot find main module\n"}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "go.mod\n"}
|
||||
}
|
||||
}, &[]shutOff{})
|
||||
_, st := runUnit(t, m, states, ws)
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "go: cannot find main module") {
|
||||
t.Errorf("the step ended %s: %q", st.State, st.Reason)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
package beside
|
||||
@@ -0,0 +1,22 @@
|
||||
package beside
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A test of a repository that reads another one, as the controller's, the node-engine's, the SDK's and the catalogue's do.
|
||||
func TestReadsTheSiblingBesideIt(t *testing.T) {
|
||||
root := os.Getenv("MESH_CHECK_BESIDE")
|
||||
if root == "" {
|
||||
t.Fatal("the sibling is not beside this check: MESH_CHECK_BESIDE is not set")
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(root, "sibling", "data.txt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(raw) != "from the sibling\n" {
|
||||
t.Fatalf("data.txt = %q", raw)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module fixture/beside
|
||||
|
||||
go 1.26
|
||||
@@ -0,0 +1,3 @@
|
||||
module fixture/probe
|
||||
|
||||
go 1.25.0
|
||||
@@ -0,0 +1,34 @@
|
||||
package probe
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestProbe tries what a change's code must not reach, named by PROBE_* (the test that runs it sets them through
|
||||
// the environment of the step), and writes what it saw to seen.json beside it.
|
||||
func TestProbe(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
read := func(p string) bool { _, err := os.ReadFile(p); return err == nil }
|
||||
seen["key"] = read(os.Getenv("PROBE_KEY"))
|
||||
seen["login"] = read(os.Getenv("PROBE_LOGIN"))
|
||||
_, err := net.LookupHost("git.novox.be")
|
||||
seen["resolved"] = err == nil
|
||||
reach := func(network, addr string) bool {
|
||||
c, err := net.DialTimeout(network, addr, 2*time.Second)
|
||||
if err == nil {
|
||||
c.Close()
|
||||
}
|
||||
return err == nil
|
||||
}
|
||||
seen["tcp"] = reach("tcp", os.Getenv("PROBE_TCP"))
|
||||
seen["socket"] = reach("unix", os.Getenv("PROBE_SOCKET"))
|
||||
seen["wroteCache"] = os.WriteFile(os.Getenv("PROBE_CACHE")+"/written-by-a-candidate", []byte("x"), 0o644) == nil
|
||||
raw, _ := json.Marshal(seen)
|
||||
if err := os.WriteFile("seen.json", raw, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,293 @@
|
||||
package main
|
||||
|
||||
// Copied from mesh-controller origin/main internal/builder/xz_framing.go (issue 1590): the build agent's walk of an xz
|
||||
// file's framing before its reader decodes it. "The build seat" below is the build module now.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// xzBlocksMax and xzStreamsMax are the most blocks and the most concatenated streams an xz file may hold for
|
||||
// the build seat to read it.
|
||||
//
|
||||
// **Why a block count is limited at all.** The xz reader (github.com/therootcompany/xz v1.0.1) keeps one
|
||||
// filter decoder for every block that names a delta or BCJ filter, and frees them only when a stream ends:
|
||||
// its block counters are reset per stream, not per block. A stream of many tiny blocks therefore grows the
|
||||
// heap with its length: an `xz --block-size=1 --delta` file of 6 MB kept about 67 MiB live, which comes to
|
||||
// about 5.6 GiB at the 512 MiB download cap (review 68 on mesh-controller pull request 264, hq issue 345).
|
||||
//
|
||||
// **Why 1,024.** At about 340 bytes of decoder a block, 1,024 blocks keep at most about 0.35 MiB live. A
|
||||
// real release is far below it: Betterbird's archive is 1 stream with 1 block, and multi-threaded xz cuts
|
||||
// blocks of three times the dictionary (24 MiB at the default preset), so even 512 MiB unpacked in blocks
|
||||
// of 512 KiB stays within it. Streams are limited to 16 because a release is one stream; the decoders are
|
||||
// reused from one stream to the next, so the stream limit only bounds the work of this walk.
|
||||
const (
|
||||
xzBlocksMax = 1024
|
||||
xzStreamsMax = 16
|
||||
)
|
||||
|
||||
// xzWithinLimits walks an xz file's framing, without decompressing anything, and refuses one holding more
|
||||
// blocks or streams than the build seat reads.
|
||||
//
|
||||
// **The framing, not the index.** Each stream's index at its end lists its blocks, but the reader compares
|
||||
// the index with the blocks only after it has decoded them all, so a file whose index lies would still be
|
||||
// decoded block by block. This walk follows the same framing the reader follows: stream header, each block
|
||||
// header, the block's LZMA2 chunks (each chunk header carries its own compressed size, so a chunk is skipped
|
||||
// without being decoded), its padding and check, then the index and the footer. It counts the blocks the
|
||||
// reader would decode, and it also refuses an index whose count of blocks is not the count walked.
|
||||
func xzWithinLimits(file string) error {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
_, _, err = xzFraming(bufio.NewReader(f))
|
||||
if errors.Is(err, errXzTooManyBlocks) || errors.Is(err, errXzTooManyStreams) {
|
||||
return err
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("it is not an xz file this build seat reads: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
errXzTooManyBlocks = fmt.Errorf("it holds more than %d xz blocks, the most this build seat reads: "+
|
||||
"its xz reader keeps a filter decoder for every block (hq issue 345)", xzBlocksMax)
|
||||
errXzTooManyStreams = fmt.Errorf("it holds more than %d concatenated xz streams, the most this build "+
|
||||
"seat reads", xzStreamsMax)
|
||||
)
|
||||
|
||||
// xzFraming counts the streams and blocks of an xz file, checking its framing as it goes. It stops with
|
||||
// errXzTooManyBlocks or errXzTooManyStreams at the first block or stream over the limit.
|
||||
func xzFraming(r *bufio.Reader) (streams, blocks int, err error) {
|
||||
full := func(n int) ([]byte, error) {
|
||||
b := make([]byte, n)
|
||||
if _, err := io.ReadFull(r, b); err != nil {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
skip := func(n int) error {
|
||||
if got, err := r.Discard(n); err != nil || got != n {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
header, err := full(12)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: header: %v", streams+1, err)
|
||||
}
|
||||
if !bytes.Equal(header[:6], []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no xz stream header", streams+1)
|
||||
}
|
||||
if header[6] != 0 || header[7]&0xf0 != 0 ||
|
||||
crc32.ChecksumIEEE(header[6:8]) != binary.LittleEndian.Uint32(header[8:12]) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: a corrupt stream header", streams+1)
|
||||
}
|
||||
if streams++; streams > xzStreamsMax {
|
||||
return streams, blocks, errXzTooManyStreams
|
||||
}
|
||||
checkSize := 0
|
||||
if check := int(header[7]); check != 0 {
|
||||
checkSize = 4 << ((check - 1) / 3)
|
||||
}
|
||||
inStream := 0
|
||||
for {
|
||||
first, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no index: %v", streams, io.ErrUnexpectedEOF)
|
||||
}
|
||||
if first == 0 {
|
||||
break // the index indicator
|
||||
}
|
||||
rest, err := full(int(first)*4 + 3)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: header: %v", streams, inStream+1, err)
|
||||
}
|
||||
if err := xzBlockHeader(append([]byte{first}, rest...)); err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream+1, err)
|
||||
}
|
||||
inStream++
|
||||
if blocks++; blocks > xzBlocksMax {
|
||||
return streams, blocks, errXzTooManyBlocks
|
||||
}
|
||||
// The block's LZMA2 chunks, up to the end marker.
|
||||
size := 0
|
||||
for {
|
||||
control, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream, io.ErrUnexpectedEOF)
|
||||
}
|
||||
size++
|
||||
if control == 0 {
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case control == 1 || control == 2: // stored: a 16-bit size less one, then the bytes
|
||||
b, err := full(2)
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
n := int(binary.BigEndian.Uint16(b)) + 1
|
||||
if err := skip(n); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size += 2 + n
|
||||
case control >= 0x80: // LZMA: unpacked and packed sizes, the properties byte on a reset
|
||||
b, err := full(4)
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
n := int(binary.BigEndian.Uint16(b[2:])) + 1
|
||||
size += 4
|
||||
if control >= 0xc0 {
|
||||
if err := skip(1); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size++
|
||||
}
|
||||
if err := skip(n); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size += n
|
||||
default:
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: an LZMA2 chunk of kind %#x",
|
||||
streams, inStream, control)
|
||||
}
|
||||
}
|
||||
if pad := (4 - size%4) % 4; pad > 0 {
|
||||
b, err := full(pad)
|
||||
if err != nil || !bytes.Equal(b, make([]byte, pad)) {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: corrupt block padding", streams, inStream)
|
||||
}
|
||||
}
|
||||
if err := skip(checkSize); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
}
|
||||
// The index: its count of records, two numbers each, padding, CRC32.
|
||||
used := 1
|
||||
count, n, err := xzVLI(r)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
used += n
|
||||
if count != uint64(inStream) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: its index lists %d blocks, but it holds %d",
|
||||
streams, count, inStream)
|
||||
}
|
||||
for i := 0; i < 2*inStream; i++ {
|
||||
_, n, err := xzVLI(r)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
used += n
|
||||
}
|
||||
if err := skip((4-used%4)%4 + 4); err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
footer, err := full(12)
|
||||
if err != nil || footer[10] != 'Y' || footer[11] != 'Z' {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no stream footer", streams)
|
||||
}
|
||||
// Stream padding: zero bytes, a multiple of four, then the end or another stream.
|
||||
zeros := 0
|
||||
for {
|
||||
b, err := r.ReadByte()
|
||||
if err == io.EOF {
|
||||
if zeros%4 != 0 {
|
||||
return streams, blocks, errors.New("stream padding not a multiple of four bytes")
|
||||
}
|
||||
return streams, blocks, nil
|
||||
}
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
if b != 0 {
|
||||
_ = r.UnreadByte()
|
||||
break
|
||||
}
|
||||
zeros++
|
||||
}
|
||||
if zeros%4 != 0 {
|
||||
return streams, blocks, errors.New("stream padding not a multiple of four bytes")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// xzBlockHeader checks one block header: its CRC32, its flags, and that its last filter is LZMA2, the only
|
||||
// one whose chunks this walk can skip (and the only last filter the reader decodes).
|
||||
func xzBlockHeader(h []byte) error {
|
||||
size := len(h)
|
||||
if crc32.ChecksumIEEE(h[:size-4]) != binary.LittleEndian.Uint32(h[size-4:]) {
|
||||
return errors.New("a corrupt block header")
|
||||
}
|
||||
flags := h[1]
|
||||
if flags&0x3c != 0 {
|
||||
return errors.New("a block header with reserved flags")
|
||||
}
|
||||
body := bytes.NewReader(h[2 : size-4])
|
||||
if flags&0x40 != 0 {
|
||||
if _, _, err := xzVLI(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if flags&0x80 != 0 {
|
||||
if _, _, err := xzVLI(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var last uint64
|
||||
for i := 0; i <= int(flags&3); i++ {
|
||||
id, _, err := xzVLI(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
props, _, err := xzVLI(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if props > uint64(body.Len()) {
|
||||
return errors.New("a corrupt block header")
|
||||
}
|
||||
if _, err := body.Seek(int64(props), io.SeekCurrent); err != nil {
|
||||
return err
|
||||
}
|
||||
last = id
|
||||
}
|
||||
if last != 0x21 {
|
||||
return fmt.Errorf("a block whose last filter is %#x, not LZMA2", last)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// xzVLI reads one of xz's variable-length integers (at most 9 bytes, 7 bits each), and how many bytes it took.
|
||||
func xzVLI(r io.ByteReader) (uint64, int, error) {
|
||||
var v uint64
|
||||
for i := 0; i < 9; i++ {
|
||||
b, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return 0, i, io.ErrUnexpectedEOF
|
||||
}
|
||||
v |= uint64(b&0x7f) << (7 * i)
|
||||
if b&0x80 == 0 {
|
||||
if i > 0 && b == 0 {
|
||||
return 0, i + 1, errors.New("a number not in its shortest form")
|
||||
}
|
||||
return v, i + 1, nil
|
||||
}
|
||||
}
|
||||
return 0, 9, errors.New("a number longer than nine bytes")
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package main
|
||||
|
||||
// Copied from mesh-controller origin/main internal/builder/xz_framing_test.go (issue 1590), with xz_framing.go: the
|
||||
// build agent's walk of an xz file's framing, which this module now uses as the build agent does.
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// xzStream is one xz stream (no integrity check) of data cut into blocks of blockSize bytes, each block's
|
||||
// LZMA2 data stored uncompressed, each block a delta filter before LZMA2 when delta is set: the shape of
|
||||
// `xz --block-size=N --filters=delta,lzma2`, built without any xz tool. listed is how many blocks its index
|
||||
// lists (-1: as many as it holds), so a stream whose index lies can be built too.
|
||||
func xzStream(data []byte, blockSize int, delta bool, listed int) []byte {
|
||||
var out bytes.Buffer
|
||||
flags := []byte{0, 0}
|
||||
out.Write([]byte{0xfd, '7', 'z', 'X', 'Z', 0x00})
|
||||
out.Write(flags)
|
||||
out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(flags)))
|
||||
type record struct{ unpadded, uncompressed uint64 }
|
||||
var records []record
|
||||
for at := 0; at < len(data); at += blockSize {
|
||||
block := data[at:min(at+blockSize, len(data))]
|
||||
if delta {
|
||||
// Delta-encoded with a distance of one; the decoder's history starts empty at each block.
|
||||
encoded := make([]byte, len(block))
|
||||
for i := range block {
|
||||
encoded[i] = block[i]
|
||||
if i > 0 {
|
||||
encoded[i] -= block[i-1]
|
||||
}
|
||||
}
|
||||
block = encoded
|
||||
}
|
||||
header := []byte{2, 0} // 12 bytes
|
||||
if delta {
|
||||
header[1] = 1 // two filters
|
||||
header = append(header, 0x03, 0x01, 0x00)
|
||||
}
|
||||
header = append(header, 0x21, 0x01, 0x00) // LZMA2, a 4 KiB dictionary
|
||||
for len(header) < 8 {
|
||||
header = append(header, 0)
|
||||
}
|
||||
header = binary.LittleEndian.AppendUint32(header, crc32.ChecksumIEEE(header))
|
||||
out.Write(header)
|
||||
size := 0
|
||||
for i, chunk := 0, block; len(chunk) > 0; i++ {
|
||||
n := min(len(chunk), 1<<16)
|
||||
control := byte(2)
|
||||
if i == 0 {
|
||||
control = 1 // the first chunk of a block resets the dictionary
|
||||
}
|
||||
out.WriteByte(control)
|
||||
out.Write(binary.BigEndian.AppendUint16(nil, uint16(n-1)))
|
||||
out.Write(chunk[:n])
|
||||
size += 3 + n
|
||||
chunk = chunk[n:]
|
||||
}
|
||||
out.WriteByte(0)
|
||||
size++
|
||||
out.Write(make([]byte, (4-size%4)%4))
|
||||
records = append(records, record{uint64(len(header) + size), uint64(len(block))})
|
||||
}
|
||||
if listed >= 0 {
|
||||
for len(records) < listed {
|
||||
records = append(records, records[0])
|
||||
}
|
||||
records = records[:listed]
|
||||
}
|
||||
vli := func(b []byte, v uint64) []byte {
|
||||
for v >= 0x80 {
|
||||
b = append(b, byte(v)|0x80)
|
||||
v >>= 7
|
||||
}
|
||||
return append(b, byte(v))
|
||||
}
|
||||
index := vli([]byte{0}, uint64(len(records)))
|
||||
for _, r := range records {
|
||||
index = vli(vli(index, r.unpadded), r.uncompressed)
|
||||
}
|
||||
for len(index)%4 != 0 {
|
||||
index = append(index, 0)
|
||||
}
|
||||
index = binary.LittleEndian.AppendUint32(index, crc32.ChecksumIEEE(index))
|
||||
out.Write(index)
|
||||
footer := binary.LittleEndian.AppendUint32(nil, uint32(len(index)/4-1))
|
||||
footer = append(footer, flags...)
|
||||
out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(footer)))
|
||||
out.Write(footer)
|
||||
out.WriteString("YZ")
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// tarHolding is a tar holding betterbird/data with body.
|
||||
func tarHolding(t *testing.T, body []byte) []byte {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
w := tar.NewWriter(&out)
|
||||
if err := w.WriteHeader(&tar.Header{Name: "betterbird/data", Mode: 0o644, Size: int64(len(body)),
|
||||
Typeflag: tar.TypeReg}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write(body)
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func writeTemp(t *testing.T, body []byte) string {
|
||||
t.Helper()
|
||||
file := filepath.Join(t.TempDir(), "r.tar.xz")
|
||||
if err := os.WriteFile(file, body, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
func TestAnXzFileOfManyDeltaBlocksIsRefusedBeforeItIsDecoded(t *testing.T) {
|
||||
// Review 68 on pull request 264: `xz --block-size=1 --delta` of a 200 kB tar kept about 67 MiB live in
|
||||
// the reader, one delta decoder per block. Its index here lists one block, as a file crafted to pass a
|
||||
// check of the index alone would; the walk counts the blocks the reader would decode.
|
||||
body := make([]byte, 200_000)
|
||||
for i := range body {
|
||||
body[i] = byte(i * 7)
|
||||
}
|
||||
crafted := xzStream(tarHolding(t, body), 1, true, 1)
|
||||
path := writeTemp(t, crafted)
|
||||
var before, after runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
err := unpackUpstream(path, t.TempDir(), "betterbird")
|
||||
runtime.ReadMemStats(&after)
|
||||
if err == nil || !strings.Contains(err.Error(), "more than 1024 xz blocks") {
|
||||
t.Fatalf("a file of %d bytes in blocks of one byte was not refused for its blocks: %v", len(crafted), err)
|
||||
}
|
||||
if grew := after.TotalAlloc - before.TotalAlloc; grew > 4<<20 {
|
||||
t.Fatalf("%d bytes were allocated for a refused file", grew)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheXzBlockLimitIsExact(t *testing.T) {
|
||||
data := make([]byte, xzBlocksMax+1)
|
||||
streams, blocks, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data[:xzBlocksMax], 1, true, -1))))
|
||||
if err != nil || streams != 1 || blocks != xzBlocksMax {
|
||||
t.Fatalf("%d blocks: %d streams, %d blocks, %v", xzBlocksMax, streams, blocks, err)
|
||||
}
|
||||
if _, _, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data, 1, true, -1)))); !errors.Is(err, errXzTooManyBlocks) {
|
||||
t.Fatalf("%d blocks: %v", xzBlocksMax+1, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzFileOfTooManyStreamsIsRefused(t *testing.T) {
|
||||
var file []byte
|
||||
for i := 0; i <= xzStreamsMax; i++ {
|
||||
file = append(file, xzStream([]byte("x"), 1, false, -1)...)
|
||||
}
|
||||
err := unpackUpstream(writeTemp(t, file), t.TempDir(), "")
|
||||
if err == nil || !strings.Contains(err.Error(), "more than 16 concatenated xz streams") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzIndexThatDoesNotListItsBlocksIsRefused(t *testing.T) {
|
||||
file := xzStream(tarHolding(t, []byte("x")), 512, false, 1)
|
||||
err := unpackUpstream(writeTemp(t, file), t.TempDir(), "betterbird")
|
||||
if err == nil || !strings.Contains(err.Error(), "its index lists 1 blocks, but it holds 4") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// releaseXzSHA256 is the sha256 of testdata/upstream-release.tar.xz: the tar stream of upstream-release.tar.gz,
|
||||
// compressed with `xz -9e` (one stream, one block).
|
||||
const releaseXzSHA256 = "7e59472615d7d654d6f72e8275c83e36203da9db1b91372dcd38a3ff7984f997"
|
||||
|
||||
// pinnedModuleServing is pinnedModule for any fixture file, served at /<its name>.
|
||||
func pinnedModuleServing(t *testing.T, fixture, sha, extra string) (*Module, *keptStates) {
|
||||
t.Helper()
|
||||
body, err := os.ReadFile(filepath.Join("testdata", fixture))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/"+fixture {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Write(body)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
m, states, _ := archiveModule(t, pinned("https://example.com/"+fixture, sha, extra))
|
||||
client := srv.Client()
|
||||
client.Transport.(*http.Transport).DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, network, srv.Listener.Addr().String())
|
||||
}
|
||||
m.HTTP = client
|
||||
return m, states
|
||||
}
|
||||
|
||||
// Issue 1590: a pinned release compressed with xz (as betterbird's is) is unpacked as the build agent unpacks it, and
|
||||
// gives the build agent's archive: the same tar under gzip and under xz packs to one stream.
|
||||
func TestAPinnedXzArchiveIsUnpackedAsTheBuildAgentDoes(t *testing.T) {
|
||||
m, states := pinnedModuleServing(t, "upstream-release.tar.xz", releaseXzSHA256, `,"within":"release/app"`)
|
||||
res, st := runBuild(t, m, states, "files")
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
sameAsBuildAgent(t, res, buildAgentWithinStream, buildAgentWithinDigest)
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
${seat:mesh-artifact-store:reach}
|
||||
@@ -3,19 +3,81 @@ module meshbuildgo
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52
|
||||
github.com/distribution/distribution/v3 v3.1.2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
github.com/therootcompany/xz v1.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/docker-credential-helpers v0.9.5 // indirect
|
||||
github.com/docker/go-events v0.0.0-20250808211157-605354379745 // indirect
|
||||
github.com/docker/go-metrics v0.1.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gorilla/handlers v1.5.2 // indirect
|
||||
github.com/gorilla/mux v1.8.1 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
|
||||
github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/prometheus/client_golang v1.23.2 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/otlptranslator v1.0.0 // indirect
|
||||
github.com/prometheus/procfs v0.20.1 // indirect
|
||||
github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 // indirect
|
||||
github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 // indirect
|
||||
github.com/redis/go-redis/v9 v9.7.3 // indirect
|
||||
github.com/sirupsen/logrus v1.10.2 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/bridges/prometheus v0.67.0 // indirect
|
||||
go.opentelemetry.io/contrib/exporters/autoexport v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.64.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/log v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/log v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.45.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/net v0.59.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
|
||||
google.golang.org/grpc v1.83.2 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
)
|
||||
|
||||
@@ -1,13 +1,67 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 h1:OKxU64kHN1yQ/GOa6LwoUwVmiQ74d83WMSZJRFPVm9c=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52 h1:5k04XjpfYCq6j7rCM6Dy6FfrsBk5iUJlbwGf6WhUOUc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/distribution/distribution/v3 v3.1.2 h1:/Bv2YIqqSR00HiO49FT93hovy6r9IlrzwUDsJjCN5qg=
|
||||
github.com/distribution/distribution/v3 v3.1.2/go.mod h1:Oz0Drs2Tx3iTXzK3DhUD4otfTfqlCxlWVC7QJeXWBP0=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/docker-credential-helpers v0.9.5 h1:EFNN8DHvaiK8zVqFA2DT6BjXE0GzfLOZ38ggPTKePkY=
|
||||
github.com/docker/docker-credential-helpers v0.9.5/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c=
|
||||
github.com/docker/go-events v0.0.0-20250808211157-605354379745 h1:yOn6Ze6IbYI/KAw2lw/83ELYvZh6hvsygTVkD0dzMC4=
|
||||
github.com/docker/go-events v0.0.0-20250808211157-605354379745/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
|
||||
github.com/docker/go-metrics v0.1.0 h1:r76KPNpstz+IvQKSWpYegSkkyzex0V3A1ZGVx6bhGlY=
|
||||
github.com/docker/go-metrics v0.1.0/go.mod h1:PciI3sONtB051kXALN1JoIlpcu54E1FuPh+4DuqEzyw=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
|
||||
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE=
|
||||
github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
|
||||
github.com/hashicorp/golang-lru/arc/v2 v2.0.5 h1:l2zaLDubNhW4XO3LnliVj0GXO3+/CGNJAg1dcN2Fpfw=
|
||||
github.com/hashicorp/golang-lru/arc/v2 v2.0.5/go.mod h1:ny6zBSQZi2JxIeYcv7kt2sH2PXJtirBN7RDhRpxPkxU=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
@@ -18,10 +72,109 @@ github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
|
||||
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
|
||||
github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos=
|
||||
github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM=
|
||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||
github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho=
|
||||
github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U=
|
||||
github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc=
|
||||
github.com/redis/go-redis/extra/redisotel/v9 v9.0.5/go.mod h1:WZjPDy7VNzn77AAfnAfVjZNvfJTYfPetfZk5yoSTLaQ=
|
||||
github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
|
||||
github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM=
|
||||
github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA=
|
||||
github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo=
|
||||
github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
|
||||
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/bridges/prometheus v0.67.0 h1:dkBzNEAIKADEaFnuESzcXvpd09vxvDZsOjx11gjUqLk=
|
||||
go.opentelemetry.io/contrib/bridges/prometheus v0.67.0/go.mod h1:Z5RIwRkZgauOIfnG5IpidvLpERjhTninpP1dTG2jTl4=
|
||||
go.opentelemetry.io/contrib/exporters/autoexport v0.67.0 h1:4fnRcNpc6YFtG3zsFw9achKn3XgmxPxuMuqIL5rE8e8=
|
||||
go.opentelemetry.io/contrib/exporters/autoexport v0.67.0/go.mod h1:qTvIHMFKoxW7HXg02gm6/Wofhq5p3Ib/A/NNt1EoBSQ=
|
||||
go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
|
||||
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0 h1:WseeVYf5dJZTsyPiyW5L14k5qsSibqXAMTSiFEDiWr0=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0/go.mod h1:SiLZnQS6Qk2eCpvr2CH/XMAOa64TWGXxEZJZCpD2Lmc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.21.0 h1:fvNHGyo3CdRv/DQveXqhqBxnKTDyRaC5sMSQxilX/A0=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.21.0/go.mod h1:zyGrjRKL2B/6+Jc/m4/otPoZqV2MY9ZjC/aBraRO7zc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 h1:MdKucPl/HbzckWWEisiNqMPhRrAOQX8r4jTuGr636gk=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0/go.mod h1:RolT8tWtfHcjajEH5wFIZ4Dgh5jpPdFXYV9pTAk/qjc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 h1:w1K+pCJoPpQifuVpsKamUdn9U0zM3xUziVOqsGksUrY=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0/go.mod h1:HBy4BjzgVE8139ieRI75oXm3EcDN+6GhD88JT1Kjvxg=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.64.0 h1:g0LRDXMX/G1SEZtK8zl8Chm4K6GBwRkjPKE36LxiTYs=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.64.0/go.mod h1:UrgcjnarfdlBDP3GjDIJWe6HTprwSazNjwsI+Ru6hro=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.21.0 h1:2lpf4hnrasYIsUyEXwnTZq5lsxrMm4T2Bwb06IctAZQ=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.21.0/go.mod h1:YWOW6h7jwApz9Pl76ie/izUsSPj0s2MdIlpqbPqaf3U=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0 h1:lSZHgNHfbmQTPfuTmWVkEu8J8qXaQwuV30pjCcAUvP8=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0 h1:s/1iRkCKDfhlh1JF26knRneorus8aOwVIDhvYx9WoDw=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0/go.mod h1:UI3wi0FXg1Pofb8ZBiBLhtMzgoTm1TYkMvn71fAqDzs=
|
||||
go.opentelemetry.io/otel/log v0.21.0 h1:SLsVDGmtyBrdw8/a2Z0bOIxou/+bN4z56GebH7T0LvA=
|
||||
go.opentelemetry.io/otel/log v0.21.0/go.mod h1:iReetQrZL9Wyg84cCkOoCmqDHS5RCFfyxC7J+r8fn8g=
|
||||
go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
|
||||
go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
|
||||
go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns=
|
||||
go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI=
|
||||
go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
|
||||
go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
|
||||
go.opentelemetry.io/otel/sdk/log v0.21.0 h1:QsE7XSR0ktQdKmRKGnR+f1ObGF32WG+7MER/P9KgmYc=
|
||||
go.opentelemetry.io/otel/sdk/log v0.21.0/go.mod h1:m9mApjCoD2/1QuKCAptjv+BrG9WKOvQLVdNx+iBldTo=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.21.0 h1:X+JBBgKlswCGYsmgL0CnoUUtlE//VB345c84jYAYkdQ=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.21.0/go.mod h1:HD1575K8e6sIFBBDd5tZB3t9DlMytWXq9FuR+Y4rfjE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
|
||||
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
|
||||
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
|
||||
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
|
||||
@@ -31,7 +31,8 @@
|
||||
],
|
||||
"requires": [
|
||||
"forge-account",
|
||||
"go-toolchain"
|
||||
"go-toolchain",
|
||||
"artifact-store"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
@@ -109,6 +110,14 @@
|
||||
"mode": "0644",
|
||||
"template": "files/gitconfig"
|
||||
},
|
||||
{
|
||||
"id": "registry",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/registry",
|
||||
"owner": "mesh-build-go",
|
||||
"mode": "0644",
|
||||
"template": "files/registry"
|
||||
},
|
||||
{
|
||||
"id": "git",
|
||||
"type": "package",
|
||||
@@ -119,6 +128,11 @@
|
||||
"type": "package",
|
||||
"package": "openssh"
|
||||
},
|
||||
{
|
||||
"id": "bubblewrap",
|
||||
"type": "package",
|
||||
"package": "bubblewrap"
|
||||
},
|
||||
{
|
||||
"id": "postgresql",
|
||||
"type": "package",
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# mesh-build-image — the image build module
|
||||
|
||||
Design 47, release R4c-9 (novox/hq ADR 0487, ADR 0466 §1, ADR 0469; issue 1703). The module that builds a module's
|
||||
`image` artifact from the Dockerfile in the module's own directory, on a build machine, as an account of its own
|
||||
(`runs-as`, no login shell, home at mode 0700). It serves every module's image, whatever the base, so it is not a
|
||||
build module for one module. It claims no seat: it is assigned to the build machines and runs dark until the builder
|
||||
uses it.
|
||||
|
||||
## Rootless
|
||||
|
||||
The tool is `buildah`, which the `buildah` module installs and provides as `rootless-image-build` with the machine's
|
||||
reach: a machine without it refuses this module at assignment, naming `buildah`. The build runs as the module's own
|
||||
account in a user namespace (`--isolation oci`), with buildah's image store and runtime state in
|
||||
`/var/lib/mesh-build-image-store` (outside the home), no mount of the host, no container runtime socket, no registry
|
||||
credential and an environment built from scratch.
|
||||
|
||||
The account's subordinate user and group ranges are not declared anywhere: the node-engine makes every `user` account
|
||||
with `useradd`, which allocates them. The module's health reads `/etc/subuid` and `/etc/subgid` and is unhealthy
|
||||
without a range for the account, without buildah, or when a user namespace cannot be made.
|
||||
|
||||
The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can read holds a key.
|
||||
|
||||
## Steps
|
||||
|
||||
- `build_image_fetch` fetches a commit over SSH into a workspace (a base commit makes the merge preview, as the other
|
||||
build modules' fetch). Long: answers a step id and ends on its key in `steps`.
|
||||
- `build_image_build` builds one image artifact: `buildah bud` with the Dockerfile of the module's directory as file
|
||||
and context, `--target`, and the images `build.on` pins as `--build-arg`. Answers the image's digest. With
|
||||
`publish: true` (a merged trunk commit only) the module's own process then pushes it to the registry under
|
||||
`<module>/<artifact>:<tag of its own>` (the first 12 hex digits of the digest unless the call gives one), waiting out
|
||||
a registry held still, and the key carries the digest the registry serves and the reference pinned by it. Without
|
||||
`publish` (a pull request) nothing is pushed.
|
||||
- It refuses, naming why, an image whose build context is a whole repository (`context`) or whose `build.on` names
|
||||
another module's artifact (`unsupportedImage` in `build.go`): those forms belong to the images build-agent builds
|
||||
until it is removed at R4g.
|
||||
- `build_image_step`, `build_image_steps`, `build_image_status` and `build_image_health` read.
|
||||
|
||||
The registry's address is placed into `<home>/registry` as `${seat:mesh-artifact-store:reach}` (host:port): the provisioner's client fills a moved module's placement, and the artifact store has not moved, so a `${bound:artifact-store:...}` is refused at composition (issue 1703).
|
||||
@@ -0,0 +1,19 @@
|
||||
package main
|
||||
|
||||
// What this module keeps on the bus, announced from its code (novox/hq ADR 0495 §2 §5; issue 1790): every state it
|
||||
// keeps, each readable by others, since another module reads each today (the builder reads steps; gitea reads keys). A state of its own that it
|
||||
// did not name here is refused at the call (stdio.Keeps).
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// Kept are the states the module keeps.
|
||||
var Kept = []announce.State{
|
||||
{Name: "keys", Readable: true},
|
||||
{Name: "steps", Readable: true},
|
||||
}
|
||||
|
||||
// keepStates announces Kept as this module's (stdio.Keeps): main calls it before it serves.
|
||||
func keepStates() error { return stdio.Keeps(Kept...) }
|
||||
@@ -0,0 +1,63 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announcetest"
|
||||
)
|
||||
|
||||
// 1790 (ADR 0495 §5): served as the runtime serves it, the module announces on the bus every state it keeps, as its
|
||||
// code declares them. The runtime's part, publishing what the bundle asks to publish, is played by relay.
|
||||
func TestTheModuleAnnouncesEveryStateItKeeps(t *testing.T) {
|
||||
const module = "mesh-build-image"
|
||||
announcetest.Check(t, module, Kept, func(url string) func() {
|
||||
t.Setenv("MESH_MODULE", module)
|
||||
t.Setenv("MESH_NODE", "shanks")
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := keepStates(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inR, inW := io.Pipe()
|
||||
outR, outW := io.Pipe()
|
||||
go func() { _ = stdio.ServeOn(module, nil, inR, outW) }()
|
||||
go relay(t, module, nc, outR, inW)
|
||||
return func() { _ = inW.Close(); _ = outW.Close(); nc.Close() }
|
||||
})
|
||||
}
|
||||
|
||||
// relay is the runtime's side of the channel: each mesh/publish the bundle asks is published on its module's event
|
||||
// subject, mesh.mod.<module>.event.<key>, and answered.
|
||||
func relay(t *testing.T, module string, nc *nats.Conn, from io.Reader, to io.Writer) {
|
||||
sc := bufio.NewScanner(from)
|
||||
sc.Buffer(make([]byte, 1<<20), 1<<24)
|
||||
enc := json.NewEncoder(to)
|
||||
for sc.Scan() {
|
||||
var m struct {
|
||||
ID json.RawMessage `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params struct {
|
||||
Key string `json:"key"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
} `json:"params"`
|
||||
}
|
||||
if json.Unmarshal(sc.Bytes(), &m) != nil || m.Method != "mesh/publish" {
|
||||
continue
|
||||
}
|
||||
if err := nc.Publish("mesh.mod."+module+".event."+m.Params.Key, m.Params.Body); err != nil {
|
||||
t.Errorf("publishing %s: %v", m.Params.Key, err)
|
||||
}
|
||||
_ = nc.Flush()
|
||||
_ = enc.Encode(map[string]any{"id": m.ID, "result": map[string]any{}})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package main
|
||||
|
||||
// The build step (novox/hq ADR 0487 §1 §2 §5 §6; issue 1703c): one `image` artifact of a module of a fetched workspace,
|
||||
// built from the Dockerfile in the module's own directory with buildah, rootless: as the module's own account, in a
|
||||
// user namespace, with its image store outside the home, no mount of the host, no registry credential and an
|
||||
// environment built from scratch. The step runs the Dockerfile and nothing else; the push of a trunk build is made
|
||||
// after the build has ended (push.go).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// BuildResult is what a passed build step says: the image's manifest digest as buildah holds it, and, for a trunk
|
||||
// build that published, the digest the registry serves and the reference pinned by it.
|
||||
type BuildResult struct {
|
||||
Artifact string `json:"artifact"`
|
||||
Digest string `json:"digest"`
|
||||
Reference string `json:"reference,omitempty"`
|
||||
Pushed bool `json:"pushed"`
|
||||
}
|
||||
|
||||
var tagOK = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`)
|
||||
|
||||
func (m *Module) buildTool() StepTool {
|
||||
return StepTool{
|
||||
Name: BuildTool,
|
||||
Description: "Build one image artifact of a module of a fetched workspace from the Dockerfile in the module's own directory, " +
|
||||
"rootless with buildah. Answers a step id at once; the step ends on its key with the image's digest. With publish " +
|
||||
"true (a merged trunk commit) the image is then pushed to the registry from this module's own process and the key " +
|
||||
"carries the digest the registry serves and the reference pinned by it; without it (a pull request) nothing is pushed.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id build_image_fetch answered"},
|
||||
"module": map[string]any{"type": "string", "description": "the module's directory in the workspace, such as modules/n8n"},
|
||||
"artifact": map[string]any{"type": "string", "description": "the image artifact's name in the module's manifest"},
|
||||
"publish": map[string]any{"type": "boolean", "description": "push the image to the registry after the build: only for a merged trunk commit"},
|
||||
"tag": map[string]any{"type": "string", "description": "the registry tag of a published image; the first 12 hex digits of the digest when left out"},
|
||||
},
|
||||
"required": []string{"workspace", "module", "artifact"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.build,
|
||||
}
|
||||
}
|
||||
|
||||
// unsupportedImage says why this module does not build the artifact, or "" when it does. It is the one place that
|
||||
// knows which image forms are out: ADR 0487 §1 takes the Dockerfile's directory as the build context and the base
|
||||
// from the Dockerfile or an image pinned in `build.on`, so a context that is another repository and a base that is
|
||||
// another module's artifact are not built (the lead's decision for R4c-9, which retires them with build-agent at R4g).
|
||||
func unsupportedImage(manifest catalogue.Manifest, a catalogue.Artifact) string {
|
||||
if a.Context != nil {
|
||||
return fmt.Sprintf("the image %s of %s builds with the repository %s as its context; mesh-build-image takes the Dockerfile's directory as the build context (ADR 0487 §1) and does not build a whole repository",
|
||||
a.Name, manifest.Module, a.Context.Repository)
|
||||
}
|
||||
if manifest.Build != nil {
|
||||
for _, on := range manifest.Build.On {
|
||||
if on.Module != "" {
|
||||
return fmt.Sprintf("the image %s of %s stands on the artifact %s of the module %s through the build argument %s; mesh-build-image builds only from the Dockerfile's directory and an image pinned in build.on (ADR 0487 §1)",
|
||||
a.Name, manifest.Module, on.Artifact, on.Module, on.Arg)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// imageArgs are the --build-arg words of the images the manifest pins in build.on, in its order.
|
||||
func imageArgs(manifest catalogue.Manifest) []string {
|
||||
var out []string
|
||||
if manifest.Build == nil {
|
||||
return out
|
||||
}
|
||||
for _, on := range manifest.Build.On {
|
||||
if on.Image != "" && on.Arg != "" {
|
||||
out = append(out, "--build-arg", on.Arg+"="+on.Image)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
name, _ := args["artifact"].(string)
|
||||
publish, _ := args["publish"].(bool)
|
||||
tag, _ := args["tag"].(string)
|
||||
if !workspaceOK.MatchString(workspace) {
|
||||
return nil, fmt.Errorf("%q is not a workspace id", workspace)
|
||||
}
|
||||
if tag != "" && !tagOK.MatchString(tag) {
|
||||
return nil, fmt.Errorf("%q is not a registry tag", tag)
|
||||
}
|
||||
tree := filepath.Join(m.Work, workspace, "tree")
|
||||
dir := filepath.Join(tree, filepath.FromSlash(module))
|
||||
if rel, err := filepath.Rel(tree, dir); err != nil || strings.HasPrefix(rel, "..") {
|
||||
return nil, fmt.Errorf("the module directory %q lies outside the workspace", module)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no module in %q of workspace %s: %w", module, workspace, err)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the manifest in %q: %w", module, err)
|
||||
}
|
||||
var art *catalogue.Artifact
|
||||
names := []string{}
|
||||
if manifest.Build != nil {
|
||||
for i := range manifest.Build.Artifacts {
|
||||
names = append(names, manifest.Build.Artifacts[i].Name)
|
||||
if manifest.Build.Artifacts[i].Name == name {
|
||||
art = &manifest.Build.Artifacts[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if art == nil {
|
||||
return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", "))
|
||||
}
|
||||
if art.Kind != "image" {
|
||||
return nil, fmt.Errorf("the artifact %s of %s is a %s: this module builds images", name, manifest.Module, art.Kind)
|
||||
}
|
||||
if why := unsupportedImage(manifest, *art); why != "" {
|
||||
return nil, &buildstep.Refusal{Text: why}
|
||||
}
|
||||
dockerfile, contextDir, err := dockerfileOf(dir, art.From)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the image %s of %s: %w", name, manifest.Module, err)
|
||||
}
|
||||
|
||||
local := localName(workspace, manifest.Module, art.Name)
|
||||
argv := append(m.storeFlags(), "bud", "--isolation", "oci", "--tag", local, "--file", dockerfile)
|
||||
if art.Target != "" {
|
||||
argv = append(argv, "--target", art.Target)
|
||||
}
|
||||
argv = append(argv, imageArgs(manifest)...)
|
||||
argv = append(argv, contextDir)
|
||||
if r := m.Run(ctx, m.env(), "buildah", argv...); r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("buildah bud of %s: %s", art.Name, lastLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
defer m.remove(local)
|
||||
digest, err := m.localDigest(ctx, local)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := BuildResult{Artifact: art.Name, Digest: digest}
|
||||
if !publish {
|
||||
return res, nil
|
||||
}
|
||||
if tag == "" {
|
||||
tag = strings.TrimPrefix(digest, "sha256:")[:12]
|
||||
}
|
||||
return m.push(ctx, workspace, manifest.Module, local, res, tag)
|
||||
}
|
||||
|
||||
// dockerfileOf is the Dockerfile an artifact's `from` names and the build context: the directory it lies in. It must
|
||||
// be a regular file inside the module's own directory (checks/imageplace holds the catalogue to the same).
|
||||
func dockerfileOf(moduleDir, from string) (dockerfile, contextDir string, err error) {
|
||||
clean := filepath.Clean(filepath.FromSlash(strings.TrimSpace(from)))
|
||||
if from == "" || filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
|
||||
return "", "", fmt.Errorf("%q is not a Dockerfile in the module's own directory", from)
|
||||
}
|
||||
dockerfile = filepath.Join(moduleDir, clean)
|
||||
info, err := os.Lstat(dockerfile)
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
return "", "", fmt.Errorf("%q is not a file in the module's directory", from)
|
||||
}
|
||||
return dockerfile, filepath.Dir(dockerfile), nil
|
||||
}
|
||||
|
||||
// localName names the image in buildah's store: one name for each artifact of a workspace.
|
||||
func localName(workspace, module, artifact string) string {
|
||||
sum := sha256.Sum256([]byte(module + "/" + artifact))
|
||||
return "mesh-build:" + workspace + "-" + hex.EncodeToString(sum[:])[:8]
|
||||
}
|
||||
|
||||
// localDigest is the manifest digest of the image buildah holds under name.
|
||||
func (m *Module) localDigest(ctx context.Context, name string) (string, error) {
|
||||
r := m.Run(ctx, m.env(), "buildah", append(m.storeFlags(), "images", "--no-trunc", "--format", "{{.Digest}}", name)...)
|
||||
digest := strings.TrimSpace(r.Stdout)
|
||||
if r.Status != 0 || r.Err != "" || !digestOK.MatchString(digest) {
|
||||
return "", fmt.Errorf("the digest of the image %s could not be read: %s", name, lastLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
var digestOK = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||||
|
||||
// remove drops the image from buildah's store, so the store does not grow with every build; a failure is said.
|
||||
func (m *Module) remove(name string) {
|
||||
r := m.Run(m.Ctx, m.env(), "buildah", append(m.storeFlags(), "rmi", "--force", name)...)
|
||||
if (r.Status != 0 || r.Err != "") && m.Say != nil {
|
||||
m.Say("the image %s could not be removed from the store: %s", name, lastLine(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
}
|
||||
|
||||
// lastLine is the last line of a command's output that says something: buildah's error is its last line.
|
||||
func lastLine(s string) string {
|
||||
lines := strings.Split(s, "\n")
|
||||
for i := len(lines) - 1; i >= 0; i-- {
|
||||
if l := strings.TrimSpace(lines[i]); l != "" {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user