Compare commits
236
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7297c972b7 | ||
|
|
653cead338 | ||
|
|
d1ca199265 | ||
|
|
bf94af0a82 | ||
|
|
476a59162f | ||
|
|
36d5993323 | ||
|
|
f8ddcc98f6 | ||
|
|
9109524502 | ||
|
|
9bd8247915 | ||
|
|
116751130f | ||
|
|
c05aa5e6ea | ||
|
|
b95b0a4750 | ||
|
|
04fa2e3286 | ||
|
|
0c61bad102 | ||
|
|
02597b66d7 | ||
|
|
457c60972c | ||
|
|
5ae337b14d | ||
|
|
2ac511ce6b | ||
|
|
e955b53bf0 | ||
|
|
454ac0fec7 | ||
|
|
3c6f2dc274 | ||
|
|
39419b1892 | ||
|
|
b51f03c4ff | ||
|
|
3d42988afa | ||
|
|
64530a8dcb | ||
|
|
e8a953818c | ||
|
|
f0f7b6737e | ||
|
|
c1cd8461bc | ||
|
|
53bf4bb84c | ||
|
|
4677b8a2f5 | ||
|
|
cc6e478c4f | ||
|
|
c17ab34a28 | ||
|
|
099605ec70 | ||
|
|
488b7bfc58 | ||
|
|
d58249d0d8 | ||
|
|
d224b74ff1 | ||
|
|
234d2d5534 | ||
|
|
78d7ef71cc | ||
|
|
1a738ed4fc | ||
|
|
c085975146 | ||
|
|
ba49a7f0f9 | ||
|
|
be7e98d25c | ||
|
|
b1413949f0 | ||
|
|
eba4bab00a | ||
|
|
216701fcba | ||
|
|
8e116cb0e6 | ||
|
|
dcf32576a8 | ||
|
|
3f8b0b4248 | ||
|
|
7c5c29421f | ||
|
|
9e41438f1b | ||
|
|
3690c149c4 | ||
|
|
671fdb7dc2 | ||
|
|
eab6918459 | ||
|
|
ff2632356b | ||
|
|
1e6b157865 | ||
|
|
5b75233406 | ||
|
|
4548b97910 | ||
|
|
1faf317069 | ||
|
|
245143d948 | ||
|
|
358ef45536 | ||
|
|
851e8782d5 | ||
|
|
175b2cd791 | ||
|
|
8530a9730f | ||
|
|
ee48e5a01e | ||
|
|
da29f77b10 | ||
|
|
854dbfb3f3 | ||
|
|
5a4ac49913 | ||
|
|
69a3b3f7e0 | ||
|
|
3cd826830b | ||
|
|
94dab68635 | ||
|
|
76e4ec4dc6 | ||
|
|
c584dc57b9 | ||
|
|
e1e5a8ff0d | ||
|
|
c85472285d | ||
|
|
f0fbe193fa | ||
|
|
26908a8950 | ||
|
|
27a7757b97 | ||
|
|
87fe2c45cf | ||
|
|
24b244bc88 | ||
|
|
5baf8ad1d7 | ||
|
|
8da33e43d4 | ||
|
|
52de4c320e | ||
|
|
5e516317e9 | ||
|
|
6b6d5cba71 | ||
|
|
8393e29621 | ||
|
|
61563abf2f | ||
|
|
071c6885d9 | ||
|
|
8c5f27627b | ||
|
|
82bd6f49ff | ||
|
|
ceb3aac9b3 | ||
|
|
98c66fb1e4 | ||
|
|
07ec186031 | ||
|
|
2aecbe8055 | ||
|
|
3def293d94 | ||
|
|
9a8e54d988 | ||
|
|
2da9329df4 | ||
|
|
ecbee53cfd | ||
|
|
dcdf4238c7 | ||
|
|
d3423f54ed | ||
|
|
0e322d087f | ||
|
|
0300a6fd99 | ||
|
|
d21b81b00a | ||
|
|
b1e610b213 | ||
|
|
0134697f97 | ||
|
|
21583d14ad | ||
|
|
a4f5df797b | ||
|
|
ac2c56f2a9 | ||
|
|
b4a7bb9dea | ||
|
|
64b9ded2db | ||
|
|
67d6a5a7c1 | ||
|
|
5fe5f26e7e | ||
|
|
b3221b6ab8 | ||
|
|
0e7e220d49 | ||
|
|
879600202a | ||
|
|
565d0a848a | ||
|
|
71e094b2de | ||
|
|
f96704fa0f | ||
|
|
86fdf74a0a | ||
|
|
843281c0e5 | ||
|
|
e9279ce3c2 | ||
|
|
c9414b1c98 | ||
|
|
ea53b2f920 | ||
|
|
99cc073e2a | ||
|
|
b94150fcdb | ||
|
|
a01715ddf5 | ||
|
|
e2136ec0c9 | ||
|
|
a30c168512 | ||
|
|
f885250913 | ||
|
|
7627503d9d | ||
|
|
351c7c4bf4 | ||
|
|
55a26b22cf | ||
|
|
100e74f283 | ||
|
|
c8e1e2b2fa | ||
|
|
d0d00f922a | ||
|
|
c2bd7760b2 | ||
|
|
f0fd53281b | ||
|
|
fb039250f6 | ||
|
|
447e6a13cd | ||
|
|
c51a46b802 | ||
|
|
15a5fe09b6 | ||
|
|
3ded9e7800 | ||
|
|
e7aaa8420e | ||
|
|
22381d7522 | ||
|
|
1c55e5d44d | ||
|
|
32b912cfe5 | ||
|
|
e572937d97 | ||
|
|
567736f779 | ||
|
|
e901769cb2 | ||
|
|
8f9c78bcba | ||
|
|
82e7e3e326 | ||
|
|
363a3d3f7e | ||
|
|
609013a263 | ||
|
|
32ad59fc64 | ||
|
|
93f655dac9 | ||
|
|
573a59e3cc | ||
|
|
d0566f9ade | ||
|
|
9d9dfda19b | ||
|
|
cdbe2945cf | ||
|
|
4e82a824be | ||
|
|
78d26867d2 | ||
|
|
0c216d99d0 | ||
|
|
145c54d19c | ||
|
|
67092161bd | ||
|
|
2df0503034 | ||
|
|
23c827e791 | ||
|
|
3138570c6b | ||
|
|
a85d903741 | ||
|
|
1cc54959af | ||
|
|
962bc359a7 | ||
|
|
67b13c42d3 | ||
|
|
4021724ca4 | ||
|
|
f27f4c8cd5 | ||
|
|
155124b59a | ||
|
|
6ec6fdaad2 | ||
|
|
663f3de975 | ||
|
|
68f7431d98 | ||
|
|
53749833bf | ||
|
|
2167b95a36 | ||
|
|
d09cbf1523 | ||
|
|
ac6712cbb5 | ||
|
|
5b32342ac2 | ||
|
|
6fb0ceeef8 | ||
|
|
c651161787 | ||
|
|
ee216e4e63 | ||
|
|
b67b5af5e6 | ||
|
|
d700ceca20 | ||
|
|
3a1017f559 | ||
|
|
a17d0c8e93 | ||
|
|
9fa6710296 | ||
|
|
05f849979c | ||
|
|
4dee77df23 | ||
|
|
8a79877a15 | ||
|
|
2139e83a8a | ||
|
|
e7ca8f7af1 | ||
|
|
eb79996336 | ||
|
|
1a271a1fb1 | ||
|
|
30d04f6bef | ||
|
|
5ff337e0dc | ||
|
|
2fabbe556e | ||
|
|
89b9212c4d | ||
|
|
40da368ee3 | ||
|
|
865f3b6455 | ||
|
|
03524ecd13 | ||
|
|
15101c9ce6 | ||
|
|
f910b0ceb8 | ||
|
|
91c3e954ae | ||
|
|
545c72bb59 | ||
|
|
7afcd0282d | ||
|
|
75156fce00 | ||
|
|
16d7b1898d | ||
|
|
f04465537b | ||
|
|
e870a7f65f | ||
|
|
e8a59171c7 | ||
|
|
ebb25757a2 | ||
|
|
6a70735489 | ||
|
|
a715c8821f | ||
|
|
b6ed8599c8 | ||
|
|
ae6f7d5b54 | ||
|
|
25176e1ccd | ||
|
|
7e1f3de77a | ||
|
|
4b1bed4b9d | ||
|
|
25d69406b7 | ||
|
|
f836c021e1 | ||
|
|
450f0bbe79 | ||
|
|
5a7076a6a2 | ||
|
|
30b22c198e | ||
|
|
753e8fc202 | ||
|
|
e5202445ba | ||
|
|
dcae699323 | ||
|
|
70ef99595a | ||
|
|
746eb885a3 | ||
|
|
2cdb9b8fc4 | ||
|
|
e830045545 | ||
|
|
454fb8a918 | ||
|
|
c32474ee40 | ||
|
|
f95f3f430e |
+12
@@ -14,6 +14,7 @@ modules/claude-code/claude-code
|
||||
modules/mesh-watcher/mesh-watcher
|
||||
modules/dnsmasq/dnsmasq
|
||||
modules/mesh-issues/cmd/mesh-issues/mesh-issues
|
||||
modules/mesh-issues/mesh-issues
|
||||
modules/mounts/cmd/mounts/mounts
|
||||
modules/mesh-knowledge/cmd/mesh-knowledge/mesh-knowledge
|
||||
modules/mesh-knowledge/mesh-knowledge
|
||||
@@ -29,3 +30,14 @@ modules/mesh-build-go/cmd/mesh-build-go/mesh-build-go
|
||||
modules/mesh-config/mesh-config
|
||||
modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image
|
||||
*.test
|
||||
|
||||
# binaries a local go build leaves in a module root
|
||||
/checks/paths/paths
|
||||
/modules/mesh-build-go/mesh-build-go
|
||||
/modules/mesh-delivery-gate/mesh-delivery-gate
|
||||
modules/mesh-config/cmd/mesh-config/mesh-config
|
||||
checks/placement/placement
|
||||
|
||||
# Built by their modules' build step (issue 1577), never committed.
|
||||
modules/forticlient/cmd/forticlient-tools/forticlient-tools
|
||||
modules/networkmanager/uplink-tools
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest.
|
||||
modules/*/module.json
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest and the files it names.
|
||||
modules
|
||||
@@ -0,0 +1,128 @@
|
||||
// Package failurekit refuses a build module whose tests do not run mesh-sdk's conformance kit (issue 1901, its thorough
|
||||
// fix): a module that claims a node-build-* seat and serves a test step must hold a test that calls conformance.Run, so
|
||||
// its parse of the test runner's output is held to the failure contract (a failing test named with its cause, never a
|
||||
// passing line in its place). A build module with no test step (mesh-build-image builds images and runs no tests) is
|
||||
// exempt from that, and said so. Every build module, test step or not, must also hold a test that calls
|
||||
// conformance.LeftRunning with its own start sweep, so a step a previous process left running is ended at its start
|
||||
// (issue 1890).
|
||||
package failurekit
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Judged is what the check says of one build module: Refused says why it fails the check, "" when it passes; Exempt
|
||||
// says why the kit does not apply.
|
||||
type Judged struct {
|
||||
Module, Refused, Exempt string
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"claims"`
|
||||
Tools []json.RawMessage `json:"tools"`
|
||||
}
|
||||
|
||||
// Judge reads every modules/<m>/module.json of the catalogue at root and judges each module that claims a node-build-*
|
||||
// seat.
|
||||
func Judge(root string) ([]Judged, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Judged
|
||||
for _, p := range manifests {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", p, err)
|
||||
}
|
||||
seat := ""
|
||||
for _, c := range m.Claims {
|
||||
if strings.HasPrefix(c.Name, "node-build-") {
|
||||
seat = c.Name
|
||||
}
|
||||
}
|
||||
if seat == "" {
|
||||
continue
|
||||
}
|
||||
dir := filepath.Dir(p)
|
||||
j := Judged{Module: filepath.Base(dir)}
|
||||
var refused []string
|
||||
test := testTool(m.Tools)
|
||||
switch {
|
||||
case test == "":
|
||||
j.Exempt = fmt.Sprintf("it claims %s and serves no test step, so the failure contract's cases do not apply", seat)
|
||||
default:
|
||||
runs, err := runsTheKit(dir, "conformance.Run(")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !runs {
|
||||
refused = append(refused, fmt.Sprintf("it claims %s and serves the test step %s, and no test of it calls conformance.Run "+
|
||||
"(mesh-sdk go/buildstep/conformance): run the kit with the module's own parse path (issue 1901)", seat, test))
|
||||
}
|
||||
}
|
||||
// Every build module keeps steps, so every one runs the left-running case, test step or not (issue 1890).
|
||||
sweeps, err := runsTheKit(dir, "conformance.LeftRunning(")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !sweeps {
|
||||
refused = append(refused, fmt.Sprintf("it claims %s, and no test of it calls conformance.LeftRunning "+
|
||||
"(mesh-sdk go/buildstep/conformance): run the kit's left-running case with the module's own start sweep (issue 1890)", seat))
|
||||
}
|
||||
j.Refused = strings.Join(refused, "; ")
|
||||
out = append(out, j)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// testTool is the name of the module's test step tool (<prefix>_test), or "".
|
||||
func testTool(tools []json.RawMessage) string {
|
||||
for _, raw := range tools {
|
||||
var name string
|
||||
if json.Unmarshal(raw, &name) != nil {
|
||||
var t struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if json.Unmarshal(raw, &t) != nil {
|
||||
continue
|
||||
}
|
||||
name = t.Name
|
||||
}
|
||||
if strings.HasSuffix(name, "_test") {
|
||||
return name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// runsTheKit says a _test.go file under dir calls the kit's function call (such as "conformance.Run(").
|
||||
func runsTheKit(dir, call string) (bool, error) {
|
||||
found := false
|
||||
err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil || found || d.IsDir() || !strings.HasSuffix(p, "_test.go") {
|
||||
return err
|
||||
}
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found = strings.Contains(string(raw), call) &&
|
||||
strings.Contains(string(raw), `"git.novox.be/novox/mesh-sdk/go/buildstep/conformance"`)
|
||||
return nil
|
||||
})
|
||||
return found, err
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package failurekit
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1901/AC7 and 1890/AC8: a build module with a test step and no kit is refused, one that runs the kit passes, one with no
|
||||
// test step is exempt from the failure cases but not from the left-running case, a module that runs only the failure
|
||||
// cases is refused, and a module that claims no node-build-* seat is not judged.
|
||||
func TestABuildModuleWithATestStepMustRunTheKit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
const imp = "package main\n\nimport \"git.novox.be/novox/mesh-sdk/go/buildstep/conformance\"\n\n"
|
||||
const sweep = "func TestSweep(t *testing.T) { conformance.LeftRunning(t, nil) }\n"
|
||||
kit := imp + "func TestKit(t *testing.T) { conformance.Run(t, conformance.GoTest, nil) }\n" + sweep
|
||||
for path, body := range map[string]string{
|
||||
"modules/a/module.json": `{"claims":[{"name":"node-build-a"}],"tools":["build_a_fetch","build_a_test"]}`,
|
||||
"modules/a/cmd/a/check_test.go": "package main\n",
|
||||
"modules/b/module.json": `{"claims":[{"name":"node-build-b"}],"tools":[{"name":"build_b_test"}]}`,
|
||||
"modules/b/cmd/b/kit_test.go": kit,
|
||||
"modules/c/module.json": `{"claims":[{"name":"node-build-c"}],"tools":["build_c_build"]}`,
|
||||
"modules/c/cmd/c/kit_test.go": imp + sweep,
|
||||
"modules/f/module.json": `{"claims":[{"name":"node-build-f"}],"tools":["build_f_build"]}`,
|
||||
"modules/g/module.json": `{"claims":[{"name":"node-build-g"}],"tools":["build_g_test"]}`,
|
||||
"modules/g/cmd/g/kit_test.go": imp + "func TestKit(t *testing.T) { conformance.Run(t, conformance.GoTest, nil) }\n",
|
||||
"modules/d/module.json": `{"claims":[{"name":"other"}],"tools":["d_test"]}`,
|
||||
"modules/e/module.json": `{"claims":[{"name":"node-build-e"}],"tools":["build_e_test"]}`,
|
||||
"modules/e/cmd/e/kit.go": kit, // not a test file
|
||||
} {
|
||||
p := filepath.Join(root, path)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := Judge(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{"a": "refused", "b": "passes", "c": "exempt", "e": "refused", "f": "refused", "g": "refused"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("judged %+v, want a, b, c, e, f and g", got)
|
||||
}
|
||||
for _, j := range got {
|
||||
is := "passes"
|
||||
if j.Refused != "" {
|
||||
is = "refused"
|
||||
} else if j.Exempt != "" {
|
||||
is = "exempt"
|
||||
}
|
||||
if is != want[j.Module] {
|
||||
t.Errorf("%s %s (%+v), want it %s", j.Module, is, j, want[j.Module])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every build module of this catalogue with a test step runs the kit.
|
||||
func TestTheCataloguesBuildModulesRunTheKit(t *testing.T) {
|
||||
got, err := Judge(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) == 0 {
|
||||
t.Fatal("no module claims a node-build-* seat: the check reads the wrong tree")
|
||||
}
|
||||
for _, j := range got {
|
||||
switch {
|
||||
case j.Refused != "":
|
||||
t.Errorf("%s: %s", j.Module, j.Refused)
|
||||
case j.Exempt != "":
|
||||
t.Logf("%s is exempt: %s", j.Module, j.Exempt)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/failurekit
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,7 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Judge("../.."): every manifest, and the directory of each module claiming a node-build-* seat, walked for a test that
|
||||
# calls the kit. The build modules are the mesh-build-* directories; a build module named otherwise needs its line here.
|
||||
modules/*/module.json
|
||||
modules/mesh-build-*
|
||||
@@ -0,0 +1,10 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# every manifest through mesh-sdk's ReadTreeManifest: the root's predefined keys and each templated module's
|
||||
# templates (files/ by default, or the directory its manifest names); and the root's kept count of resources that do not say how they are ready.
|
||||
modules/*/module.json
|
||||
predefined.json
|
||||
modules/*/files
|
||||
modules/*/templates
|
||||
health-undeclared
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest.
|
||||
modules/*/module.json
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest, and whether the path each image artifact builds from is there.
|
||||
modules
|
||||
@@ -0,0 +1,7 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest, a TypeScript module's package.json and whether its lock is there.
|
||||
modules/*/module.json
|
||||
modules/*/package.json
|
||||
modules/*/package-lock.json
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest.
|
||||
modules/*/module.json
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/nobinaries
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,78 @@
|
||||
// Package nobinaries refuses a compiled program under modules/ (issue 1577): it is build output, made by the module's build step,
|
||||
// and `go build` in a module's folder writes one beside go.mod where `git add -A` takes it.
|
||||
package nobinaries
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Kind says what executable format the first bytes of a file are, or "" when they are none.
|
||||
func Kind(head []byte) string {
|
||||
switch {
|
||||
case bytes.HasPrefix(head, []byte("\x7fELF")):
|
||||
return "ELF"
|
||||
case bytes.HasPrefix(head, []byte("MZ")):
|
||||
return "PE"
|
||||
case bytes.HasPrefix(head, []byte{0xcf, 0xfa, 0xed, 0xfe}), bytes.HasPrefix(head, []byte{0xce, 0xfa, 0xed, 0xfe}),
|
||||
bytes.HasPrefix(head, []byte{0xfe, 0xed, 0xfa, 0xcf}), bytes.HasPrefix(head, []byte{0xfe, 0xed, 0xfa, 0xce}),
|
||||
bytes.HasPrefix(head, []byte{0xca, 0xfe, 0xba, 0xbe}) && len(head) >= 8 && head[7] < 0x20 && head[6] == 0 && head[5] == 0 && head[4] == 0:
|
||||
return "Mach-O"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Files are the files of the catalogue's root that the change holds under modules/: the tracked ones when the root is a git
|
||||
// checkout (an ignored binary a local build left is not the change's), else every file there.
|
||||
func Files(root string) ([]string, error) {
|
||||
cmd := exec.Command("git", "-C", root, "ls-files", "-z", "--", "modules")
|
||||
if out, err := cmd.Output(); err == nil && len(out) > 0 {
|
||||
var files []string
|
||||
for _, f := range strings.Split(strings.TrimRight(string(out), "\x00"), "\x00") {
|
||||
if _, err := os.Lstat(filepath.Join(root, f)); err == nil { // a file the change deletes is not there
|
||||
files = append(files, f)
|
||||
}
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
var files []string
|
||||
err := filepath.WalkDir(filepath.Join(root, "modules"), func(p string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || !d.Type().IsRegular() {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
files = append(files, rel)
|
||||
return nil
|
||||
})
|
||||
return files, err
|
||||
}
|
||||
|
||||
// Find names each file of files (paths from root) that is a compiled program, with its format.
|
||||
func Find(root string, files []string) ([]string, error) {
|
||||
var found []string
|
||||
for _, f := range files {
|
||||
p := filepath.Join(root, f)
|
||||
if st, err := os.Lstat(p); err != nil || !st.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
fh, err := os.Open(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
head := make([]byte, 8)
|
||||
n, err := io.ReadFull(fh, head)
|
||||
fh.Close()
|
||||
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
if k := Kind(head[:n]); k != "" {
|
||||
found = append(found, fmt.Sprintf("%s (%s)", f, k))
|
||||
}
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package nobinaries
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFindNamesEachCompiledProgramAndNothingElse(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
files := map[string][]byte{
|
||||
"modules/a/a": []byte("\x7fELF\x02\x01\x01\x00rest"),
|
||||
"modules/b/cmd/b/b.exe": []byte("MZ\x90\x00"),
|
||||
"modules/c/c": {0xcf, 0xfa, 0xed, 0xfe, 7, 0, 0, 1},
|
||||
"modules/d/main.go": []byte("package main\n"),
|
||||
"modules/e/short": []byte("\x7f"),
|
||||
"modules/f/empty": {},
|
||||
"modules/g/Foo.class": {0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 52}, // Java, not Mach-O
|
||||
}
|
||||
var names []string
|
||||
for f, b := range files {
|
||||
p := filepath.Join(root, f)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, b, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names = append(names, f)
|
||||
}
|
||||
got, err := Find(root, names)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("found %v; wanted modules/a/a, modules/b/cmd/b/b.exe and modules/c/c only", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's own tree holds no compiled program: a compiled program is build output, so remove it and add it to .gitignore.
|
||||
func TestTheCatalogueHoldsNoCompiledProgram(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
files, err := Files(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found, err := Find(root, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range found {
|
||||
t.Errorf("%s: a compiled program is build output: remove it and add it to .gitignore (issue 1577)", f)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# every file under modules/ (git ls-files and a walk).
|
||||
modules
|
||||
@@ -0,0 +1,7 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# that none of the old check scripts stands at the root.
|
||||
?merge-check.sh
|
||||
?merge-check-typescript.sh
|
||||
?check-nested-modules.sh
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest; its lists are this package's own files.
|
||||
modules/*/module.json
|
||||
@@ -0,0 +1,6 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# check("../.."): the root's list and every file of the modules it names.
|
||||
particulars
|
||||
modules
|
||||
@@ -251,7 +251,8 @@ func main() {
|
||||
}
|
||||
var grown []string
|
||||
if baseDir == "" {
|
||||
fmt.Println("NOT COMPARED: no base branch's lists were given, so additions to them were not judged")
|
||||
fmt.Fprintln(os.Stderr, "NOT COMPARED: no base branch's lists were given (-base <dir>), so additions to them were not judged; that is a failure, not a pass (issue 1295)")
|
||||
os.Exit(2)
|
||||
} else {
|
||||
baseShared, _, err1 := list(filepath.Join(baseDir, "shared"))
|
||||
baseKnown, _, err2 := list(filepath.Join(baseDir, "known"))
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest and every file of a module.
|
||||
modules
|
||||
@@ -32,8 +32,8 @@ func TestNoModuleNamesAnothersState(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// And neither list gains a line the trunk's copies at the merge base do not hold, unless the line says why after `# allow:`. Skipped,
|
||||
// and said, where no trunk is beside.
|
||||
// And neither list gains a line the trunk's copies at the merge base do not hold, unless the line says why after `# allow:`. Fails, saying
|
||||
// NOT COMPARED, where no trunk is beside (issue 1295).
|
||||
func TestTheListsOfPathsHoldNoLineTheTrunksDoNot(t *testing.T) {
|
||||
sharedPath, okShared := trunk.Copy(t, "checks/paths/shared")
|
||||
knownPath, okKnown := trunk.Copy(t, "checks/paths/known")
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
module checks/placement
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f
|
||||
@@ -0,0 +1,18 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f h1:AKzV973UTWe0OYFoIjgGu0f+B26xh8QsAHzCpsTIjM8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f h1:mimAGERguQmCGYVNdRTrFcZ1dhpU6GWo+Q3EDhHOrVk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
@@ -0,0 +1,80 @@
|
||||
// Command placement is the catalogue's check of its placement files (novox/hq ADR 0549 §1, ADR 0555; issue 1126): every
|
||||
// `placement/<node>.json` reads as a placement of the node its name says, and the files together are refused, every
|
||||
// finding named at once, for two modules of one node on one host port, a pin, binding or holding naming a provider not
|
||||
// placed where it says, a seat held once for the mesh held twice, two nodes with one address on the private network, or
|
||||
// more than one hub (mesh-sdk delivery.CheckPlacements).
|
||||
//
|
||||
// A module the catalogue does not hold and a node not enrolled are the composition check's to refuse: placement names
|
||||
// modules of every repository the indexer reads and the nodes the mesh enrolled, neither of which this tree holds.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Check reads every placement file under root and answers each finding; none is a catalogue whose placement holds. A
|
||||
// catalogue with no placement directory has none to check.
|
||||
func Check(root string) ([]string, error) {
|
||||
dir := filepath.Join(root, delivery.PlacementDir)
|
||||
entries, err := os.ReadDir(dir)
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var found []string
|
||||
placements := map[string]delivery.Placement{}
|
||||
for _, e := range entries {
|
||||
rel := delivery.PlacementDir + "/" + e.Name()
|
||||
node, ok := delivery.PlacementFileNode(rel)
|
||||
if e.IsDir() || !ok {
|
||||
found = append(found, fmt.Sprintf("%s is not a placement file: the directory holds only <node>.json, one per node", rel))
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := delivery.DecodePlacement(raw)
|
||||
if err != nil {
|
||||
found = append(found, fmt.Sprintf("%s does not read as a placement: %v", rel, err))
|
||||
continue
|
||||
}
|
||||
if p.Revision != "" {
|
||||
found = append(found, fmt.Sprintf("%s names a revision, which only the indexer writes into the key", rel))
|
||||
}
|
||||
placements[node] = p
|
||||
}
|
||||
all := func(string) bool { return true }
|
||||
found = append(found, delivery.CheckPlacements(placements, all, all)...)
|
||||
sort.Strings(found)
|
||||
return found, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: placement <catalogue root>")
|
||||
os.Exit(2)
|
||||
}
|
||||
found, err := Check(os.Args[1])
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Println(f)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The catalogue's placement files as the merge check reads them (ADR 0549 §1, ADR 0555): the real tree holds every
|
||||
// rule, and a tree that breaks them is refused with every finding named at once.
|
||||
func TestPlacementFiles(t *testing.T) {
|
||||
found, err := Check("../..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range found {
|
||||
t.Errorf("%s (novox/hq ADR 0549 §1)", f)
|
||||
}
|
||||
|
||||
root := t.TempDir()
|
||||
write := func(name, body string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Join(root, "placement"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "placement", name), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if found, err := Check(root); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a catalogue with no placement directory: %v, %v", found, err)
|
||||
}
|
||||
write("ace.json", `{"v":1,"node":"ace","decided":{"hub":true,"address":"10.77.0.1"},"modules":[{"module":"a","ports":{"80":8080}},{"module":"b","ports":{"81":8080}}]}`)
|
||||
write("g14.json", `{"v":1,"node":"g14","decided":{"hub":true,"address":"10.77.0.1"},"modules":[{"module":"c","bindings":{"pg":{"node":"ace","module":"pg"}}}]}`)
|
||||
write("novox.json", `{"v":1,"node":"shanks","modules":[]}`)
|
||||
write("bad.json", `{"v":1,"node":"bad","modules":{}}`)
|
||||
write("dated.json", `{"v":1,"node":"dated","revision":"abc","modules":[]}`)
|
||||
write("notes.md", "x")
|
||||
found, err = Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"a and b are both published on host port 8080", "bound to ace/pg", "both have the address 10.77.0.1",
|
||||
"ace, g14 are each placed as it", "placement file of novox names the node", "bad.json does not read", "dated.json names a revision",
|
||||
"notes.md is not a placement file"} {
|
||||
hit := false
|
||||
for _, f := range found {
|
||||
hit = hit || strings.Contains(f, want)
|
||||
}
|
||||
if !hit {
|
||||
t.Errorf("no finding says %q; found %v", want, found)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1,6 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# the root's list of predefined keys and every manifest that names one.
|
||||
predefined.json
|
||||
modules/*/module.json
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest and the files it names.
|
||||
modules
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest.
|
||||
modules/*/module.json
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest and its templates.
|
||||
modules
|
||||
@@ -0,0 +1,5 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# Check("../.."): every manifest.
|
||||
modules/*/module.json
|
||||
+14
-18
@@ -3,14 +3,14 @@
|
||||
//
|
||||
// The checker fetches the change's own repository at its base commit beside the change, as it fetches a sibling, and says where
|
||||
// in MESH_CHECK_BESIDE; the copy is under the repository's name (the same convention the other repositories' tests use). With
|
||||
// MESH_CHECK_BESIDE unset, or set without the catalogue's copy in it, the comparison is skipped and the test says NOT COMPARED
|
||||
// WITH THE TRUNK, naming why: the checker that runs before R4g's delivery places siblings but not the change's own repository,
|
||||
// so R4g's own check cannot hold the copy (issue 1295 notes that a missing copy becomes a failure once that checker is gone). A trunk that holds no such file had
|
||||
// MESH_CHECK_BESIDE unset, or set without the catalogue's copy in it, the test fails saying NOT COMPARED WITH THE TRUNK and
|
||||
// why: the checker places the change's own repository since R4g (issue 1295), so a missing copy is a fault, never a pass. A trunk that holds no such file had
|
||||
// none: the list's first copy, which nothing was added to.
|
||||
package trunk
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
@@ -23,13 +23,11 @@ const Repository = "mesh-catalog"
|
||||
const BesideEnv = "MESH_CHECK_BESIDE"
|
||||
|
||||
// Copy is the trunk's copy of the file at rel (a path from the catalogue's root), written to a file of the test's own: its path,
|
||||
// and whether the trunk holds one (false: its first copy). It skips the test, saying so, when no copy of the catalogue is beside.
|
||||
// and whether the trunk holds one (false: its first copy). It fails the test when no copy of the catalogue is beside.
|
||||
func Copy(t testing.TB, rel string) (string, bool) {
|
||||
t.Helper()
|
||||
raw, found, skip, err := Read(rel)
|
||||
raw, found, err := Read(rel)
|
||||
switch {
|
||||
case skip != "":
|
||||
t.Skip(skip)
|
||||
case err != nil:
|
||||
t.Fatal(err)
|
||||
case !found:
|
||||
@@ -43,27 +41,25 @@ func Copy(t testing.TB, rel string) (string, bool) {
|
||||
return path, true
|
||||
}
|
||||
|
||||
// Read is the trunk's copy of rel and whether the trunk holds one. skip is the words to skip the comparison with, when no trunk is
|
||||
// named; err is a trunk named and not there, or unreadable.
|
||||
func Read(rel string) (content []byte, found bool, skip string, err error) {
|
||||
// Read is the trunk's copy of rel and whether the trunk holds one. err is a trunk not beside the check (MESH_CHECK_BESIDE unset,
|
||||
// or without the catalogue's directory), or a copy unreadable: a comparison that cannot be made is never a pass.
|
||||
func Read(rel string) (content []byte, found bool, err error) {
|
||||
beside := os.Getenv(BesideEnv)
|
||||
if beside == "" {
|
||||
return nil, false, "NOT COMPARED WITH THE TRUNK: " + BesideEnv + " is not set, so the trunk's copy of " + rel +
|
||||
" is not beside this run; the checker places it", nil
|
||||
return nil, false, fmt.Errorf("NOT COMPARED WITH THE TRUNK: %s is not set, so the trunk's copy of %s is not beside this run; the checker places the change's own repository there (issue 1295)", BesideEnv, rel)
|
||||
}
|
||||
dir := filepath.Join(beside, Repository)
|
||||
if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) {
|
||||
return nil, false, "NOT COMPARED WITH THE TRUNK: " + Repository + " is not beside this check in " + BesideEnv + "=" + beside +
|
||||
", so " + rel + " is not judged against the trunk's copy (the checker before R4g places no copy of the change's own repository)", nil
|
||||
return nil, false, fmt.Errorf("NOT COMPARED WITH THE TRUNK: %s is not beside this check in %s=%s, so %s is not judged against the trunk's copy; the checker places the change's own repository there at its trunk (issue 1295)", Repository, BesideEnv, beside, rel)
|
||||
} else if err != nil {
|
||||
return nil, false, "", err
|
||||
return nil, false, err
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, rel))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, false, "", nil
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, "", err
|
||||
return nil, false, err
|
||||
}
|
||||
return raw, true, "", nil
|
||||
return raw, true, nil
|
||||
}
|
||||
|
||||
+14
-14
@@ -17,27 +17,27 @@ func TestTheCloneBesideTheCheckIsTheTrunk(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(BesideEnv, beside)
|
||||
raw, found, skip, err := Read("checks/x/known")
|
||||
if err != nil || skip != "" || !found || string(raw) != "a\nb\n" {
|
||||
t.Fatalf("read %q found %v skip %q err %v", raw, found, skip, err)
|
||||
raw, found, err := Read("checks/x/known")
|
||||
if err != nil || !found || string(raw) != "a\nb\n" {
|
||||
t.Fatalf("read %q found %v err %v", raw, found, err)
|
||||
}
|
||||
if _, found, skip, err := Read("checks/x/absent"); err != nil || skip != "" || found {
|
||||
t.Fatalf("a file the trunk does not hold: found %v skip %q err %v; wanted its first copy", found, skip, err)
|
||||
if _, found, err := Read("checks/x/absent"); err != nil || found {
|
||||
t.Fatalf("a file the trunk does not hold: found %v err %v; wanted its first copy", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithNothingBesideTheComparisonIsSkippedAndSaidNotMade(t *testing.T) {
|
||||
func TestWithNothingBesideTheComparisonFailsAndSaysNotMade(t *testing.T) {
|
||||
t.Setenv(BesideEnv, "")
|
||||
_, found, skip, err := Read("checks/x/known")
|
||||
if found || err != nil || len(skip) < 12 || skip[:12] != "NOT COMPARED" {
|
||||
t.Fatalf("found %v, skip %q, err %v; wanted NOT COMPARED", found, skip, err)
|
||||
_, found, err := Read("checks/x/known")
|
||||
if found || err == nil || !strings.Contains(err.Error(), "NOT COMPARED") {
|
||||
t.Fatalf("found %v, err %v; wanted a failure saying NOT COMPARED", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACloneBesideWithoutTheCatalogueIsSkippedAndSaidNotMade(t *testing.T) {
|
||||
t.Setenv(BesideEnv, t.TempDir()) // the siblings are there, the catalogue's own base is not (the checker before R4g)
|
||||
_, found, skip, err := Read("checks/x/known")
|
||||
if found || err != nil || len(skip) < 12 || skip[:12] != "NOT COMPARED" || !strings.Contains(skip, "not beside") {
|
||||
t.Fatalf("found %v, skip %q, err %v; wanted NOT COMPARED naming the missing copy", found, skip, err)
|
||||
func TestACloneBesideWithoutTheCatalogueFailsAndSaysNotMade(t *testing.T) {
|
||||
t.Setenv(BesideEnv, t.TempDir()) // the siblings are there, the catalogue's own base is not
|
||||
_, found, err := Read("checks/x/known")
|
||||
if found || err == nil || !strings.Contains(err.Error(), "NOT COMPARED") || !strings.Contains(err.Error(), "not beside") {
|
||||
t.Fatalf("found %v, err %v; wanted a failure naming the missing copy", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# run("../.."): the root's lists of retired words and lines, and every module's texts.
|
||||
retired-words
|
||||
retired-lines
|
||||
modules
|
||||
@@ -0,0 +1,9 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# the copies of this package in every module that carries one.
|
||||
modules/xorg/internal/desktop/*.go
|
||||
modules/lemurs/internal/desktop/*.go
|
||||
modules/i3/internal/desktop/*.go
|
||||
modules/xterm/internal/desktop/*.go
|
||||
modules/adwaita/internal/desktop/*.go
|
||||
@@ -0,0 +1,25 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# the copies of desktop.go and its tests in every module that carries one.
|
||||
modules/blueman/cmd/blueman-tools/desktop.go
|
||||
modules/blueman/cmd/blueman-tools/desktop_test.go
|
||||
modules/blueman/cmd/blueman-tools/copies_test.go
|
||||
modules/forticlient/cmd/forticlient-tools/desktop.go
|
||||
modules/forticlient/cmd/forticlient-tools/desktop_test.go
|
||||
modules/forticlient/cmd/forticlient-tools/copies_test.go
|
||||
modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go
|
||||
modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go
|
||||
modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go
|
||||
modules/nm-applet/cmd/nm-applet-tools/desktop.go
|
||||
modules/nm-applet/cmd/nm-applet-tools/desktop_test.go
|
||||
modules/nm-applet/cmd/nm-applet-tools/copies_test.go
|
||||
modules/openrazer/cmd/openrazer-tools/desktop.go
|
||||
modules/openrazer/cmd/openrazer-tools/desktop_test.go
|
||||
modules/openrazer/cmd/openrazer-tools/copies_test.go
|
||||
modules/polychromatic/cmd/polychromatic-tools/desktop.go
|
||||
modules/polychromatic/cmd/polychromatic-tools/desktop_test.go
|
||||
modules/polychromatic/cmd/polychromatic-tools/copies_test.go
|
||||
modules/slack/cmd/slack-tools/desktop.go
|
||||
modules/slack/cmd/slack-tools/desktop_test.go
|
||||
modules/slack/cmd/slack-tools/copies_test.go
|
||||
@@ -0,0 +1,6 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# mesh-mail's store.json is the same as brother's; no module scan is left.
|
||||
modules/mesh-mail/files/store.json
|
||||
?modules/scan
|
||||
@@ -15,6 +15,13 @@ package main
|
||||
// the desk says in its standing when that happened (`answers-lost-at`), and the router, which alone knows
|
||||
// what is still open and unanswered, says that again under its handle — replacing the notification in place,
|
||||
// so its buttons work again. Nothing resolved, silenced or cancelled comes back.
|
||||
// - **A store that does not answer does not lose where a message is shown, while the desk runs.** A message
|
||||
// is shown first and its notification ids kept after; when the state store does not answer then (the bus
|
||||
// reloading its accounts, 2026-10-26 09:18: the very outage that raised the condition), the work is taken
|
||||
// all the same and the ids are held in memory, recalled before the store, and kept at the start of the next
|
||||
// work and at the minute's standing. Otherwise the clearing of that message finds no notification to
|
||||
// replace and the urgent one stays on screen for good. A desk that restarts while it still holds some loses
|
||||
// where those are shown; the log says so when they are held.
|
||||
// - **Standing**: it can send when machines are named in its settings; which of them answered last is in its
|
||||
// status. A message no machine took is said to the router as failed, and tried again by it.
|
||||
|
||||
@@ -93,6 +100,9 @@ type Desk struct {
|
||||
pause map[string]pauseSeen
|
||||
// lost is when each machine's notifier last started again: the answers its notifications waited for are lost.
|
||||
lost map[string]time.Time
|
||||
// unkept is what was shown and not yet kept in the store, because it did not answer: recalled from here
|
||||
// before the store, and kept at the next piece of work.
|
||||
unkept map[string]Shown
|
||||
}
|
||||
|
||||
func (d *Desk) init() {
|
||||
@@ -107,6 +117,9 @@ func (d *Desk) init() {
|
||||
if d.lost == nil {
|
||||
d.lost = map[string]time.Time{}
|
||||
}
|
||||
if d.unkept == nil {
|
||||
d.unkept = map[string]Shown{}
|
||||
}
|
||||
if d.Now == nil {
|
||||
d.Now = time.Now
|
||||
}
|
||||
@@ -138,6 +151,7 @@ func (d *Desk) Take(subject string, body json.RawMessage) error {
|
||||
d.Logf("[desk-channel] work on %s could not be read; taken and dropped", subject)
|
||||
return nil
|
||||
}
|
||||
d.KeepUnkept()
|
||||
machines := d.Machines()
|
||||
if m.Secret {
|
||||
// Shown only where the notifier said lately it is not paused (the review of 2026-10-09): one held back
|
||||
@@ -167,12 +181,16 @@ func (d *Desk) Take(subject string, body json.RawMessage) error {
|
||||
switch {
|
||||
case edit:
|
||||
var err error
|
||||
if before, err = d.Store.Get(m.Handle); err != nil {
|
||||
if before, err = d.recall(m.Handle); err != nil {
|
||||
return err
|
||||
}
|
||||
if before == nil {
|
||||
d.Logf("[desk-channel] an edit of %s, which nothing here records as shown: shown as a new message; "+
|
||||
"a notification it was to replace, if any, stays", m.Handle)
|
||||
}
|
||||
case !m.Secret:
|
||||
var err error
|
||||
if before, err = d.Store.Get(m.Handle); err != nil {
|
||||
if before, err = d.recall(m.Handle); err != nil {
|
||||
d.Logf("[desk-channel] what is shown under this handle could not be read (%v): shown as a new message", err)
|
||||
before = nil
|
||||
}
|
||||
@@ -218,12 +236,91 @@ func (d *Desk) Take(subject string, body json.RawMessage) error {
|
||||
// Shown once and kept nowhere of this module's (a link's code): no edit or re-showing reaches it.
|
||||
return nil
|
||||
}
|
||||
if err := d.Store.Put(m.Handle, s); err != nil {
|
||||
d.Logf("[desk-channel] shown, and what was shown could not be kept: an edit of it will not reach it: %v", err)
|
||||
}
|
||||
d.keep(m.Handle, s)
|
||||
return nil
|
||||
}
|
||||
|
||||
// keep keeps what a handle shows in the store. When the store does not answer it is held in memory, where an
|
||||
// edit, a reminder and a click still find it, and kept later (KeepUnkept). The work is taken either way: offered
|
||||
// again it would race the message's own later edits, and a redelivery is bounded where an outage is not.
|
||||
func (d *Desk) keep(handle string, s Shown) {
|
||||
err := d.Store.Put(handle, s)
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if err == nil {
|
||||
delete(d.unkept, handle)
|
||||
return
|
||||
}
|
||||
d.unkept[handle] = s
|
||||
d.Logf("[desk-channel] shown, and where it is shown could not be kept yet (%v): held in memory, so its edit "+
|
||||
"and clearing still reach it, and kept when the store answers; %d held, lost if the desk restarts first",
|
||||
err, len(d.unkept))
|
||||
}
|
||||
|
||||
// KeepUnkept keeps in the store what it did not answer for before: at the start of each piece of work and at
|
||||
// the minute's standing. It stops at the first write the store does not answer, so one call waits on it once.
|
||||
func (d *Desk) KeepUnkept() {
|
||||
d.init()
|
||||
d.mu.Lock()
|
||||
held := make(map[string]Shown, len(d.unkept))
|
||||
for h, s := range d.unkept {
|
||||
held[h] = s
|
||||
}
|
||||
d.mu.Unlock()
|
||||
for h, s := range held {
|
||||
if err := d.Store.Put(h, s); err != nil {
|
||||
return
|
||||
}
|
||||
d.mu.Lock()
|
||||
if cur, ok := d.unkept[h]; ok && sameShown(cur, s) {
|
||||
delete(d.unkept, h)
|
||||
}
|
||||
d.mu.Unlock()
|
||||
d.Logf("[desk-channel] where %s is shown is kept now", h)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Desk) unkeptOf(handle string) (Shown, bool) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
s, ok := d.unkept[handle]
|
||||
return s, ok
|
||||
}
|
||||
|
||||
// recall is what a handle shows: from memory when the store did not keep it yet, else from the store.
|
||||
func (d *Desk) recall(handle string) (*Shown, error) {
|
||||
if s, ok := d.unkeptOf(handle); ok {
|
||||
return &s, nil
|
||||
}
|
||||
return d.Store.Get(handle)
|
||||
}
|
||||
|
||||
// handles is every handle shown: the store's and those held in memory.
|
||||
func (d *Desk) handles() ([]string, error) {
|
||||
hs, err := d.Store.Handles()
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
seen := map[string]bool{}
|
||||
for _, h := range hs {
|
||||
seen[h] = true
|
||||
}
|
||||
for h := range d.unkept {
|
||||
if !seen[h] {
|
||||
hs = append(hs, h)
|
||||
}
|
||||
}
|
||||
if err != nil && len(d.unkept) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
return hs, nil
|
||||
}
|
||||
|
||||
func sameShown(a, b Shown) bool {
|
||||
x, err1 := json.Marshal(a)
|
||||
y, err2 := json.Marshal(b)
|
||||
return err1 == nil && err2 == nil && string(x) == string(y)
|
||||
}
|
||||
|
||||
// show puts one message on one machine's notifier, in place of `replace` when it names one.
|
||||
func (d *Desk) show(machine string, m asks.Message, replace int) (int, error) {
|
||||
urgency := "normal"
|
||||
@@ -308,12 +405,12 @@ func (d *Desk) Chosen(event, machine, ticket string) {
|
||||
}
|
||||
|
||||
func (d *Desk) find(ticket string) (string, *Shown) {
|
||||
handles, err := d.Store.Handles()
|
||||
handles, err := d.handles()
|
||||
if err != nil {
|
||||
return "", nil
|
||||
}
|
||||
for _, h := range handles {
|
||||
s, err := d.Store.Get(h)
|
||||
s, err := d.recall(h)
|
||||
if err != nil || s == nil {
|
||||
continue
|
||||
}
|
||||
@@ -356,9 +453,9 @@ func (d *Desk) Started(event, machine string) int {
|
||||
d.lost[machine] = d.Now()
|
||||
d.mu.Unlock()
|
||||
n := 0
|
||||
if handles, err := d.Store.Handles(); err == nil {
|
||||
if handles, err := d.handles(); err == nil {
|
||||
for _, h := range handles {
|
||||
if s, err := d.Store.Get(h); err == nil && s != nil && len(s.Message.Buttons) > 0 {
|
||||
if s, err := d.recall(h); err == nil && s != nil && len(s.Message.Buttons) > 0 {
|
||||
if _, here := s.On[machine]; here {
|
||||
n++
|
||||
}
|
||||
@@ -472,6 +569,7 @@ func (d *Desk) Status() map[string]any {
|
||||
"it proves nothing about who clicked: any process of the operator's account can choose an action, so it answers acknowledgements only",
|
||||
"an answer is taken only on a machine its message was shown on; a message with buttons stays until dismissed",
|
||||
"what it remembers is never shown again: when a notifier started again, the answers its notifications waited for are lost; the desk says when in its standing (answers-lost-at), and the router says again what is still open and unanswered, replacing those notifications in place — nothing resolved, silenced or cancelled comes back (issue 384)",
|
||||
"a store that does not answer does not lose where a message is shown while the desk runs: it is held in memory, its edit and clearing still replace it in place, and it is kept when the store answers again; a desk restarted before then loses where those are shown, and its log said how many it held",
|
||||
"it can send while a named machine's notifier said lately that it is not paused; a link's code is shown only on such a machine, kept nowhere, and sent secret so the notifier keeps it out of its history",
|
||||
}}
|
||||
if s.Why != "" {
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -363,9 +364,64 @@ func TestAMessageSaidAgainUnderItsHandleReplacesItsNotification(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// downStore is a store that does not answer while down: the bus reloading its accounts (2026-10-26 09:18).
|
||||
type downStore struct {
|
||||
memShown
|
||||
down *bool
|
||||
}
|
||||
|
||||
func (s downStore) Get(h string) (*Shown, error) {
|
||||
if *s.down {
|
||||
return nil, errors.New("nats: no responders available for request")
|
||||
}
|
||||
return s.memShown.Get(h)
|
||||
}
|
||||
func (s downStore) Put(h string, v Shown) error {
|
||||
if *s.down {
|
||||
return errors.New("nats: no responders available for request")
|
||||
}
|
||||
return s.memShown.Put(h, v)
|
||||
}
|
||||
|
||||
// The review of PR 176: a replace that fails on a machine keeps that machine's notification on record, so a
|
||||
// later edit or clearing still reaches it; failing everywhere, it is said to the router as failed.
|
||||
// later edit or clearing still reaches it; failing everywhere, it is said to the router as failed. And
|
||||
// (2026-10-26 09:18, an urgent "A new build of mesh-controller cannot be delivered" left on screen for hours
|
||||
// after its clearing): a message shown while the store does not answer keeps where it is shown in memory, its
|
||||
// work is taken, its clearing replaces it in place, and it is kept once the store answers.
|
||||
func TestAFailedReplaceKeepsTheNotificationItWasToReplace(t *testing.T) {
|
||||
// The raise, then its clearing, taken while the store does not answer; the store then stays down or comes
|
||||
// back. Each is taken (never offered again, where a redelivered raise would follow its clearing), the
|
||||
// clearing replaces the urgent notification in place, and the screen and the store end on the clearing.
|
||||
for _, back := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("the store does not answer, back %v", back), func(t *testing.T) {
|
||||
d, shown, _, store := newDesk(t, "laptop")
|
||||
down := true
|
||||
d.Store = downStore{store, &down}
|
||||
raised, _ := json.Marshal(asks.Message{Handle: "m9", Title: "A new build cannot be delivered", Urgent: true})
|
||||
if err := d.Take(asks.ChannelSubject(asks.Send, Kind), raised); err != nil {
|
||||
t.Fatalf("the raise was not taken: %v", err)
|
||||
}
|
||||
cleared, _ := json.Marshal(asks.Message{Handle: "m9", Title: "Cleared: a new build can be delivered"})
|
||||
if err := d.Take(asks.ChannelSubject(asks.Edit, Kind), cleared); err != nil {
|
||||
t.Fatalf("the clearing was not taken: %v", err)
|
||||
}
|
||||
d.KeepUnkept() // the minute's standing, store still down
|
||||
down = !back
|
||||
d.KeepUnkept()
|
||||
take(t, d, asks.Send, asks.Message{Handle: "m10", Title: "another"})
|
||||
if len(*shown) != 3 || (*shown)[1].args["replace_id"] != 1 || (*shown)[1].args["summary"] != "Cleared: a new build can be delivered" {
|
||||
t.Fatalf("the clearing did not replace the urgent notification: %+v", *shown)
|
||||
}
|
||||
s, kept := store["m9"]
|
||||
if back != kept || (kept && s.Message.Title != "Cleared: a new build can be delivered") {
|
||||
t.Fatalf("store back %v, kept %v: %+v", back, kept, store)
|
||||
}
|
||||
if got, _ := d.recall("m9"); got == nil || got.Message.Title != "Cleared: a new build can be delivered" {
|
||||
t.Fatalf("recalled %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
d, shown, said, store := newDesk(t, "laptop", "desktop")
|
||||
take(t, d, asks.Send, asks.Message{Handle: "m1", Title: "2 failed services on shanks"})
|
||||
first := store["m1"].On
|
||||
|
||||
@@ -100,6 +100,7 @@ func run(d *Desk) {
|
||||
}()
|
||||
go func() {
|
||||
for {
|
||||
d.KeepUnkept()
|
||||
d.SayStanding()
|
||||
time.Sleep(time.Minute)
|
||||
}
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,25 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# the copies of desktop.go and its tests in every module that carries one.
|
||||
modules/blueman/cmd/blueman-tools/desktop.go
|
||||
modules/blueman/cmd/blueman-tools/desktop_test.go
|
||||
modules/blueman/cmd/blueman-tools/copies_test.go
|
||||
modules/forticlient/cmd/forticlient-tools/desktop.go
|
||||
modules/forticlient/cmd/forticlient-tools/desktop_test.go
|
||||
modules/forticlient/cmd/forticlient-tools/copies_test.go
|
||||
modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go
|
||||
modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go
|
||||
modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go
|
||||
modules/nm-applet/cmd/nm-applet-tools/desktop.go
|
||||
modules/nm-applet/cmd/nm-applet-tools/desktop_test.go
|
||||
modules/nm-applet/cmd/nm-applet-tools/copies_test.go
|
||||
modules/openrazer/cmd/openrazer-tools/desktop.go
|
||||
modules/openrazer/cmd/openrazer-tools/desktop_test.go
|
||||
modules/openrazer/cmd/openrazer-tools/copies_test.go
|
||||
modules/polychromatic/cmd/polychromatic-tools/desktop.go
|
||||
modules/polychromatic/cmd/polychromatic-tools/desktop_test.go
|
||||
modules/polychromatic/cmd/polychromatic-tools/copies_test.go
|
||||
modules/slack/cmd/slack-tools/desktop.go
|
||||
modules/slack/cmd/slack-tools/desktop_test.go
|
||||
modules/slack/cmd/slack-tools/copies_test.go
|
||||
@@ -275,6 +275,61 @@ reported no key gets no entry, and its client says it waits. Keys titled `mesh:`
|
||||
a key added by hand never is. Every consumer is made again every ten minutes and at once when a key changes.
|
||||
`go test ./cmd/forge-account-provider ./internal/forgeaccount` runs it against a fake forge.
|
||||
|
||||
## The mesh's accounts read every repository (the operator, 2026-10-26)
|
||||
|
||||
The tools act as the plain account `mesh-tools` (ADR 0365 §2, issue 469), which writes to the organisation `novox`
|
||||
through its team; the build modules fetch over ssh as `nox-builder` (ADR 0466 §3), in a team that reads the
|
||||
organisation's code; the machines' agents use `nox-agent` (ADR 0410 §1). Any other repository each saw only when public,
|
||||
so a private repository of a user answered every tool 404 and every build's fetch "not authorized to read". `readall.ts`
|
||||
makes each of these accounts (one list, `meshAccounts` in `tools-account.ts`, whose two Go names
|
||||
`internal/forgeaccount`'s test holds to its own constants) a collaborator that **reads only** on each repository it
|
||||
cannot read: one repository at a time, no organisation membership, the least the forge offers. An account the forge
|
||||
does not hold yet (the forge-account provider makes `nox-builder` and `nox-agent` when their first consumer asks) is
|
||||
named and asked nothing; the start's pass is run again ten minutes apart until every account was there once.
|
||||
|
||||
- **When:** one pass over every repository of every user and organisation at the start (a forge not answering yet is
|
||||
asked again a minute apart until one pass ran), and on the forge's system webhook: `forge.repository.created` and
|
||||
`forge.fork` at once, and the first event of any repository this process has not seen readable (the forge may not
|
||||
send a creation for every owner). Nothing polls for it.
|
||||
- **Not caught:** a repository made private after it was seen readable. The forge sends no event for a change of
|
||||
visibility, and the start's pass already saw it readable, so its events ask nothing. It is granted at the module's
|
||||
next start, or now by `gitea_grant_read`; `gitea_unreadable_repos` shows it meanwhile. A repository whose permission
|
||||
the forge will not answer is named as refused (or, in the listing, with permission `unknown`) and the pass goes on.
|
||||
- **With what:** the site administrator's login (`MESH_GITEA_ADMIN_PASSWORD_FILE`), because adding a collaborator asks a
|
||||
repository's administration, which on another user's repository only the site administrator holds. It is used for
|
||||
five calls alone: list every repository, read one, whether an account exists, read an account's permission on one,
|
||||
add an account as a read collaborator. The tools' own calls keep their token.
|
||||
- **Tools:** `gitea_unreadable_repos` (per account, every repository it cannot read, the permission the forge gives
|
||||
it there and why, a count per account and the accounts the forge does not hold) and `gitea_grant_read` (the same
|
||||
grant by hand, now, on every repository or on those named, for every account or those named; each grant read back
|
||||
from the forge, a refusal answered in the forge's own words).
|
||||
|
||||
### Teams are read with the administrator's login
|
||||
|
||||
`gitea_team_rights` (issue 1563) answers the forge's teams of an organisation (novox when not given), or one team: its
|
||||
members, permission, units (read, write or admin per unit), whether it covers all repositories, and whether it is
|
||||
read-only. With a `user` it adds the teams that user is in and its effective right on every repository (or those named),
|
||||
listing those above read: the check that a build key's user, `nox-builder` in `builders`, reads and pushes nothing. The
|
||||
tools' token lacks `read:organization` (the forge answers 403), so the read goes with the administrator's login, GET only.
|
||||
|
||||
### The merging account writes the repositories the mesh delivers
|
||||
|
||||
`mesh-tools` sets every commit status and merges; outside the organisation it only reads (above), so a pull request of
|
||||
a user's repository could be checked and asked but never merged (issue 1945: car-hunter #2's status refused 403 after
|
||||
the operator's Approve). `writes.ts` makes it a collaborator that **writes** on each repository that builds a registered
|
||||
module, and on no other (the operator, 2026-10-26): the build modules' and the agents' accounts keep read.
|
||||
|
||||
- **Which:** the indexer's state `mesh-delivery-indexer.modules` (in this module's `reads`), one key per registered
|
||||
module, each naming its repository. A repository of `novox` needs nothing: the team writes it. Every other repository
|
||||
of a user stays read-only.
|
||||
- **When:** the state's watch hands every module at the start and each one registered after; a repository not yet
|
||||
written is granted then, read back from the forge, and said on the bus as `gitea.repo.writable` {owner, repo,
|
||||
account}, on which mesh-delivery-gate reads that repository again. Nothing polls.
|
||||
- **Tools:** `gitea_unwritable_repos` (every delivered repository the account cannot write, its permission and why)
|
||||
and `gitea_grant_write` (the same grant by hand, on every delivered repository or those named; what it granted, what
|
||||
was written already, what the forge refused in its own words, and the repositories left read-only and why).
|
||||
`gitea_commit_statuses` answers `merge.writes`, whether the account may write the repository at all.
|
||||
|
||||
## Files it does not know
|
||||
|
||||
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
||||
|
||||
+28
-9
@@ -83,6 +83,9 @@ export function checkStatusOf(change: OwedChange): { owner: string; repo: string
|
||||
return { owner, repo, sha, revision, status };
|
||||
}
|
||||
|
||||
/** How many mesh/check writes this module has at the forge at once. */
|
||||
export const WRITES_AT_ONCE = 4;
|
||||
|
||||
/** The one writer of mesh/check: each owed revision of the checker's, in revision order per key. */
|
||||
export class CheckStatuses {
|
||||
/** The newest revision written of each key, and the newest seen; the oldest keys go past what any forge holds open. */
|
||||
@@ -91,6 +94,9 @@ export class CheckStatuses {
|
||||
/** The turn each key's writes wait on, and the asks again still running. */
|
||||
private turns = new Map<string, Promise<unknown>>();
|
||||
private again = new Set<Promise<void>>();
|
||||
/** Writes at the forge at once, at most WRITES_AT_ONCE: a watch's start hands over every key together (issue 1726). */
|
||||
private writing = 0;
|
||||
private queued: (() => void)[] = [];
|
||||
private forge: CheckForge;
|
||||
private say: (line: string) => void;
|
||||
private sleep: (ms: number) => Promise<void>;
|
||||
@@ -123,7 +129,7 @@ export class CheckStatuses {
|
||||
this.say(`[gitea] ${asked.owner}/${asked.repo}@${asked.sha.slice(0, 12)}: mesh/check at revision ${asked.revision} is older than ${newest}, already written, and is dropped`);
|
||||
return "dropped";
|
||||
}
|
||||
await this.forge.setCommitStatus(asked.owner, asked.repo, asked.sha, asked.status);
|
||||
await this.atTheForge(() => this.forge.setCommitStatus(asked.owner, asked.repo, asked.sha, asked.status));
|
||||
this.remember(this.written, key, asked.revision);
|
||||
return "written";
|
||||
});
|
||||
@@ -133,19 +139,32 @@ export class CheckStatuses {
|
||||
return turn;
|
||||
}
|
||||
|
||||
/** What the watch hands over: written, or when the forge refuses, asked again on a bound in the background until it is
|
||||
* written or a newer revision of the key was seen. Never throws: a watch's handler that throws loses the change. */
|
||||
/** What the watch hands over: taken at once and written in the background, or when the forge refuses, asked again on a
|
||||
* bound until it is written or a newer revision of the key was seen. Never throws: a watch's handler that throws loses
|
||||
* the change. It never waits on the forge (novox/hq issue 1726): the runtime hands a watch's current values one at a
|
||||
* time, waits for each to be taken and gives the whole set 10 s; a forge write per key made a large owed outlast that
|
||||
* bound at every attempt, so the watch never held and no verdict reached mesh/check. Each key's writes still take
|
||||
* turns in revision order (apply). */
|
||||
async take(change: OwedChange): Promise<void> {
|
||||
const run = this.apply(change).then(() => {}, (err) => this.askAgain(change, err));
|
||||
this.again.add(run);
|
||||
void run.finally(() => this.again.delete(run));
|
||||
}
|
||||
|
||||
/** Runs one forge write when fewer than WRITES_AT_ONCE are running, else after one ends. */
|
||||
private async atTheForge<T>(write: () => Promise<T>): Promise<T> {
|
||||
if (this.writing >= WRITES_AT_ONCE) await new Promise<void>((r) => this.queued.push(r));
|
||||
else this.writing++;
|
||||
try {
|
||||
await this.apply(change);
|
||||
} catch (err) {
|
||||
const run = this.askAgain(change, err);
|
||||
this.again.add(run);
|
||||
void run.finally(() => this.again.delete(run));
|
||||
return await write();
|
||||
} finally {
|
||||
const next = this.queued.shift();
|
||||
if (next) next();
|
||||
else this.writing--;
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolves when no ask again is running. */
|
||||
/** Resolves when no write or ask again is running. */
|
||||
async settled(): Promise<void> {
|
||||
while (this.again.size) await Promise.all([...this.again]);
|
||||
}
|
||||
|
||||
+245
-14
@@ -9,6 +9,7 @@ import { ConfiguredToken, MintedToken, type TokenSource } from "./token.js";
|
||||
import { MAIN_BRANCH, mainRuleBody, protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js";
|
||||
import { Refused, checkSentPath, ruleCovers, pathSegment, pathSegments } from "./guard.js";
|
||||
import { isCommit } from "./checks.js";
|
||||
import type { ForgeTeam } from "./teamrights.js";
|
||||
|
||||
/** A repository's path on the forge, its owner and name each one segment (guard.ts pathSegment): uniform for every
|
||||
* call, whether the names came from a caller or from the forge's own answer. */
|
||||
@@ -50,6 +51,9 @@ export interface GiteaRepo {
|
||||
default_branch?: string;
|
||||
/** When anything last moved in it — a push, and so a merge. */
|
||||
updated_at?: string;
|
||||
/** Whether the account asking may write it (the forge's permissions.push or admin): a commit status and a merge need
|
||||
* it. Absent when the forge did not say. */
|
||||
writes?: boolean;
|
||||
}
|
||||
|
||||
/** An issue, with its labels flattened to names. */
|
||||
@@ -98,6 +102,10 @@ export interface GiteaPull {
|
||||
html_url: string;
|
||||
/** The description: where a pull request says it goes after another (`after: <repository>`, novox/hq ADR 0239). */
|
||||
body?: string;
|
||||
/** Whether the forge would merge it into its base now (gitea's `mergeable`): false while it is in conflict, while the
|
||||
* forge is still working that out after its base moved, or while it is a work in progress; absent when the forge did not
|
||||
* say. What a group's merge reads of every member before it merges any (novox/hq issue 1275). */
|
||||
mergeable?: boolean;
|
||||
}
|
||||
|
||||
/** An open pull request with what its base requires and what its head has: what gitea_open_pulls answers. */
|
||||
@@ -167,8 +175,27 @@ function ignoreSecretSetting(cfg: Record<string, unknown>, key: string): void {
|
||||
console.error(`the setting ${key} is ignored: it comes from the vault`);
|
||||
}
|
||||
|
||||
/** How long the forge may take to answer one request before it is cut and said (novox/hq issue 1577: a request for a 5 MB
|
||||
* file never ended, and every read through the module timed out behind it). */
|
||||
export const FORGE_REQUEST_TIMEOUT_MS = 20_000;
|
||||
|
||||
/** The most a tool's answer carries of one file: the bus's message bound is 1 MiB and a file travels as text (issue 1577). */
|
||||
export const MAX_FILE_BYTES = 512 * 1024;
|
||||
|
||||
/** One request to the forge, cut at the limit and failing loud naming the request. */
|
||||
async function timedFetch(url: string, init: RequestInit, label: string, timeoutMs: number): Promise<Response> {
|
||||
try {
|
||||
return await fetch(url, { ...init, signal: init.signal ? AbortSignal.any([init.signal, AbortSignal.timeout(timeoutMs)]) : AbortSignal.timeout(timeoutMs) });
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.name === "TimeoutError") throw new Error(`Gitea API ${label}: no answer in ${timeoutMs / 1000} s, the request was cut`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export class GiteaClient {
|
||||
readonly baseUrl: string;
|
||||
/** The limit of one request; a test sets it short. */
|
||||
requestTimeoutMs = FORGE_REQUEST_TIMEOUT_MS;
|
||||
private readonly tokens: TokenSource;
|
||||
|
||||
/** A token given as a string is one somebody configured; a source decides for itself (token.ts). */
|
||||
@@ -223,14 +250,14 @@ export class GiteaClient {
|
||||
|
||||
private send(path: string, options: RequestInit, token: string): Promise<Response> {
|
||||
checkSentPath(path);
|
||||
return fetch(`${this.baseUrl}/api/v1${path}`, {
|
||||
return timedFetch(`${this.baseUrl}/api/v1${path}`, {
|
||||
...options,
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `token ${token}`,
|
||||
...(options.headers as Record<string, string> | undefined),
|
||||
},
|
||||
});
|
||||
}, `${options.method ?? "GET"} ${path}`, this.requestTimeoutMs);
|
||||
}
|
||||
|
||||
/** Generic authenticated API call — the escape hatch for endpoints without a dedicated method.
|
||||
@@ -533,7 +560,7 @@ export class GiteaClient {
|
||||
* **A listing the forge cut is no listing** (it says `truncated` on a page with nothing more to read, or the pages
|
||||
* run past a bound): the call fails naming the repository and the commit, never answering the directories it did
|
||||
* see, since a directory left out would be taken for one without a module. Reads only. */
|
||||
async commitTrees(owner: string, repo: string, sha: string, paths?: Record<string, string[]>): Promise<{ root: string; trees: Record<string, string>; paths?: Record<string, Record<string, string>> }> {
|
||||
async commitTrees(owner: string, repo: string, sha: string, paths?: Record<string, string[]>, listed?: string[]): Promise<{ root: string; trees: Record<string, string>; paths?: Record<string, Record<string, string>>; listed?: Record<string, Record<string, string>> }> {
|
||||
const base = `${repoPath(owner, repo)}/git/trees/${commitSegment(sha)}?recursive=true`;
|
||||
const cut = (why: string) => new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} was cut (${why}); no trees are answered from a part`);
|
||||
const entries: any[] = [];
|
||||
@@ -560,7 +587,23 @@ export class GiteaClient {
|
||||
if (!tree) throw new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} holds ${e.path} but not the tree of ${dir}`);
|
||||
trees[dir] = tree;
|
||||
}
|
||||
if (paths === undefined) return { root, trees };
|
||||
// **Listed directories** (novox/hq ADR 0549 §2, issue 1126): each asked directory's files directly under it, by name
|
||||
// to blob id, from the same listing: how the forge bridge finds the catalogue's placement files, one per node, at a
|
||||
// merge. A directory the commit lacks lists nothing (the catalogue before its first placement file).
|
||||
let files: Record<string, Record<string, string>> | undefined;
|
||||
if (listed !== undefined) {
|
||||
files = {};
|
||||
for (const dir of listed) {
|
||||
files[dir] = {};
|
||||
const prefix = dir === "" ? "" : `${dir}/`;
|
||||
for (const e of entries) {
|
||||
if (e?.type !== "blob" || typeof e.path !== "string" || !e.path.startsWith(prefix)) continue;
|
||||
const name = e.path.slice(prefix.length);
|
||||
if (name !== "" && !name.includes("/")) files[dir][name] = String(e.sha);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (paths === undefined) return files === undefined ? { root, trees } : { root, trees, listed: files };
|
||||
// **Named paths** (novox/hq ADR 0499 §3, issue 1758): each asked path's git object id at the commit, a file's or a
|
||||
// directory's, from the same listing, by directory and by path relative to it. A path the listing lacks fails
|
||||
// naming both, and nothing is answered in part.
|
||||
@@ -575,7 +618,7 @@ export class GiteaClient {
|
||||
named[dir][p] = id;
|
||||
}
|
||||
}
|
||||
return { root, trees, paths: named };
|
||||
return files === undefined ? { root, trees, paths: named } : { root, trees, paths: named, listed: files };
|
||||
}
|
||||
|
||||
/** Whether a repository holds a file at a commit: true, false for a 404, thrown when the forge cannot say. */
|
||||
@@ -764,6 +807,22 @@ export class GiteaClient {
|
||||
/** One file's contents at a ref (default the repository's default branch), decoded. */
|
||||
async getFile(owner: string, repo: string, path: string, ref?: string): Promise<{ path: string; ref?: string; sha: string; size: number; content: string }> {
|
||||
const qs = ref ? `?ref=${encodeURIComponent(ref)}` : "";
|
||||
// The size first, from the file's directory listing, which carries no content: a file over what an answer can carry is
|
||||
// refused without its body being fetched (issue 1577). A file the listing does not show is asked for as before.
|
||||
const clean = path.replace(/^\/+|\/+$/g, "");
|
||||
pathSegments("file path", clean); // a path that is another path or user is refused before anything reaches the forge
|
||||
const slash = clean.lastIndexOf("/");
|
||||
const dir = slash < 0 ? "" : clean.slice(0, slash);
|
||||
let listing: any[] | null = null;
|
||||
try {
|
||||
listing = await this.request<any[]>(`${repoPath(owner, repo)}/contents${dir ? "/" + pathSegments("file path", dir) : ""}${qs}`);
|
||||
} catch (err) {
|
||||
if (!(err instanceof Error) || !/: 404 /.test(err.message)) throw err;
|
||||
}
|
||||
const entry = Array.isArray(listing) ? listing.find((e) => e?.path === clean) : undefined;
|
||||
if (entry && Number(entry.size) > MAX_FILE_BYTES) {
|
||||
throw new Error(`the file is ${Number(entry.size)} bytes, over the ${MAX_FILE_BYTES / 1024} KiB a tool answer can carry; read it from a clone`);
|
||||
}
|
||||
const f = await this.request<any>(`${repoPath(owner, repo)}/contents/${pathSegments("file path", path.replace(/^\/+|\/+$/g, ""))}${qs}`);
|
||||
if (!f || f.type !== "file") throw new Error(`Gitea API: ${path} is not a file`);
|
||||
const content = f.encoding === "base64" ? Buffer.from(String(f.content ?? ""), "base64").toString("utf8") : String(f.content ?? "");
|
||||
@@ -789,14 +848,29 @@ export class GiteaClient {
|
||||
|
||||
/** Merge a pull request — never forced past its checks, and, given the head that was checked, only that head:
|
||||
* the forge refuses the merge when the branch moved since (novox/hq ADR 0266). */
|
||||
/** How long a merge waits before it asks again after the forge's 405 "Please try again later" (novox/hq issue 1275):
|
||||
* the forge says so while it works out whether the pull request merges, for a few seconds after its base moved. */
|
||||
static mergeRetryAfter: number[] = [1000, 2000, 4000];
|
||||
|
||||
async mergePullRequest(owner: string, repo: string, index: number, method = "merge", deleteBranch = false,
|
||||
headCommit?: string): Promise<void> {
|
||||
indexOf(index);
|
||||
await this.request(`${repoPath(owner, repo)}/pulls/${index}/merge`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ Do: method, delete_branch_after_merge: deleteBranch, force_merge: false,
|
||||
...(headCommit ? { head_commit_id: headCommit } : {}) }),
|
||||
});
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
await this.request(`${repoPath(owner, repo)}/pulls/${index}/merge`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ Do: method, delete_branch_after_merge: deleteBranch, force_merge: false,
|
||||
...(headCommit ? { head_commit_id: headCommit } : {}) }),
|
||||
});
|
||||
return;
|
||||
} catch (err) {
|
||||
// Only the forge's transient answer is asked again, briefly; every other refusal, and this one past the last
|
||||
// wait, is the caller's to say. head_commit_id keeps a head that moved in between from landing.
|
||||
const wait = GiteaClient.mergeRetryAfter[attempt];
|
||||
if (wait === undefined || !tryAgainLater(err)) throw err;
|
||||
await new Promise((r) => setTimeout(r, wait));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Mappers: the wire shape is broad and unstable; the mesh sees only these fields. ----
|
||||
@@ -812,6 +886,8 @@ export class GiteaClient {
|
||||
html_url: r.html_url,
|
||||
default_branch: r.default_branch,
|
||||
updated_at: r.updated_at ?? undefined,
|
||||
...(r.permissions && typeof r.permissions === "object"
|
||||
? { writes: r.permissions.push === true || r.permissions.admin === true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -842,6 +918,7 @@ export class GiteaClient {
|
||||
updated_at: p.updated_at ?? undefined,
|
||||
html_url: p.html_url,
|
||||
body: typeof p.body === "string" ? p.body : undefined,
|
||||
mergeable: typeof p.mergeable === "boolean" && !p.merged ? p.mergeable : undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -853,6 +930,25 @@ interface AdminResponse {
|
||||
readonly body: any;
|
||||
}
|
||||
|
||||
/** A repository as the site administrator sees it: what saying why an account cannot read it needs (readall.ts). */
|
||||
export interface ForgeRepository {
|
||||
full_name: string;
|
||||
owner: string;
|
||||
name: string;
|
||||
private: boolean;
|
||||
archived: boolean;
|
||||
/** The owner's own visibility: public, limited or private. A public repository of an owner not public is hidden
|
||||
* from whoever is no member. */
|
||||
owner_visibility: string;
|
||||
}
|
||||
|
||||
function forgeRepository(r: any): ForgeRepository {
|
||||
const owner = String(r?.owner?.login ?? r?.owner?.username ?? "");
|
||||
const name = String(r?.name ?? "");
|
||||
return { full_name: String(r?.full_name ?? `${owner}/${name}`), owner, name, private: r?.private === true,
|
||||
archived: r?.archived === true, owner_visibility: String(r?.owner?.visibility ?? "public") };
|
||||
}
|
||||
|
||||
/**
|
||||
* The forge's admin client, over **basic auth** — gitea's own code, living in the module, used only
|
||||
* by the provisioner (novox/hq ADR 0048/0076).
|
||||
@@ -896,14 +992,14 @@ export class GiteaAdmin {
|
||||
* throwing on it — the caller decides which non-2xx codes are idempotent successes. Only an
|
||||
* unexpected status becomes an error, and only where the caller says so. */
|
||||
private async request(path: string, options: RequestInit = {}): Promise<AdminResponse> {
|
||||
const res = await fetch(`${this.baseUrl}/api/v1${path}`, {
|
||||
const res = await timedFetch(`${this.baseUrl}/api/v1${path}`, {
|
||||
...options,
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: this.authorization,
|
||||
...(options.headers as Record<string, string> | undefined),
|
||||
},
|
||||
});
|
||||
}, `${options.method ?? "GET"} ${path}`, FORGE_REQUEST_TIMEOUT_MS);
|
||||
const text = await res.text();
|
||||
let body: any = null;
|
||||
if (text) {
|
||||
@@ -970,6 +1066,105 @@ export class GiteaAdmin {
|
||||
return repos;
|
||||
}
|
||||
|
||||
// ---- The tools' account reads every repository (readall.ts) ----
|
||||
|
||||
/** Every repository on the forge, of every user and organisation: the search as the site administrator, whom the
|
||||
* forge answers every repository, private ones and those of private owners included. Only a read. */
|
||||
async everyRepository(): Promise<ForgeRepository[]> {
|
||||
const all: ForgeRepository[] = [];
|
||||
for (let page = 1; page <= 200; page++) {
|
||||
const res = await this.request(`/repos/search?limit=50&page=${page}`);
|
||||
if (res.status !== 200) GiteaAdmin.fail("/repos/search", res);
|
||||
const list: any[] = Array.isArray(res.body?.data) ? res.body.data : [];
|
||||
all.push(...list.map(forgeRepository));
|
||||
if (list.length < 50) break;
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
/** One repository as the site administrator sees it, or null when the forge holds none by that name. */
|
||||
async repository(owner: string, repo: string): Promise<ForgeRepository | null> {
|
||||
const res = await this.request(repoPath(owner, repo));
|
||||
if (res.status === 404) return null;
|
||||
if (res.status !== 200) GiteaAdmin.fail(repoPath(owner, repo), res);
|
||||
return forgeRepository(res.body);
|
||||
}
|
||||
|
||||
/** Whether the forge holds an account of this name. */
|
||||
async accountExists(user: string): Promise<boolean> {
|
||||
const path = `/users/${encodeURIComponent(user)}`;
|
||||
const res = await this.request(path);
|
||||
if (res.status === 404) return false;
|
||||
if (res.status !== 200) GiteaAdmin.fail(path, res);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** What the forge gives an account on a repository, effective: none, read, write, admin or owner. It is the repository as
|
||||
* that account sees it (the site administrator acts as the account with `sudo`), so access through an organisation's team
|
||||
* counts. The collaborator endpoint is not used: it names only a collaborator's own permission, and says "read" of an
|
||||
* account that writes through a team (novox/hq issue 1945). */
|
||||
async repositoryPermission(owner: string, repo: string, user: string): Promise<string> {
|
||||
const path = `${repoPath(owner, repo)}?sudo=${encodeURIComponent(user)}`;
|
||||
const res = await this.request(path);
|
||||
if (res.status === 404) return "none";
|
||||
if (res.status !== 200) GiteaAdmin.fail(path, res);
|
||||
const p = res.body?.permissions;
|
||||
if (p?.admin === true) return res.body?.owner?.login === user ? "owner" : "admin";
|
||||
if (p?.push === true) return "write";
|
||||
return p?.pull === true ? "read" : "none";
|
||||
}
|
||||
|
||||
/** Make an account a collaborator of a repository that reads only. The forge asks a repository's administrator for
|
||||
* it, which on a repository of another user only the site administrator is. Idempotent: the forge answers 204 again. */
|
||||
async addReadCollaborator(owner: string, repo: string, user: string): Promise<void> {
|
||||
await this.addCollaborator(owner, repo, user, "read");
|
||||
}
|
||||
|
||||
/** Make an account a collaborator of a repository that writes: pushes a branch, sets a commit status, merges (writes.ts:
|
||||
* only on a repository that builds a registered module). An existing read collaborator is changed to write. */
|
||||
async addWriteCollaborator(owner: string, repo: string, user: string): Promise<void> {
|
||||
await this.addCollaborator(owner, repo, user, "write");
|
||||
}
|
||||
|
||||
/** The rule that applies to main in a repository, as its administrator reads it: a rule named main, else the first glob
|
||||
* rule covering main; null when none does. With the administrator's login. */
|
||||
async mainRule(owner: string, repo: string): Promise<BranchProtection | null> {
|
||||
const rules = await this.branchProtections(owner, repo);
|
||||
return rules.find((r) => (r.rule_name ?? r.branch_name) === MAIN_BRANCH)
|
||||
?? rules.find((r) => ruleCovers(String(r.rule_name ?? r.branch_name ?? ""), MAIN_BRANCH)) ?? null;
|
||||
}
|
||||
|
||||
/** Put an account on main's merge whitelist in a repository of a user, and nothing else (the operator, 2026-10-26:
|
||||
* write, the rule untouched). Only an existing rule whose merge whitelist is on is changed, and only its whitelist: the
|
||||
* account is added to the logins already there. Push, status checks, approvals and the administrator override stay as
|
||||
* the operator set them; with no rule, or the whitelist off, nothing is written (whoever writes may merge). Read back:
|
||||
* the forge drops a whitelist login that cannot write the repository and still answers success, so a whitelist read back
|
||||
* without the account fails loud. Answers the rule as the forge holds it after. With the administrator's login. */
|
||||
async ensureMergerListed(owner: string, repo: string, account: string): Promise<{ changed: boolean; rule: BranchProtection | null }> {
|
||||
const rule = await this.mainRule(owner, repo);
|
||||
const listed = rule?.merge_whitelist_usernames ?? [];
|
||||
if (!rule || rule.enable_merge_whitelist !== true || listed.includes(account)) return { changed: false, rule };
|
||||
const name = String(rule.rule_name ?? rule.branch_name);
|
||||
const path = `${repoPath(owner, repo)}/branch_protections/${encodeURIComponent(name)}`;
|
||||
const res = await this.request(path, { method: "PATCH", body: JSON.stringify({ merge_whitelist_usernames: [...listed, account] }) });
|
||||
if (res.status !== 200) GiteaAdmin.fail(path, res);
|
||||
const after = (await this.branchProtections(owner, repo)).find((r) => (r.rule_name ?? r.branch_name) === name);
|
||||
if (!after) throw new Error(`main's rule ${name} of ${owner}/${repo} was written, but the forge does not list it`);
|
||||
const held = after.merge_whitelist_usernames ?? [];
|
||||
if (after.enable_merge_whitelist === true && !held.includes(account)) {
|
||||
throw new Error(`${account} was added to the merge whitelist of main's rule ${name} of ${owner}/${repo}, but the forge ` +
|
||||
`holds [${held.join(", ")}]: it drops a login that cannot write the repository`);
|
||||
}
|
||||
return { changed: true, rule: after };
|
||||
}
|
||||
|
||||
private async addCollaborator(owner: string, repo: string, user: string, permission: "read" | "write"): Promise<void> {
|
||||
const path = `${repoPath(owner, repo)}/collaborators/${encodeURIComponent(user)}`;
|
||||
const res = await this.request(path, { method: "PUT", body: JSON.stringify({ permission }) });
|
||||
if (res.status === 204 || res.status === 200) return;
|
||||
GiteaAdmin.fail(path, res);
|
||||
}
|
||||
|
||||
/** Ensure the npm-owner org exists. 201 created, 2xx/404-then-created, and 422/409 (a concurrent
|
||||
* create won the race) are all success. */
|
||||
async ensureOrg(name: string): Promise<void> {
|
||||
@@ -1031,6 +1226,35 @@ export class GiteaAdmin {
|
||||
return match ? Number(match.id) : null;
|
||||
}
|
||||
|
||||
/** The teams of an organisation, as the forge's Team objects (issue 1563). With the administrator's login: the tools'
|
||||
* account lacks read:organization, so the list answers it 403. Only a read. */
|
||||
async teams(org: string): Promise<ForgeTeam[]> {
|
||||
const all: ForgeTeam[] = [];
|
||||
for (let page = 1; page <= 50; page++) {
|
||||
const path = `/orgs/${encodeURIComponent(org)}/teams?limit=50&page=${page}`;
|
||||
const res = await this.request(path);
|
||||
if (res.status !== 200) GiteaAdmin.fail(`/orgs/${org}/teams`, res);
|
||||
const list: ForgeTeam[] = Array.isArray(res.body) ? res.body : [];
|
||||
all.push(...list);
|
||||
if (list.length < 50) break;
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
/** The logins of a team's members (GET /teams/{id}/members). Only a read. */
|
||||
async teamMembers(teamId: number): Promise<string[]> {
|
||||
const all: string[] = [];
|
||||
for (let page = 1; page <= 50; page++) {
|
||||
const path = `/teams/${teamId}/members?limit=50&page=${page}`;
|
||||
const res = await this.request(path);
|
||||
if (res.status !== 200) GiteaAdmin.fail(`/teams/${teamId}/members`, res);
|
||||
const list: any[] = Array.isArray(res.body) ? res.body : [];
|
||||
all.push(...list.map((u) => String(u?.login ?? u?.username ?? "")).filter(Boolean));
|
||||
if (list.length < 50) break;
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
/** Ensure a user exists with exactly this password. Created if absent; if already there, its
|
||||
* password is patched — so the mesh minting a new secret takes on the next reconcile.
|
||||
*
|
||||
@@ -1088,9 +1312,9 @@ export class GiteaAdmin {
|
||||
* other answer rejects (novox/hq issue 120).
|
||||
*/
|
||||
async holdsTeamMember(org: string, team: string, username: string, password: string): Promise<boolean> {
|
||||
const me = await fetch(`${this.baseUrl}/api/v1/user`, {
|
||||
const me = await timedFetch(`${this.baseUrl}/api/v1/user`, {
|
||||
headers: { Authorization: "Basic " + Buffer.from(`${username}:${password}`).toString("base64") },
|
||||
});
|
||||
}, "GET /user", FORGE_REQUEST_TIMEOUT_MS);
|
||||
if (me.status === 401 || me.status === 403) return false;
|
||||
if (me.status !== 200) throw new Error(`Gitea GET /user as ${username}: ${me.status}`);
|
||||
const teams = await this.request(`/orgs/${encodeURIComponent(org)}/teams?limit=50`);
|
||||
@@ -1160,3 +1384,10 @@ export function directoriesAbove(paths: string[]): string[] {
|
||||
function encodePath(path: string): string {
|
||||
return pathSegments("file path", path);
|
||||
}
|
||||
|
||||
/** Whether an error is the forge's 405 "Please try again later" on a merge: its answer while the pull request's
|
||||
* mergeability is being worked out (or it is in conflict, which a later ask says the same way). */
|
||||
export function tryAgainLater(err: unknown): boolean {
|
||||
const words = err instanceof Error ? err.message : String(err);
|
||||
return / 405 /.test(words) && /try again later/i.test(words);
|
||||
}
|
||||
|
||||
@@ -65,6 +65,7 @@ func configFromEnv(getenv func(string) string) (Config, error) {
|
||||
HostFile: getenv("MESH_FORGE_HOST_FILE"),
|
||||
OriginsFile: getenv("MESH_GITEA_ORIGINS_FILE"),
|
||||
Port: port,
|
||||
HostKeyFile: getenv("MESH_GITEA_HOST_KEY_FILE"), // optional
|
||||
Forge: forgehttp.Client{URL: strings.TrimRight(getenv("MESH_GITEA_URL"), "/"), User: getenv("MESH_GITEA_ADMIN_USER"),
|
||||
PasswordFile: getenv("MESH_GITEA_ADMIN_PASSWORD_FILE"), Timeout: 20 * time.Second},
|
||||
Runner: execRunner{},
|
||||
|
||||
@@ -44,10 +44,13 @@ type Config struct {
|
||||
// origins, one per line.
|
||||
HostFile, OriginsFile string
|
||||
// Port is where the forge's sshd is published on the mesh.
|
||||
Port int
|
||||
Forge forgehttp.Client
|
||||
Runner Runner
|
||||
Log func(format string, a ...any)
|
||||
Port int
|
||||
// HostKeyFile is the forge's own ed25519 host key (the .pub file in its data directory). It is read first; when it
|
||||
// is unset or unreadable the forge's sshd is asked with ssh-keyscan instead (issue 1939).
|
||||
HostKeyFile string
|
||||
Forge forgehttp.Client
|
||||
Runner Runner
|
||||
Log func(format string, a ...any)
|
||||
}
|
||||
|
||||
// Service answers the consumers of the provision `forge-account` (novox/hq ADR 0399, ADR 0400; issue 955): for each
|
||||
@@ -448,14 +451,30 @@ func (s *Service) httpsBase() (string, error) {
|
||||
return "", fmt.Errorf("%s names no https origin", s.cfg.OriginsFile)
|
||||
}
|
||||
|
||||
// scan reads the forge's ed25519 host key from its own sshd and writes it as the machines' known_hosts line.
|
||||
// scan gives the forge's ed25519 host key as the machines' known_hosts line. The key the forge itself holds is read
|
||||
// from its file first; only when that cannot be read is the forge's sshd asked over the network, and when that fails
|
||||
// too (the forge's container is restarting) the key read before is kept, as a host key does not change (issue 1939).
|
||||
func (s *Service) scan(ctx context.Context) (string, error) {
|
||||
host, err := s.hostName()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if s.cfg.HostKeyFile != "" {
|
||||
raw, ferr := os.ReadFile(s.cfg.HostKeyFile)
|
||||
if ferr == nil {
|
||||
if line, perr := knownHostLine("127.0.0.1 "+strings.TrimSpace(string(raw)), host, s.cfg.Port); perr == nil {
|
||||
return line, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
out, err := s.cfg.Runner.Run(ctx, "ssh-keyscan", "-t", "ed25519", "-p", fmt.Sprint(s.cfg.Port), "127.0.0.1")
|
||||
if err != nil {
|
||||
s.mu.Lock()
|
||||
kept := s.known
|
||||
s.mu.Unlock()
|
||||
if kept != "" {
|
||||
return kept, nil
|
||||
}
|
||||
return "", fmt.Errorf("the forge's ssh host key cannot be read: %w", err)
|
||||
}
|
||||
return knownHostLine(string(out), host, s.cfg.Port)
|
||||
|
||||
@@ -696,3 +696,42 @@ func TestNothingIsSaidForAPerNodeUserWithoutMeshKeys(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The forge's own host key file is read first: no scan over the network is needed (issue 1939).
|
||||
func TestTheHostKeyIsReadFromTheForgesOwnFile(t *testing.T) {
|
||||
r := newRig(t)
|
||||
f := filepath.Join(t.TempDir(), "ssh_host_ed25519_key.pub")
|
||||
if err := os.WriteFile(f, []byte("ssh-ed25519 AAAAC3FILEKEY forge@host\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.svc.cfg.HostKeyFile = f
|
||||
r.runner.err = errors.New("sshd down")
|
||||
got, err := r.svc.scan(context.Background())
|
||||
if err != nil || got != "[novox.internal]:222 ssh-ed25519 AAAAC3FILEKEY" {
|
||||
t.Fatalf("scan = %q, %v", got, err)
|
||||
}
|
||||
if r.runner.args != nil {
|
||||
t.Errorf("the forge's sshd was asked: %v", r.runner.args)
|
||||
}
|
||||
}
|
||||
|
||||
// A file that cannot be read falls back to the forge's sshd.
|
||||
func TestAnUnreadableHostKeyFileFallsBackToTheScan(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.svc.cfg.HostKeyFile = filepath.Join(t.TempDir(), "missing.pub")
|
||||
got, err := r.svc.scan(context.Background())
|
||||
if err != nil || got != "[novox.internal]:222 ssh-ed25519 AAAAC3HOSTKEY" {
|
||||
t.Fatalf("scan = %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// While the forge restarts, a key read before is kept: the sshd is not up, the key has not changed.
|
||||
func TestARestartingForgeKeepsTheKeyReadBefore(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.svc.known = "[novox.internal]:222 ssh-ed25519 AAAAC3HOSTKEY"
|
||||
r.runner.err = errors.New("connection refused")
|
||||
got, err := r.svc.scan(context.Background())
|
||||
if err != nil || got != r.svc.known {
|
||||
t.Fatalf("scan = %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
+55
-4
@@ -58,8 +58,11 @@ import { CHECK_CONTEXT, CHECK_STATUS, REPO_CHECK_CONTEXT, commentFor, composedGa
|
||||
import { GateWriter } from "./gate.js";
|
||||
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
|
||||
import { keepSiblings, noteComposed, noteVerdict, readSiblings, siblingsFile, siblingsOf, weighSiblings } from "./siblings.js";
|
||||
import { bindInBackground } from "./bind.js";
|
||||
import { bindInBackground, bindUntilHeld } from "./bind.js";
|
||||
import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
|
||||
import { ReadAll, readAllTools } from "./readall.js";
|
||||
import { DeliveredWrites, INDEXER_MODULES, writeTools, type ModulesChange } from "./writes.js";
|
||||
import { TOOLS_ACCOUNT, TOOLS_ORG, meshAccounts } from "./tools-account.js";
|
||||
|
||||
// Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
|
||||
// nothing to watch; log and stay quiet rather than crash the runtime. With one, the first poll mints
|
||||
@@ -500,8 +503,21 @@ if (!gitea) pace.unheard("the forge is not watched: no way to a token", false);
|
||||
// Said with `emit`, not the SDK's `keeps`, which the toolchain's SDK does not have yet: switch to `keeps` once that SDK is 0.1.15 or later (issue 1852).
|
||||
await emit("states", statesAnnouncement(process.env.MESH_MODULE || "gitea", process.env.MESH_NODE));
|
||||
|
||||
// **The mesh's accounts read every repository** (readall.ts; the operator, 2026-10-26): the tools', the build modules'
|
||||
// and the machines' agents' (meshAccounts, tools-account.ts), granted with the site
|
||||
// administrator's login, which the module holds for setting the forge up and uses here for that grant alone. Made on
|
||||
// first use, so a module whose admin login is missing still serves the rest and the tools say why.
|
||||
let readAllOnce: ReadAll | null = null;
|
||||
const readAll = (): ReadAll => (readAllOnce ??= new ReadAll(GiteaAdmin.fromEnv(), meshAccounts(process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT)));
|
||||
|
||||
// **The merging account writes every repository that builds a registered module** (writes.ts; the operator,
|
||||
// 2026-10-26; issue 1945): which those are is the indexer's state, watched below. Made on first use, as readAll.
|
||||
let writesOnce: DeliveredWrites | null = null;
|
||||
const writes = (): DeliveredWrites => (writesOnce ??= new DeliveredWrites(GiteaAdmin.fromEnv(),
|
||||
process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT, TOOLS_ORG, (l) => console.log(`[gitea] ${l}`), (type, body) => emit(type, body)));
|
||||
|
||||
// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729).
|
||||
registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({
|
||||
registerModuleTools("gitea", () => [eventsTool(pace), ...readAllTools(readAll), ...writeTools(writes), ...(gitea ? [commitStatusTool({
|
||||
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
||||
gates,
|
||||
}), mergeTool(gitea, { gates })] : [])]);
|
||||
@@ -543,7 +559,12 @@ async function listenForForge(client: GiteaClient): Promise<boolean> {
|
||||
const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery),
|
||||
refused: (reason, from) => pace.refused(reason, from),
|
||||
publish: (event) => emit(event.name, event.body),
|
||||
act: (event) => actOn(client, event) });
|
||||
act: async (event) => {
|
||||
// A repository made, or one not yet seen readable, is granted read before anything acts on it (readall.ts).
|
||||
await Promise.resolve().then(() => readAll().onEvent(event)).catch((err) =>
|
||||
console.error(`[gitea] whether the mesh's accounts read the repository of ${event.name} could not be settled — ${err instanceof Error ? err.message : err}`));
|
||||
await actOn(client, event);
|
||||
} });
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(port, host, () => resolve());
|
||||
@@ -589,7 +610,37 @@ if (gitea) {
|
||||
};
|
||||
// **The forge's events first, the polls as the safety net** (novox/hq issue 729): while verified deliveries arrive the
|
||||
// polls slow to every five minutes and announce only what no event said (Pace); otherwise they keep 30 s.
|
||||
await listenForForge(client);
|
||||
// **Not awaited** (novox/hq issue 1726): the receiver's set-up asks the forge's admin API, and the runtime kills a bundle
|
||||
// whose import has not ended in its 10 s handshake and never starts it again; a forge slow to answer while every bundle
|
||||
// of the node starts left gitea_commit_status, gitea_merge_pull_request and gitea_events served nowhere. Until it is
|
||||
// listening, Pace keeps the polls at 30 s, so nothing is missed meanwhile; listenForForge says its own failure.
|
||||
void listenForForge(client);
|
||||
// The start's one pass over every repository (readall.ts), not awaited for the same reason. A forge not answering yet
|
||||
// is asked again, a minute apart, until one pass has run; after that only the forge's events grant.
|
||||
// An account of the list the forge does not hold yet (the forge-account provider makes nox-builder and nox-agent when
|
||||
// their first consumer asks, and the forge sends no event for a user made) is looked for again ten minutes apart
|
||||
// until every account was granted once; the line of each pass names it.
|
||||
const readAllAtStart = (): void => void Promise.resolve().then(() => readAll().atStart()).then((r) => {
|
||||
if (r.absent.length) setTimeout(readAllAtStart, 600_000);
|
||||
}, (err) => {
|
||||
console.error(`[gitea] whether the mesh's accounts read every repository could not be settled; asked again in a minute — ${err instanceof Error ? err.message : err}`);
|
||||
setTimeout(readAllAtStart, 60_000);
|
||||
});
|
||||
try {
|
||||
readAll();
|
||||
readAllAtStart();
|
||||
} catch (err) {
|
||||
console.error(`[gitea] the mesh's accounts are not given read on every repository: no site administrator's login — ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
// The indexer's modules, watched: the watch hands every key's latest value at its start and then each change, so a
|
||||
// repository that builds a registered module is made writable at the start and when its first module is registered
|
||||
// (writes.ts). A read not granted yet is said and asked again until it holds.
|
||||
try {
|
||||
const w = writes();
|
||||
void bindUntilHeld(INDEXER_MODULES, () => state<unknown>(INDEXER_MODULES).watch((change) => w.take(change as ModulesChange)));
|
||||
} catch (err) {
|
||||
console.error(`[gitea] the merging account is not given write on the delivered repositories: no site administrator's login — ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
// Main takes no push and only the tools' account merges, in every repository, kept and not only when a token is
|
||||
// minted: at the start and every ten minutes after, so a repository made later gets it too (issue 948, ADR 0398 §2).
|
||||
// Only while the setting keep-main-rules is true (mainrules.ts); false, the default, leaves every repository's rule as set.
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package forgeaccount
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"os"
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestForgeUserOfAMachineAccountIsTheOneAgentUser(t *testing.T) {
|
||||
if got := ForgeUser("g14", "machine"); got != "nox-agent" {
|
||||
@@ -15,6 +19,18 @@ func TestForgeUserOfAMachineAccountIsTheOneAgentUser(t *testing.T) {
|
||||
if got := NodeAccount("g14"); got != "agent-g14" {
|
||||
t.Fatalf("the token tool's per-node account = %q", got)
|
||||
}
|
||||
// The accounts that read every repository (readall.ts, meshAccounts in tools-account.ts) name these two users: the
|
||||
// list is held to the Go constants here, so the two cannot drift apart (the operator, 2026-10-26).
|
||||
ts, err := os.ReadFile("../../tools-account.ts")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, want := range map[string]string{"BUILDER_ACCOUNT": BuilderUser, "AGENT_ACCOUNT": AgentUser} {
|
||||
m := regexp.MustCompile(`export const ` + name + ` = "([^"]+)";`).FindSubmatch(ts)
|
||||
if m == nil || string(m[1]) != want {
|
||||
t.Errorf("tools-account.ts %s = %q, want %q (internal/forgeaccount)", name, m, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveKeysTitledTakesOnlyTheMeshKeysOfThatTitle(t *testing.T) {
|
||||
|
||||
@@ -15,6 +15,19 @@ export function keepMainRules(env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
/** The setting check-gates-merge (the operator, 2026-10-25 22:56: "run the check locally only, but not as a gate, it loses
|
||||
* us too much time"): whether the merge tool judges mesh/check before it merges. Absent, unreadable or anything but false
|
||||
* is true; false lets a head the operator's Approve names merge with mesh/check pending, failed, missing or judged on an
|
||||
* older trunk (mesh-delivery-gate's setting of the same name stops waiting for it before the Approve). */
|
||||
export function checkGatesMerge(env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
const file = env.MESH_GITEA_CONFIG_FILE;
|
||||
if (!file) return true;
|
||||
try {
|
||||
const v = (JSON.parse(readFileSync(file, "utf8")) as Record<string, unknown>)["check-gates-merge"];
|
||||
return !(v === false || v === "false");
|
||||
} catch { return true; }
|
||||
}
|
||||
|
||||
/** The 10-minute step: puts the rules right only when kept; else says once that they are left and writes nothing. */
|
||||
export function mainRulesStep(keep: boolean, put: () => Promise<string[]>, say: (line: string) => void): () => Promise<void> {
|
||||
let said = false;
|
||||
|
||||
@@ -19,6 +19,7 @@ import { mergeableUnder, newestByContext } from "./checks.js";
|
||||
import { MERGE_METHODS, Refused, checkName } from "./guard.js";
|
||||
import { CHECK_CONTEXT, CHECK_STATUS, REPO_CHECK_CONTEXT, maintained, requiredOf } from "./pulls.js";
|
||||
import { approvedHeadsOf, groupRefusal, memberKey, namesSiblings, notApproved, readSiblings, siblingOutOfDate, siblingsFile } from "./siblings.js";
|
||||
import { checkGatesMerge } from "./mainrules.js";
|
||||
import { announcementOf, askedWords, outOfDateWords, sameCommit, tookFile, tookWords, twelve, verdictAtTrunk, waitsForComposed } from "./trunk.js";
|
||||
|
||||
/** A head named for a merge: at least 12 hex characters of its sha (novox/hq issue 381), up to a sha-256 commit's 64. */
|
||||
@@ -31,11 +32,13 @@ export interface MergeDeps {
|
||||
emit?: (type: string, body: unknown) => Promise<void>;
|
||||
stateDir?: string;
|
||||
now?: () => Date;
|
||||
/** Whether mesh/check gates the merge: the setting check-gates-merge, read at each merge (mainrules.ts). */
|
||||
checkGates?: () => boolean;
|
||||
}
|
||||
|
||||
export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition {
|
||||
const deps = { ...given, emit: given.emit ?? announce, stateDir: given.stateDir ?? process.env.MESH_GITEA_STATE_DIR,
|
||||
now: given.now ?? (() => new Date()) };
|
||||
now: given.now ?? (() => new Date()), checkGates: given.checkGates ?? (() => checkGatesMerge()) };
|
||||
return {
|
||||
name: "gitea_merge_pull_request",
|
||||
description: "Merge a pull request at the head you name — the commit you reviewed — when that is still its head, " +
|
||||
@@ -126,7 +129,10 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
||||
}
|
||||
const kept = maintained(required);
|
||||
const byCheck = required.includes(CHECK_STATUS) || kept.length === 0;
|
||||
if (byCheck) {
|
||||
// **check-gates-merge false** (the operator, 2026-10-25): mesh/check is information, not a gate. A head the operator's
|
||||
// Approve names merges whatever its mesh/check says, pending, failed, missing or judged on an older trunk; without
|
||||
// approved_heads the check still judges.
|
||||
if (byCheck && (deps.checkGates() || !approved)) {
|
||||
// **mesh/check is the checker's** (ADR 0447 §4): it is judged here against the trunk's head, and refused when it
|
||||
// is not current; nothing is written or asked — the checker checks a head again when its trunk moves.
|
||||
const check = checks[CHECK_STATUS];
|
||||
|
||||
@@ -40,6 +40,7 @@
|
||||
],
|
||||
"emits": [
|
||||
"repo.created",
|
||||
"repo.writable",
|
||||
"issue.opened",
|
||||
"pull.merged",
|
||||
"pull.updated",
|
||||
@@ -181,6 +182,12 @@
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"check-gates-merge": {
|
||||
"type": "boolean",
|
||||
"description": "Whether gitea_merge_pull_request judges mesh/check before it merges. True (the default): a head merges only on a passing mesh/check judged against the trunk's head (ADR 0447, ADR 0534 §4). False (the operator, 2026-10-25 22:56: \"run the check locally only, but not as a gate, it loses us too much time\"): a head the operator's Approve names in approved_heads merges whatever its mesh/check says (pending, failed, missing or judged on an older trunk); a head that moved, a branch no rule guards and a merge git refuses are still refused. Set together with mesh-delivery-gate's check-gates-merge.",
|
||||
"trusted": true,
|
||||
"default": true
|
||||
},
|
||||
"main-mergers": {
|
||||
"type": "list",
|
||||
"description": "The forge logins that may merge into main beside the tools' account, on every repository of the organisation (ADR 0534 §2): the operator's own, so the operator can always merge by hand at the forge. The tools' account is always on main's merge whitelist; no agent's account belongs here.",
|
||||
@@ -316,7 +323,8 @@
|
||||
"takes-config": [
|
||||
"url",
|
||||
"keep-main-rules",
|
||||
"main-mergers"
|
||||
"main-mergers",
|
||||
"check-gates-merge"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -537,6 +545,7 @@
|
||||
"MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret",
|
||||
"MESH_GITEA_ORIGINS_FILE": "${dir:mesh-state}/origins",
|
||||
"MESH_GITEA_SSH_PORT": "222",
|
||||
"MESH_GITEA_HOST_KEY_FILE": "${dir:data}/ssh/ssh_host_ed25519_key.pub",
|
||||
"MESH_FORGE_HOST_FILE": "/etc/gitea/forge-host"
|
||||
}
|
||||
}
|
||||
@@ -559,6 +568,7 @@
|
||||
"mesh-build-go.keys",
|
||||
"mesh-build-node.keys",
|
||||
"mesh-build-image.keys",
|
||||
"mesh-delivery-checker.owed"
|
||||
"mesh-delivery-checker.owed",
|
||||
"mesh-delivery-indexer.modules"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
// The mesh's own forge accounts read every repository on the forge (the operator, 2026-10-26: "the mesh's forge account
|
||||
// should get read access to all repositories"; and "why wouldn't the forge account get access to all repo's?").
|
||||
//
|
||||
// **Every account the mesh's parts use, from one list** (MESH_ACCOUNTS, tools-account.ts): the tools' `mesh-tools`, the
|
||||
// build modules' `nox-builder` (their fetch over ssh, ADR 0466 §3) and the machines' `nox-agent` (ssh-client's key,
|
||||
// ADR 0410 §1). The first pass granted the tools' account alone, so the build modules' fetch of jschoubben/car-hunter
|
||||
// still failed on ace and g14 ("nox-builder ... is not authorized to read"). The two Go names are the ones of
|
||||
// internal/forgeaccount, whose test holds the list to them. An account the forge does not hold yet (its consumer never
|
||||
// asked) is said and left out, never a refusal per repository.
|
||||
//
|
||||
// Since issue 469 the tools act as the plain account `mesh-tools`, which writes to the organisation novox's
|
||||
// repositories through its team (tools-account.ts). Every other repository it saw only when public: a private
|
||||
// repository of a user (jschoubben/car-hunter, jschoubben/snake) answered its tools 404 and was missing from every
|
||||
// list, while that user's public ones were there. So the module makes its account a collaborator that reads only on
|
||||
// each repository it cannot read, the least the forge offers: one repository, read, no organisation membership.
|
||||
//
|
||||
// - **at the start**, one pass over every repository of every user and organisation;
|
||||
// - **on the forge's own event** (the system webhook, webhook.ts): `forge.repository.created` and `forge.fork` at
|
||||
// once, and the first event of any repository this process has not yet seen readable, since the forge may not
|
||||
// send a repository's creation for every owner. Nothing polls for it.
|
||||
//
|
||||
// **Not caught: a repository made private later.** The forge sends no event for a change of visibility, and a public
|
||||
// repository is already seen readable by the start's pass, so its events ask nothing. Such a repository is granted at
|
||||
// the module's next start, or now by `gitea_grant_read`; `gitea_unreadable_repos` shows it meanwhile.
|
||||
//
|
||||
// **The grant needs the site administrator's login.** Adding a collaborator asks the repository's administration,
|
||||
// which on another user's repository only the site administrator holds; the module holds that login for setting the
|
||||
// forge up (MESH_GITEA_ADMIN_PASSWORD_FILE). Here it is used through `ReadAllForge` alone: list the repositories, read
|
||||
// the account's permission on one, add the account as a collaborator that reads. Nothing else, and never for the tools'
|
||||
// own calls, which keep their token.
|
||||
//
|
||||
// What the forge refuses (a repository that went meanwhile, an archived one if the forge will not take a collaborator
|
||||
// on it) is said per repository and not retried; the tool `gitea_unreadable_repos` shows what is left, and
|
||||
// `gitea_grant_read` grants again now.
|
||||
|
||||
import type { ForgeRepository } from "./client.js";
|
||||
|
||||
/** What the module asks of the forge for this, with the site administrator's login (GiteaAdmin, client.ts). */
|
||||
export interface ReadAllForge {
|
||||
everyRepository(): Promise<ForgeRepository[]>;
|
||||
repository(owner: string, repo: string): Promise<ForgeRepository | null>;
|
||||
/** Whether the forge holds the account at all. */
|
||||
accountExists(user: string): Promise<boolean>;
|
||||
repositoryPermission(owner: string, repo: string, user: string): Promise<string>;
|
||||
addReadCollaborator(owner: string, repo: string, user: string): Promise<void>;
|
||||
}
|
||||
|
||||
/** A repository an account cannot read, and why. */
|
||||
export interface Unreadable {
|
||||
account: string;
|
||||
repository: string;
|
||||
permission: string;
|
||||
private: boolean;
|
||||
archived: boolean;
|
||||
why: string;
|
||||
}
|
||||
|
||||
export interface Granted {
|
||||
/** Repositories looked at (for each account). */
|
||||
checked: number;
|
||||
/** The accounts looked at: those the forge holds. */
|
||||
accounts: string[];
|
||||
/** Accounts of the list the forge does not hold yet: nothing is granted to them until it does. */
|
||||
absent: string[];
|
||||
/** Made readable now, by a read collaborator: each repository and the account. */
|
||||
granted: { account: string; repository: string }[];
|
||||
/** The forge refused the grant: each repository, the account and the forge's own words. */
|
||||
refused: { account: string; repository: string; why: string }[];
|
||||
}
|
||||
|
||||
const READS = new Set(["read", "write", "admin", "owner"]);
|
||||
|
||||
const said = (err: unknown): string => (err instanceof Error ? err.message : String(err));
|
||||
|
||||
/** Why the account cannot read a repository, from what the forge says of it. */
|
||||
export function whyUnreadable(repo: ForgeRepository, account: string): string {
|
||||
const parts: string[] = [];
|
||||
if (repo.private) parts.push(`a private repository of ${repo.owner}, on which ${account} is no collaborator and in no team that covers it`);
|
||||
else if (repo.owner_visibility !== "public") {
|
||||
parts.push(`a public repository of ${repo.owner}, whose own visibility is ${repo.owner_visibility}: the forge shows it to members only`);
|
||||
} else parts.push(`the forge answers ${account} no read on it`);
|
||||
if (repo.archived) parts.push("archived");
|
||||
return parts.join("; ");
|
||||
}
|
||||
|
||||
export class ReadAll {
|
||||
/** Per account, the repositories this process has seen it read, by full name: an event of a repository every account
|
||||
* was seen reading asks nothing more. */
|
||||
readonly readable = new Map<string, Set<string>>();
|
||||
readonly accounts: readonly string[];
|
||||
|
||||
constructor(private readonly forge: ReadAllForge, accounts: readonly string[] | string,
|
||||
private readonly log: (line: string) => void = (l) => console.log(`[gitea] ${l}`)) {
|
||||
this.accounts = [...new Set(typeof accounts === "string" ? [accounts] : accounts)];
|
||||
if (!this.accounts.length) throw new Error("no forge account to give read: the list is empty");
|
||||
for (const a of this.accounts) this.readable.set(a, new Set());
|
||||
}
|
||||
|
||||
/** The accounts of the list (or of those named) the forge holds, and those it does not hold yet. */
|
||||
private async present(only?: string[]): Promise<{ accounts: string[]; absent: string[] }> {
|
||||
const wanted = only && only.length ? only : [...this.accounts];
|
||||
for (const a of wanted) if (!this.accounts.includes(a)) throw new Error(`"${a}" is not one of the mesh's accounts: ${this.accounts.join(", ")}`);
|
||||
const accounts: string[] = [];
|
||||
const absent: string[] = [];
|
||||
for (const a of wanted) ((await this.forge.accountExists(a)) ? accounts : absent).push(a);
|
||||
return { accounts, absent };
|
||||
}
|
||||
|
||||
/** Every repository each account cannot read, and why, as the forge says it now. */
|
||||
async unreadable(only?: string[]): Promise<{ checked: number; accounts: { account: string; unreadable: number }[]; absent: string[]; unreadable: Unreadable[] }> {
|
||||
const { accounts, absent } = await this.present(only);
|
||||
const repos = await this.forge.everyRepository();
|
||||
const out: Unreadable[] = [];
|
||||
for (const account of accounts) {
|
||||
for (const repo of repos) {
|
||||
try {
|
||||
const u = await this.judge(repo, account);
|
||||
if (u) out.push(u);
|
||||
} catch (err) {
|
||||
// A permission the forge would not answer is said for that repository; the rest are still looked at.
|
||||
out.push({ account, repository: repo.full_name, permission: "unknown", private: repo.private, archived: repo.archived,
|
||||
why: `the forge did not answer ${account}'s permission on it — ${said(err)}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
return { checked: repos.length, accounts: accounts.map((account) => ({ account, unreadable: out.filter((u) => u.account === account).length })),
|
||||
absent, unreadable: out };
|
||||
}
|
||||
|
||||
/** Grant read on every repository an account cannot read (or on those named, `owner/name`), for every account of the
|
||||
* list the forge holds (or for those named). */
|
||||
async grant(only?: string[], accountsOnly?: string[]): Promise<Granted> {
|
||||
const { accounts, absent } = await this.present(accountsOnly);
|
||||
let repos: ForgeRepository[];
|
||||
if (only && only.length) {
|
||||
repos = [];
|
||||
for (const full of only) {
|
||||
const [owner, name, ...rest] = full.split("/");
|
||||
if (!owner || !name || rest.length) throw new Error(`"${full}" is not a repository: say owner/name`);
|
||||
const repo = await this.forge.repository(owner, name);
|
||||
if (!repo) throw new Error(`the forge holds no repository ${full}`);
|
||||
repos.push(repo);
|
||||
}
|
||||
} else repos = await this.forge.everyRepository();
|
||||
const result: Granted = { checked: repos.length, accounts, absent, granted: [], refused: [] };
|
||||
for (const account of accounts) {
|
||||
for (const repo of repos) {
|
||||
const repository = repo.full_name;
|
||||
let u: Unreadable | null;
|
||||
try {
|
||||
u = await this.judge(repo, account);
|
||||
} catch (err) {
|
||||
// One repository whose permission cannot be read is a refusal of its own, never the end of the pass.
|
||||
result.refused.push({ account, repository, why: `its permission could not be read — ${said(err)}` });
|
||||
continue;
|
||||
}
|
||||
if (!u) continue;
|
||||
try {
|
||||
await this.forge.addReadCollaborator(repo.owner, repo.name, account);
|
||||
} catch (err) {
|
||||
result.refused.push({ account, repository, why: said(err) });
|
||||
continue;
|
||||
}
|
||||
// Read back: the forge's own word that the account reads it now, never the grant's answer alone.
|
||||
let after: string;
|
||||
try {
|
||||
after = await this.forge.repositoryPermission(repo.owner, repo.name, account);
|
||||
} catch (err) {
|
||||
result.refused.push({ account, repository, why: `granted, but its permission could not be read back — ${said(err)}` });
|
||||
continue;
|
||||
}
|
||||
if (READS.has(after)) {
|
||||
this.readable.get(account)!.add(repository);
|
||||
result.granted.push({ account, repository });
|
||||
} else {
|
||||
result.refused.push({ account, repository, why: `the grant was answered, but the forge still gives ${account} "${after}"` });
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** The start's one pass: grant what is missing and say it. Answers the result; a failure is the caller's to say. */
|
||||
async atStart(): Promise<Granted> {
|
||||
const r = await this.grant();
|
||||
const per = r.accounts.map((a) => {
|
||||
const g = r.granted.filter((x) => x.account === a).map((x) => x.repository);
|
||||
return `${a} ${g.length} granted read now${g.length ? ` (${g.join(", ")})` : ""}`;
|
||||
});
|
||||
this.log(`the mesh's accounts read every repository: ${r.checked} looked at; ${per.join("; ")}` +
|
||||
`${r.absent.length ? `; not on the forge yet, so not granted: ${r.absent.join(", ")}` : ""}` +
|
||||
`${r.refused.length ? `; ${r.refused.length} refused: ` + r.refused.map((x) => `${x.account} on ${x.repository} — ${x.why}`).join("; ") : ""}`);
|
||||
return r;
|
||||
}
|
||||
|
||||
/** A forge event: a repository made (or forked), or the first event of one some account was not seen reading, is
|
||||
* granted read now to every account that cannot read it. */
|
||||
async onEvent(event: { name: string; body: unknown }): Promise<void> {
|
||||
const b = (event.body ?? {}) as { owner?: string; repo?: string };
|
||||
if (!b.owner || !b.repo) return;
|
||||
const full = `${b.owner}/${b.repo}`;
|
||||
if (event.name === "forge.repository.deleted") {
|
||||
for (const seen of this.readable.values()) seen.delete(full);
|
||||
return;
|
||||
}
|
||||
const made = event.name === "forge.repository.created" || event.name === "forge.fork";
|
||||
// Seen readable once by every account, its events ask nothing: so a repository made private after that is not
|
||||
// caught here (the forge says no change of visibility), only at the next start or by gitea_grant_read.
|
||||
if (!made && [...this.readable.values()].every((seen) => seen.has(full))) return;
|
||||
const r = await this.grant([full]);
|
||||
for (const x of r.granted) this.log(`${x.account} was given read on ${full}, on the forge's ${event.name}`);
|
||||
for (const x of r.refused) this.log(`${x.account} could not be given read on ${x.repository} — ${x.why}`);
|
||||
}
|
||||
|
||||
private async judge(repo: ForgeRepository, account: string): Promise<Unreadable | null> {
|
||||
const permission = await this.forge.repositoryPermission(repo.owner, repo.name, account);
|
||||
const seen = this.readable.get(account)!;
|
||||
if (READS.has(permission)) {
|
||||
seen.add(repo.full_name);
|
||||
return null;
|
||||
}
|
||||
seen.delete(repo.full_name);
|
||||
return { account, repository: repo.full_name, permission, private: repo.private, archived: repo.archived, why: whyUnreadable(repo, account) };
|
||||
}
|
||||
}
|
||||
|
||||
type Tool = { name: string; description: string; input: Record<string, unknown>; run: (args: Readonly<Record<string, unknown>>) => Promise<unknown> };
|
||||
|
||||
const strings = (v: unknown): string[] | undefined => (Array.isArray(v) ? v.map(String) : undefined);
|
||||
|
||||
/** The two tools: what each account cannot read and why, and the grant now. */
|
||||
export function readAllTools(readAll: () => ReadAll): Tool[] {
|
||||
return [
|
||||
{
|
||||
name: "gitea_unreadable_repos",
|
||||
description: "Every repository on the forge one of the mesh's own forge accounts cannot read, and why, per account: " +
|
||||
"the tools' (mesh-tools), the build modules' fetch over ssh (nox-builder) and the machines' agents' (nox-agent). " +
|
||||
"For each, the permission the forge gives it there, whether the repository is private or archived, and in words " +
|
||||
"the reason (a private repository of a user with the account no collaborator, an owner not public); a count per " +
|
||||
"account, and the accounts the forge does not hold yet. Empty when every account reads every repository. Read " +
|
||||
"with the site administrator's login, which alone sees every repository.",
|
||||
input: {
|
||||
accounts: { type: "array", items: { type: "string" }, description: "the accounts to look at; empty or absent, every one of the mesh's" },
|
||||
},
|
||||
run: async (args) => readAll().unreadable(strings(args.accounts)),
|
||||
},
|
||||
{
|
||||
name: "gitea_grant_read",
|
||||
description: "Give each of the mesh's own forge accounts (mesh-tools, nox-builder, nox-agent) read on every " +
|
||||
"repository it cannot read now, or on the repositories named, or for the accounts named: each becomes a " +
|
||||
"collaborator that reads only, the least the forge offers. The module does this itself at its start and on the " +
|
||||
"forge's event of a repository made; this is the same act by hand, such as when a build's fetch says its account " +
|
||||
"is not authorized to read a repository. Answers how many were looked at, which were granted to which account " +
|
||||
"(each read back from the forge), which the forge refused, in its own words, and the accounts it does not hold " +
|
||||
"yet. Needs the site administrator's login, used for this grant alone.",
|
||||
input: {
|
||||
repositories: { type: "array", items: { type: "string" }, description: "owner/name of each repository; empty or absent, every repository" },
|
||||
accounts: { type: "array", items: { type: "string" }, description: "the accounts to grant; empty or absent, every one of the mesh's" },
|
||||
},
|
||||
run: async (args) => readAll().grant(strings(args.repositories), strings(args.accounts)),
|
||||
},
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// Who is in a forge team and what the team may do (issue 1563; novox/hq ADR 0466 §3, issue 971).
|
||||
//
|
||||
// The build keys sit on one forge user, nox-builder, in the team `builders`, which reads every repository and pushes
|
||||
// nothing. The tools' account cannot read a team (the forge answers /orgs/novox/teams 403: it lacks read:organization),
|
||||
// so the read goes with the site administrator's login, and only the read, as the branch protection read does (ADR 0375):
|
||||
// GET /orgs/{org}/teams, GET /teams/{id}/members and, for a user's effective right on a repository, the repository as
|
||||
// that user sees it (GiteaAdmin.repositoryPermission, which counts access through a team). Nothing is written.
|
||||
|
||||
/** A team as the forge's Team object answers it (GET /orgs/{org}/teams): the fields read here. */
|
||||
export interface ForgeTeam {
|
||||
id: number;
|
||||
name: string;
|
||||
description?: string;
|
||||
/** The team's overall permission: none, read, write, admin, owner. */
|
||||
permission?: string;
|
||||
includes_all_repositories?: boolean;
|
||||
can_create_org_repo?: boolean;
|
||||
/** Per unit, the right the team has: units_map {"repo.code": "read", ...}. */
|
||||
units_map?: Record<string, string>;
|
||||
units?: string[];
|
||||
}
|
||||
|
||||
/** What the module asks of the forge for this, with the site administrator's login (GiteaAdmin, client.ts). */
|
||||
export interface TeamForge {
|
||||
teams(org: string): Promise<ForgeTeam[]>;
|
||||
teamMembers(teamId: number): Promise<string[]>;
|
||||
accountExists(user: string): Promise<boolean>;
|
||||
everyRepository(): Promise<{ full_name: string; owner: string; name: string }[]>;
|
||||
repositoryPermission(owner: string, repo: string, user: string): Promise<string>;
|
||||
}
|
||||
|
||||
export interface TeamRights {
|
||||
team: string;
|
||||
id: number;
|
||||
description: string;
|
||||
members: string[];
|
||||
/** The team's overall permission. */
|
||||
permission: string;
|
||||
includes_all_repositories: boolean;
|
||||
can_create_org_repo: boolean;
|
||||
/** Per unit, read / write / admin / none. */
|
||||
units: Record<string, string>;
|
||||
/** True when no unit of the team is above read: the team pushes nothing, writes no issue, no pull request. */
|
||||
read_only: boolean;
|
||||
}
|
||||
|
||||
const RANK: Record<string, number> = { none: 0, read: 1, write: 2, admin: 3, owner: 4 };
|
||||
|
||||
/** One team as a person reads it. A team the forge gives no units_map is read from its `permission`, which is what the
|
||||
* older forge versions answer. */
|
||||
export function teamRights(t: ForgeTeam, members: string[]): TeamRights {
|
||||
const units = t.units_map && Object.keys(t.units_map).length ? { ...t.units_map }
|
||||
: Object.fromEntries((t.units ?? []).map((u) => [u, t.permission ?? "none"]));
|
||||
const highest = Math.max(0, ...Object.values(units).map((r) => RANK[r] ?? 0));
|
||||
return {
|
||||
team: t.name, id: t.id, description: t.description ?? "", members: [...members].sort(), permission: t.permission ?? "none",
|
||||
includes_all_repositories: t.includes_all_repositories === true, can_create_org_repo: t.can_create_org_repo === true,
|
||||
units, read_only: highest <= RANK.read && (RANK[t.permission ?? "none"] ?? 0) <= RANK.read,
|
||||
};
|
||||
}
|
||||
|
||||
export interface UserRights {
|
||||
user: string;
|
||||
/** The teams of the organisation the user is a member of. */
|
||||
teams: string[];
|
||||
/** How many repositories the forge gives the user each right on, effective (through a team or as a collaborator). */
|
||||
repositories: Record<string, number>;
|
||||
/** The repositories where the user may do more than read: empty for a read-only user. */
|
||||
writes: { repository: string; permission: string }[];
|
||||
/** Every repository and the right on it; only when asked. */
|
||||
every?: { repository: string; permission: string }[];
|
||||
}
|
||||
|
||||
/** The teams of an organisation (or the one named), each with its members and rights; and for one user, the teams it is
|
||||
* in and its effective right on every repository (or on those named), as the forge says it. Only reads. */
|
||||
export async function readTeamRights(forge: TeamForge, org: string, opts: { team?: string; user?: string; repositories?: string[]; list?: boolean } = {}):
|
||||
Promise<{ organisation: string; teams: TeamRights[]; user?: UserRights }> {
|
||||
if (opts.user && !(await forge.accountExists(opts.user))) throw new Error(`no forge user ${opts.user}`);
|
||||
const all = await forge.teams(org);
|
||||
const named = opts.team ? all.filter((t) => t.name === opts.team) : all;
|
||||
if (opts.team && !named.length) throw new Error(`no team "${opts.team}" in ${org}: it holds ${all.map((t) => t.name).join(", ") || "none"}`);
|
||||
// Membership is read for every team when a user is asked, so the user's teams are all found.
|
||||
const wanted = opts.user ? all : named;
|
||||
const rights: TeamRights[] = [];
|
||||
for (const t of wanted) rights.push(teamRights(t, await forge.teamMembers(t.id)));
|
||||
const shown = opts.team ? rights.filter((r) => r.team === opts.team) : rights;
|
||||
const out: { organisation: string; teams: TeamRights[]; user?: UserRights } = { organisation: org, teams: shown };
|
||||
if (!opts.user) return out;
|
||||
const user = opts.user;
|
||||
const mine = rights.filter((r) => r.members.includes(user)).map((r) => r.team);
|
||||
const repos = (await forge.everyRepository()).filter((r) => !opts.repositories?.length || opts.repositories.includes(r.full_name));
|
||||
const counts: Record<string, number> = {};
|
||||
const writes: { repository: string; permission: string }[] = [];
|
||||
const every: { repository: string; permission: string }[] = [];
|
||||
for (const r of repos) {
|
||||
const permission = await forge.repositoryPermission(r.owner, r.name, user);
|
||||
counts[permission] = (counts[permission] ?? 0) + 1;
|
||||
if ((RANK[permission] ?? 0) > RANK.read) writes.push({ repository: r.full_name, permission });
|
||||
every.push({ repository: r.full_name, permission });
|
||||
}
|
||||
out.user = { user, teams: mine, repositories: counts, writes, ...(opts.list ? { every } : {}) };
|
||||
return out;
|
||||
}
|
||||
@@ -109,3 +109,53 @@ test("a bundle whose read of the checker's owed is granted late writes the lates
|
||||
assert.match(got.statuses[0], /"state":"success".*pass at 4/);
|
||||
assert.match(err, /mesh-delivery-checker\.owed could not be bound/, "the refusal is said");
|
||||
});
|
||||
|
||||
// **The bundle's import does not wait on the forge** (novox/hq issue 1726, 2026-10-24 12:37). The webhook receiver's set-up
|
||||
// asks the forge's admin API for its system webhook; awaited at the top of index.ts, a forge slow to answer (busy while
|
||||
// every bundle of the node starts) held the import past node-tools' 10 s handshake, the bundle was killed and never
|
||||
// started again, and gitea_commit_status, gitea_merge_pull_request and gitea_events were served nowhere: the forge
|
||||
// bridge's calls found no responders and no new pull request was announced. Here the forge stand-in takes every
|
||||
// request and never answers: the import resolves at once all the same, with the bundle's tools registered.
|
||||
test("importing the bundle resolves, its tools registered, while the forge never answers the webhook's set-up", async () => {
|
||||
const index = pathToFileURL(join(here, "..", "dist", "index.js")).href;
|
||||
const script = `
|
||||
const http = await import("node:http");
|
||||
const fs = await import("node:fs");
|
||||
const os = await import("node:os");
|
||||
const path = await import("node:path");
|
||||
const forge = http.createServer(() => {});
|
||||
await new Promise((r) => forge.listen(0, "127.0.0.1", r));
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "gitea-1726-"));
|
||||
fs.writeFileSync(path.join(dir, "admin.password"), "pw");
|
||||
process.env.MESH_GITEA_URL = "http://127.0.0.1:" + forge.address().port;
|
||||
process.env.MESH_GITEA_STATE_DIR = dir;
|
||||
process.env.MESH_GITEA_ADMIN_PASSWORD_FILE = path.join(dir, "admin.password");
|
||||
process.env.MESH_GITEA_HOOK_PORT = "0";
|
||||
process.env.MESH_GITEA_HOOK_HOST = "127.0.0.1";
|
||||
const { useBroker } = await import("@novox/mesh-sdk/messaging");
|
||||
useBroker(() => ({
|
||||
subscribe: async () => () => {}, publish: async () => {}, request: async () => { throw new Error("no responders"); },
|
||||
handle: async () => () => {}, close: async () => {},
|
||||
}));
|
||||
await import(${JSON.stringify(index)});
|
||||
const { collectTools } = await import("@novox/mesh-sdk/tools");
|
||||
console.log(JSON.stringify({ imported: true, tools: collectTools().flatMap((r) => r.tools.map((t) => t.name)) }));
|
||||
process.exit(0);`;
|
||||
const env = { ...process.env, MESH_GITEA_TOKEN: "x", MESH_GITEA_ADMIN_USER: "admin", MESH_GITEA_HOOK_URL: "http://127.0.0.1:1" };
|
||||
for (const k of ["MESH_SERVED_MODULE", "MESH_GITEA_ADMIN_PASSWORD", "MESH_GITEA_CONFIG_FILE"]) delete env[k];
|
||||
const child = spawn(process.execPath, ["--input-type=module", "-e", script], { cwd: join(here, ".."), env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let out = "";
|
||||
let err = "";
|
||||
child.stdout.on("data", (d) => (out += d));
|
||||
child.stderr.on("data", (d) => (err += d));
|
||||
const ended = new Promise<void>((r) => child.on("exit", () => r()));
|
||||
const timer = setTimeout(() => child.kill("SIGKILL"), 3000);
|
||||
await ended;
|
||||
clearTimeout(timer);
|
||||
const line = out.split("\n").find((l) => l.startsWith("{"));
|
||||
assert.ok(line, `the import did not resolve within 3 s while the forge did not answer; stdout: ${out.slice(0, 400)} stderr: ${err.slice(0, 600)}`);
|
||||
const got = JSON.parse(line!) as { tools: string[] };
|
||||
for (const t of ["gitea_events", "gitea_commit_status", "gitea_merge_pull_request"]) {
|
||||
assert.ok(got.tools.includes(t), `${t} is registered: ${got.tools}`);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -133,3 +133,45 @@ test("no tool writes mesh/check: gitea_check_status is gone and gitea_commit_sta
|
||||
const index = readFileSync(new URL("../index.ts", import.meta.url), "utf8");
|
||||
assert.ok(!/gitea_check_status|checkStatusTool/.test(index), "index.ts serves no gitea_check_status");
|
||||
});
|
||||
|
||||
test("the watch's current values are taken without waiting on the forge, so a slow forge never times the watch out (issue 1726)", async () => {
|
||||
// The tool runtime hands a watch's current values one at a time and waits for each to be taken; it gives the whole set
|
||||
// 10 s (mesh-tools StateTimeout) and otherwise ends the watch. On 2026-10-23 the owed held ~200 keys, each written to the
|
||||
// forge before the next was handed over: the set took longer than 10 s, the watch ended at every attempt, and no
|
||||
// verdict reached the forge's mesh/check again.
|
||||
const { CheckStatuses, watchOwed } = await import("../check.ts");
|
||||
const sent: string[] = [];
|
||||
let release: () => void = () => {};
|
||||
const slow = new Promise<void>((r) => { release = r; });
|
||||
const f = { setCommitStatus: async (_o: string, _r: string, s: string, st: any) => { await slow; sent.push(`${s.slice(0, 4)} ${st.state}`); } };
|
||||
const checks = new CheckStatuses(f, () => {}, noSleep);
|
||||
const keys = Array.from({ length: 200 }, (_, i) => key(i.toString(16).padStart(4, "0") + sha.slice(4), "novox/mesh-sdk", i + 1));
|
||||
let handedOver = 0;
|
||||
let attempts = 0;
|
||||
const watch = async (handler: (c: any) => Promise<void>) => {
|
||||
if (++attempts > 1) return; // a second attempt is not what this test waits for: the first must hold
|
||||
// The runtime's bound on the current values: the forge has answered none of them yet.
|
||||
const timedOut = new Promise<"timeout">((r) => setTimeout(() => r("timeout"), 200));
|
||||
const delivered = (async () => { for (const [i, k] of keys.entries()) { await handler(put(k, owed(), i + 1, true)); handedOver++; } return "delivered" as const; })();
|
||||
if (await Promise.race([delivered, timedOut]) === "timeout") throw new Error("the current values of mesh-delivery-checker.owed did not arrive in 10s");
|
||||
};
|
||||
const said: string[] = [];
|
||||
await watchOwed(watch, checks, noSleep, (l) => said.push(l));
|
||||
assert.equal(handedOver, 200, "every current value was taken before the forge answered");
|
||||
assert.ok(!said.some((l) => /could not be bound/.test(l)), said.join(" | "));
|
||||
assert.equal(sent.length, 0, "nothing is written before the forge answers");
|
||||
release();
|
||||
await checks.settled();
|
||||
assert.equal(sent.length, 200, "every owed status is written once the forge answers");
|
||||
});
|
||||
|
||||
test("the writes a watch's start hands over go to the forge a few at a time, never all at once (issue 1726)", async () => {
|
||||
const { CheckStatuses, WRITES_AT_ONCE } = await import("../check.ts");
|
||||
let running = 0, most = 0, done = 0;
|
||||
const f = { setCommitStatus: async () => { running++; most = Math.max(most, running); await new Promise((r) => setTimeout(r, 1)); running--; done++; } };
|
||||
const checks = new CheckStatuses(f, () => {}, noSleep);
|
||||
for (let i = 0; i < 50; i++) await checks.take(put(key(i.toString(16).padStart(4, "0") + sha.slice(4), "novox/mesh-sdk", i + 1), owed(), i + 1, true));
|
||||
await checks.settled();
|
||||
assert.equal(done, 50);
|
||||
assert.equal(most, WRITES_AT_ONCE);
|
||||
});
|
||||
|
||||
@@ -183,6 +183,15 @@ test("named paths: each asked path's git object id at the commit, a file's or a
|
||||
});
|
||||
assert.deepEqual(got.trees, { "": ROOT, "modules/a": h("4"), "modules/b/c": h("8") });
|
||||
assert.equal((await client.commitTrees("novox", "hq", COMMIT)).paths, undefined, "no paths asked, none answered");
|
||||
// Listed directories (ADR 0549 §2, issue 1126): the files directly under each, not deeper; a missing one lists none.
|
||||
const listed = await client.commitTrees("novox", "hq", COMMIT, undefined, ["modules/a", "modules/b", "", "placement"]);
|
||||
assert.deepEqual(listed.listed, {
|
||||
"modules/a": { "main.go": h("6"), "module.json": h("5") },
|
||||
"modules/b": {},
|
||||
"": { "README.md": h("1"), "module.json": h("2") },
|
||||
placement: {},
|
||||
});
|
||||
assert.equal((await client.commitTrees("novox", "hq", COMMIT)).listed, undefined, "no directory asked, none listed");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
@@ -209,7 +218,12 @@ test("gitea_commit_trees takes paths as {directory: [paths]} and refuses another
|
||||
const { client, server, asked } = await serve([{ sha: COMMIT, url: "u", tree: LISTING, truncated: false, page: 1, total_count: LISTING.length }]);
|
||||
try {
|
||||
const tool = getGiteaTools(client).find((t) => t.name === "gitea_commit_trees")!;
|
||||
assert.ok("paths" in tool.input, "the tool names the argument");
|
||||
assert.ok("paths" in tool.input && "listed" in tool.input, "the tool names the arguments");
|
||||
const listedAnswer: any = await tool.run({ owner: "novox", repo: "hq", sha: COMMIT, listed: ["modules/d"] } as any);
|
||||
assert.deepEqual(listedAnswer.listed, { "modules/d": { "notes.md": h("b") } });
|
||||
for (const bad of ["modules/d", ["../x"], ["/abs"], [1]]) {
|
||||
await assert.rejects(tool.run({ owner: "novox", repo: "hq", sha: COMMIT, listed: bad } as any), /listed/, JSON.stringify(bad));
|
||||
}
|
||||
const got: any = await tool.run({ owner: "novox", repo: "hq", sha: COMMIT, paths: { "modules/a": ["main.go"] } } as any);
|
||||
assert.deepEqual(got.paths, { "modules/a": { "main.go": h("6") } });
|
||||
const before = asked.length;
|
||||
|
||||
@@ -525,6 +525,18 @@ test("gitea_merge_pull_request judges mesh/check on a main requiring no status,
|
||||
|
||||
state.statuses = line(TRUNK);
|
||||
assert.equal(((await merge()) as Record<string, unknown>).merged, true);
|
||||
|
||||
// check-gates-merge false (the operator, 2026-10-25): a head the Approve names merges whatever its mesh/check says;
|
||||
// without approved_heads it is refused as before.
|
||||
const ungated = mergeTool(client, { gates, stateDir, emit: async () => {}, checkGates: () => false });
|
||||
const mergeUngated = (args: Record<string, unknown> = { approved_heads }) =>
|
||||
ungated.run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED, ...args });
|
||||
for (const statuses of [[], line(TRUNK, "pending", "checking"), line(TRUNK, "failure", "fail"), line(MOVED)]) {
|
||||
state.merged = false;
|
||||
state.statuses = statuses;
|
||||
await assert.rejects(mergeUngated({}), /Approve/);
|
||||
assert.equal(((await mergeUngated()) as Record<string, unknown>).merged, true, JSON.stringify(statuses));
|
||||
}
|
||||
});
|
||||
|
||||
test("a rule named for a branch is not made weaker than the glob rule that guards it now", async () => {
|
||||
@@ -628,6 +640,7 @@ test("no tool's caller text becomes another path or another user: `..` and `?sud
|
||||
await tool("gitea_delete_branch").run({ ...repo, branch: "feat/x" });
|
||||
await tool("gitea_get_file").run({ ...repo, path: "modules/gitea/module.json" }).catch(() => undefined);
|
||||
assert.deepEqual(state.asked, ["DELETE /repos/novox/mesh-catalog/branches/feat/x",
|
||||
"GET /repos/novox/mesh-catalog/contents/modules/gitea", // the file's size first (issue 1577)
|
||||
"GET /repos/novox/mesh-catalog/contents/modules/gitea/module.json"]);
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { createServer } from "node:http";
|
||||
|
||||
// **The forge's 405 "Please try again later" on a merge is asked again briefly** (novox/hq issue 1275, 2026-10-26 13:40):
|
||||
// mesh-delivery#73's merge was answered so a few seconds after its base moved, and counted as refused at once. The forge
|
||||
// says it while it works out whether the pull request merges; the merge is asked again after a short wait, and every other
|
||||
// refusal is answered at once. A pull request's own mergeable is carried, so a group can be read whole before it merges.
|
||||
|
||||
/** A forge whose merge of novox/hq#7 answers each of `answers` in turn (the last for ever), and whose pull request says
|
||||
* `mergeable`. */
|
||||
async function serve(answers: { status: number; body: string }[], mergeable?: boolean) {
|
||||
const { GiteaClient } = await import("../client.ts");
|
||||
let merges = 0;
|
||||
const server = createServer((req, res) => {
|
||||
const url = new URL(req.url ?? "", "http://x");
|
||||
if (url.pathname === "/api/v1/repos/novox/hq/pulls/7/merge" && req.method === "POST") {
|
||||
const a = answers[Math.min(merges++, answers.length - 1)];
|
||||
res.statusCode = a.status;
|
||||
return res.end(a.body);
|
||||
}
|
||||
if (url.pathname === "/api/v1/repos/novox/hq/pulls/7") {
|
||||
return res.end(JSON.stringify({ number: 7, state: "open", merged: false, base: { ref: "main" },
|
||||
head: { ref: "b", sha: "a".repeat(40) }, html_url: "h/7", ...(mergeable === undefined ? {} : { mergeable }) }));
|
||||
}
|
||||
res.statusCode = 404;
|
||||
res.end("{}");
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, r));
|
||||
const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t");
|
||||
GiteaClient.mergeRetryAfter = [1, 1, 1];
|
||||
return { client, server, merges: () => merges };
|
||||
}
|
||||
|
||||
const later = { status: 405, body: '{"message":"Please try again later"}' };
|
||||
const ok = { status: 200, body: "" };
|
||||
|
||||
test("a merge answered 'try again later' while the forge works out its mergeability, then merged", async (t) => {
|
||||
const cases: { name: string; answers: { status: number; body: string }[]; merged: boolean; asked: number }[] = [
|
||||
{ name: "merged on the third ask", answers: [later, later, ok], merged: true, asked: 3 },
|
||||
{ name: "still 'later' after the last wait: refused, with the forge's words", answers: [later], merged: false, asked: 4 },
|
||||
{ name: "a refusal of another kind is answered at once", answers: [{ status: 409, body: '{"message":"head moved"}' }], merged: false, asked: 1 },
|
||||
];
|
||||
for (const c of cases) {
|
||||
await t.test(c.name, async () => {
|
||||
const { client, server, merges } = await serve(c.answers);
|
||||
try {
|
||||
const done = client.mergePullRequest("novox", "hq", 7, "merge", false, "a".repeat(40));
|
||||
if (c.merged) await done;
|
||||
else await assert.rejects(done, (err: Error) => /40[59]/.test(err.message));
|
||||
assert.equal(merges(), c.asked);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("a pull request carries the forge's mergeable, and none when the forge did not say", async () => {
|
||||
for (const mergeable of [true, false, undefined]) {
|
||||
const { client, server } = await serve([ok], mergeable);
|
||||
try {
|
||||
assert.equal((await client.getPullRequest("novox", "hq", 7)).mergeable, mergeable);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -90,3 +90,48 @@ test("a pass pages until it reaches back past its floor, and no further", async
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
// **One forge request never holds the others, and a file too big for an answer is not downloaded** (novox/hq issue 1577): a
|
||||
// request for a 4.9 MB file never ended and every read through the module timed out behind it.
|
||||
test("a request the forge never answers is cut at the limit and says which request", async () => {
|
||||
const { GiteaClient } = await import("../client.ts");
|
||||
const server = createServer((req) => { if (!req.url?.includes("fast")) return; });
|
||||
await new Promise<void>((r) => server.listen(0, r));
|
||||
const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t");
|
||||
client.requestTimeoutMs = 150;
|
||||
try {
|
||||
const started = Date.now();
|
||||
await assert.rejects(client.api("/repos/novox/hq/pulls"), /Gitea API GET \/repos\/novox\/hq\/pulls: no answer in 0\.15 s/);
|
||||
assert.ok(Date.now() - started < 3000);
|
||||
} finally {
|
||||
server.closeAllConnections();
|
||||
await new Promise<void>((r) => server.close(() => r()));
|
||||
}
|
||||
});
|
||||
|
||||
test("a file over 512 KiB is refused from its directory listing, and its body is never asked for", async () => {
|
||||
const { GiteaClient } = await import("../client.ts");
|
||||
const asked: string[] = [];
|
||||
const server = createServer((req, res) => {
|
||||
asked.push(req.url ?? "");
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
if (req.url?.startsWith("/api/v1/repos/novox/hq/contents/bin?")) {
|
||||
return res.end(JSON.stringify([{ path: "bin/big", type: "file", size: 5_000_000 }, { path: "bin/small", type: "file", size: 5 }]));
|
||||
}
|
||||
if (req.url?.startsWith("/api/v1/repos/novox/hq/contents/bin/small")) {
|
||||
return res.end(JSON.stringify({ type: "file", encoding: "base64", sha: "s", size: 5, content: Buffer.from("hello").toString("base64") }));
|
||||
}
|
||||
res.statusCode = 404;
|
||||
res.end("{}");
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, r));
|
||||
const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t");
|
||||
try {
|
||||
await assert.rejects(client.getFile("novox", "hq", "bin/big", "main"),
|
||||
/the file is 5000000 bytes, over the 512 KiB a tool answer can carry; read it from a clone/);
|
||||
assert.ok(!asked.some((u) => u.includes("/contents/bin/big")), asked.join("\n"));
|
||||
assert.equal((await client.getFile("novox", "hq", "bin/small", "main")).content, "hello");
|
||||
} finally {
|
||||
await new Promise<void>((r) => server.close(() => r()));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
// Issue 1563: no tool read a forge team's members and rights, so the build keys' read-only user (nox-builder, team
|
||||
// builders, ADR 0466 §3) could not be checked live. The tools' token answers /orgs/novox/teams 403; the read goes with
|
||||
// the administrator's login, GET only.
|
||||
//
|
||||
// The answers below keep the shape of the forge's own API (gitea's swagger, "Team": id, name, description,
|
||||
// organization, includes_all_repositories, permission, units, units_map, can_create_org_repo; "User": id, login, ...;
|
||||
// repos/search: {ok, data:[Repository]}; Repository.permissions {admin, push, pull}). The live answer is read once the
|
||||
// tool is delivered (issue 1563 AC2).
|
||||
|
||||
import { test, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer } from "node:http";
|
||||
|
||||
import { GiteaAdmin, GiteaClient } from "../dist/client.js";
|
||||
import { ConfiguredToken } from "../dist/token.js";
|
||||
import { getGiteaTools } from "../dist/tools/index.js";
|
||||
|
||||
const adminBasic = "Basic " + Buffer.from("mesh-admin:secret").toString("base64");
|
||||
const org = { id: 2, username: "novox", name: "novox" };
|
||||
const team = (id: number, name: string, over: Record<string, unknown>) => ({
|
||||
id, name, description: "", organization: org, includes_all_repositories: true, permission: "read", can_create_org_repo: false,
|
||||
units: ["repo.code"], units_map: { "repo.code": "read" }, ...over,
|
||||
});
|
||||
const teams = [
|
||||
team(5, "builders", { description: "the build modules' fetch" }),
|
||||
team(3, "mesh-tools", { permission: "write", units: ["repo.code", "repo.issues", "repo.pulls"],
|
||||
units_map: { "repo.code": "write", "repo.issues": "write", "repo.pulls": "write" } }),
|
||||
];
|
||||
const members: Record<string, string[]> = { "5": ["nox-builder"], "3": ["mesh-tools"] };
|
||||
const user = (login: string) => ({ id: 23, login, username: login, email: `${login}@noreply.localhost` });
|
||||
const repo = (owner: string, name: string) => ({ full_name: `${owner}/${name}`, name, owner: { login: owner }, private: true, archived: false });
|
||||
const repos = [repo("novox", "mesh-catalog"), repo("jschoubben", "car-hunter")];
|
||||
const seen: string[] = [];
|
||||
|
||||
const server = createServer((req, res) => {
|
||||
const url = new URL(req.url ?? "", "http://x");
|
||||
const path = url.pathname.replace(/^\/api\/v1/, "");
|
||||
const as = req.headers.authorization === adminBasic ? "admin" : "tools";
|
||||
seen.push(`${req.method} ${path} as ${as}`);
|
||||
const send = (status: number, body: unknown) => { res.writeHead(status, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); };
|
||||
if (as !== "admin" && path.startsWith("/orgs/")) return send(403, { message: "token does not have at least one of required scope(s): [read:organization]" });
|
||||
if (path === "/orgs/novox/teams") return send(200, teams);
|
||||
const m = path.match(/^\/teams\/(\d+)\/members$/);
|
||||
if (m) return send(200, (members[m[1]] ?? []).map(user));
|
||||
const u = path.match(/^\/users\/([^/]+)$/);
|
||||
if (u) return send(u[1] === "nobody" ? 404 : 200, u[1] === "nobody" ? { message: "not found" } : user(u[1]));
|
||||
if (path === "/repos/search") return send(200, { ok: true, data: repos });
|
||||
const r = path.match(/^\/repos\/([^/]+)\/([^/]+)$/);
|
||||
if (r) {
|
||||
const sudo = url.searchParams.get("sudo");
|
||||
const found = repos.find((x) => x.full_name === `${r[1]}/${r[2]}`);
|
||||
// nox-builder reads the organisation's repositories through its team and the other one not at all; mesh-tools writes.
|
||||
if (!found || (sudo === "nox-builder" && r[1] !== "novox")) return send(404, { message: "not found" });
|
||||
const push = sudo === "mesh-tools";
|
||||
return send(200, { ...found, permissions: { admin: false, push, pull: true } });
|
||||
}
|
||||
send(404, { message: "not found" });
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
|
||||
after(() => new Promise<void>((r) => server.close(() => r())));
|
||||
const { port } = server.address() as { port: number };
|
||||
const url = `http://127.0.0.1:${port}`;
|
||||
const tools = getGiteaTools(new GiteaClient(url, new ConfiguredToken("tools-token")), () => new GiteaAdmin(url, "mesh-admin", "secret"));
|
||||
const run = (args: Record<string, unknown>) => tools.find((t) => t.name === "gitea_team_rights")!.run(args) as Promise<any>;
|
||||
|
||||
test("gitea_team_rights answers every team of the organisation with its members and units, read as the administrator", async () => {
|
||||
const out = await run({});
|
||||
assert.equal(out.organisation, "novox");
|
||||
assert.deepEqual(out.teams.map((t: any) => t.team), ["builders", "mesh-tools"]);
|
||||
const builders = out.teams[0];
|
||||
assert.deepEqual(builders.members, ["nox-builder"]);
|
||||
assert.deepEqual(builders.units, { "repo.code": "read" });
|
||||
assert.equal(builders.includes_all_repositories, true);
|
||||
assert.equal(builders.read_only, true);
|
||||
assert.equal(out.teams[1].read_only, false);
|
||||
assert.ok(seen.every((s) => s.startsWith("GET ") && s.endsWith("as admin")), `only reads, as the administrator: ${seen.join("; ")}`);
|
||||
});
|
||||
|
||||
test("gitea_team_rights answers one team by name, and refuses an unknown one naming the teams there are", async () => {
|
||||
const out = await run({ team: "builders" });
|
||||
assert.deepEqual(out.teams.map((t: any) => t.team), ["builders"]);
|
||||
await assert.rejects(run({ team: "nobody" }), /no team "nobody" in novox: it holds builders, mesh-tools/);
|
||||
});
|
||||
|
||||
test("gitea_team_rights answers a user's teams and effective right on each repository: nox-builder writes nowhere", async () => {
|
||||
const out = await run({ user: "nox-builder" });
|
||||
assert.deepEqual(out.user.teams, ["builders"]);
|
||||
assert.deepEqual(out.user.repositories, { read: 1, none: 1 });
|
||||
assert.deepEqual(out.user.writes, []);
|
||||
const tools2 = await run({ user: "mesh-tools", list: true });
|
||||
assert.deepEqual(tools2.user.writes, [{ repository: "novox/mesh-catalog", permission: "write" }, { repository: "jschoubben/car-hunter", permission: "write" }]);
|
||||
assert.equal(tools2.user.every.length, 2);
|
||||
});
|
||||
|
||||
test("the tools' own token is refused the teams, which is why the administrator's login reads them", async () => {
|
||||
const res = await fetch(`${url}/api/v1/orgs/novox/teams`, { headers: { Authorization: "token tools-token" } });
|
||||
assert.equal(res.status, 403);
|
||||
});
|
||||
|
||||
test("a user the forge does not know is refused, not answered as writing nowhere", async () => {
|
||||
await assert.rejects(run({ user: "nobody" }), /no forge user nobody/);
|
||||
});
|
||||
@@ -538,7 +538,7 @@ test("the tools register once there is a way to a token, and the first call mint
|
||||
"gitea_get_file",
|
||||
"gitea_list_branches",
|
||||
"gitea_delete_branch",
|
||||
"gitea_branch_protection_get", "gitea_branch_protection_set",
|
||||
"gitea_branch_protection_get", "gitea_team_rights", "gitea_branch_protection_set",
|
||||
"gitea_note_append", "gitea_delivery_view", "gitea_commit_statuses", "gitea_open_pulls", "gitea_commit_trees", "gitea_contains",
|
||||
"gitea_list_labels", "gitea_create_label",
|
||||
"gitea_api",
|
||||
|
||||
@@ -16,7 +16,7 @@ import { join } from "node:path";
|
||||
|
||||
import { MintedToken } from "../dist/token.js";
|
||||
import { GiteaClient } from "../dist/client.js";
|
||||
import { TOOLS_ACCOUNT, TOOLS_TEAM_UNITS } from "../dist/tools-account.js";
|
||||
import { TOOLS_ACCOUNT, TOOLS_TEAM_UNITS, BUILDER_ACCOUNT, AGENT_ACCOUNT, meshAccounts } from "../dist/tools-account.js";
|
||||
import { repoCheckFacts } from "../dist/pulls.js";
|
||||
|
||||
const ADMIN = "mesh-admin";
|
||||
@@ -26,6 +26,19 @@ interface Account { admin: boolean; password: string; active: boolean }
|
||||
interface Token { id: number; name: string; sha1: string; user: string; scopes: string[] }
|
||||
interface Team { id: number; name: string; units: any; members: Set<string> }
|
||||
|
||||
interface ForgeRepo { owner: string; name: string; full: string; private: boolean; archived?: boolean; ownerVisibility: string; collaborators: Map<string, string> }
|
||||
const forgeRepos: ForgeRepo[] = [];
|
||||
const userRules = new Map<string, any>();
|
||||
const repoFaults = { refuse: new Set<string>(), permission: new Set<string>() };
|
||||
function addRepo(full: string, isPrivate: boolean, ownerVisibility = "public"): ForgeRepo {
|
||||
const [owner, name] = full.split("/");
|
||||
const r = { owner, name, full, private: isPrivate, ownerVisibility, collaborators: new Map<string, string>() };
|
||||
forgeRepos.push(r);
|
||||
return r;
|
||||
}
|
||||
const repoJSON = (r: ForgeRepo) => ({ full_name: r.full, name: r.name, owner: { login: r.owner, visibility: r.ownerVisibility },
|
||||
private: r.private, archived: r.archived === true });
|
||||
|
||||
function fakeForge() {
|
||||
const users = new Map<string, Account>([[ADMIN, { admin: true, password: PASSWORD, active: true }]]);
|
||||
const tokens: Token[] = [];
|
||||
@@ -61,11 +74,18 @@ function fakeForge() {
|
||||
return null;
|
||||
};
|
||||
const isAdmin = (u: string | null) => u !== null && users.get(u)?.admin === true;
|
||||
const permissionOf = (u: string | null, r: ForgeRepo): string => {
|
||||
if (isAdmin(u)) return "admin";
|
||||
if (u === r.owner) return "owner";
|
||||
if (r.owner === "novox" && teams.some((t) => t.members.has(u ?? "") && t.units.units_map?.["repo.code"] === "write")) return "write";
|
||||
if (u && r.collaborators.has(u)) return r.collaborators.get(u)!;
|
||||
return !r.private && r.ownerVisibility === "public" ? "read" : "none";
|
||||
};
|
||||
|
||||
const server = createServer(async (req, res) => {
|
||||
const url = new URL(req.url ?? "/", "http://fake");
|
||||
const p = url.pathname.replace(/^\/api\/v1/, "");
|
||||
const me = who(req);
|
||||
const me = url.searchParams.has("sudo") && isAdmin(who(req)) ? url.searchParams.get("sudo") : who(req);
|
||||
calls.push(`${req.method} ${p} as ${me}`);
|
||||
if (me === null) return json(res, 401, { message: "unauthorized" });
|
||||
|
||||
@@ -149,6 +169,23 @@ function fakeForge() {
|
||||
if (m[2] === "feat/unguarded") return json(res, 200, { name: m[2], protected: false });
|
||||
return json(res, 404, { message: "branch does not exist" });
|
||||
}
|
||||
// A repository of a user: its rules are its administrator's (a site administrator, or a collaborator with admin).
|
||||
m = p.match(/^\/repos\/(?!novox\/)([^/]+)\/([^/]+)\/branch_protections(?:\/(.*))?$/);
|
||||
if (m) {
|
||||
const key = `${m[1]}/${m[2]}`;
|
||||
const repo = forgeRepos.find((r) => r.full === key);
|
||||
if (!repo || !["admin", "owner"].includes(permissionOf(me, repo))) return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" });
|
||||
if (req.method === "GET" && m[3] === undefined) return json(res, 200, userRules.has(key) ? [userRules.get(key)] : []);
|
||||
// As gitea does: a whitelist login that cannot write the repository is dropped, and the write still answered.
|
||||
const keep = (b: any) => (Array.isArray(b.merge_whitelist_usernames)
|
||||
? { ...b, merge_whitelist_usernames: b.merge_whitelist_usernames.filter((u: string) => ["write", "admin", "owner"].includes(permissionOf(u, repo))) } : b);
|
||||
if (req.method === "PATCH" && m[3] === "main") {
|
||||
if (!userRules.has(key)) return json(res, 404, { message: "branch protection does not exist" });
|
||||
Object.assign(userRules.get(key), keep(await read(req)));
|
||||
return json(res, 200, userRules.get(key));
|
||||
}
|
||||
if (req.method === "POST") { const b = keep(await read(req)); userRules.set(key, b); return json(res, 201, b); }
|
||||
}
|
||||
// gitea's reqAdmin on the whole group: reading the rules is administration too.
|
||||
m = p.match(/^\/repos\/novox\/([^/]+)\/branch_protections(?:\/(.*))?$/);
|
||||
if (m) {
|
||||
@@ -176,7 +213,37 @@ function fakeForge() {
|
||||
if (!writes) return json(res, 403, { message: "write on code required" });
|
||||
return json(res, 201, { context: (await read(req)).context, creator: { login: me } });
|
||||
}
|
||||
if (req.method === "GET" && p === "/repos/search") return json(res, 200, { data: [] });
|
||||
// Every repository of every owner, and who reads it (readall.ts): the site administrator reads all; the tools' account
|
||||
// a repository of novox through its team, a public one of a public owner, and one it is a collaborator of.
|
||||
if (req.method === "GET" && p === "/repos/search") {
|
||||
const page = Number(url.searchParams.get("page") ?? "1");
|
||||
const seen = forgeRepos.filter((r) => permissionOf(me, r) !== "none");
|
||||
return json(res, 200, { data: page === 1 ? seen.map(repoJSON) : [] });
|
||||
}
|
||||
m = p.match(/^\/repos\/([^/]+)\/([^/]+)(?:\/collaborators\/([^/]+)(\/permission)?)?$/);
|
||||
if (m) {
|
||||
const repo = forgeRepos.find((r) => r.owner === decodeURIComponent(m![1]) && r.name === decodeURIComponent(m![2]));
|
||||
if (repo && req.method === "GET" && !m[3] && url.searchParams.has("sudo") && repoFaults.permission.has(repo.full)) return json(res, 500, { message: "the forge is not well" });
|
||||
if (!repo || permissionOf(me, repo) === "none") return json(res, 404, { message: "repository does not exist" });
|
||||
if (req.method === "GET" && !m[3]) {
|
||||
const mine = permissionOf(me, repo);
|
||||
return json(res, 200, { ...repoJSON(repo), permissions: { pull: true, push: ["write", "admin", "owner"].includes(mine), admin: ["admin", "owner"].includes(mine) } });
|
||||
}
|
||||
const user = decodeURIComponent(m[3] ?? "");
|
||||
if (req.method === "GET" && m[4]) {
|
||||
if (repoFaults.permission.has(repo.full)) return json(res, 500, { message: "the forge is not well" });
|
||||
if (!isAdmin(me) && me !== user) return json(res, 403, { message: "only the user or an administrator" });
|
||||
// As gitea answers: the collaborator's own permission, not what a team gives the account.
|
||||
return json(res, 200, { permission: repo.collaborators.get(user) ?? "read" });
|
||||
}
|
||||
if (req.method === "PUT" && !m[4]) {
|
||||
// gitea's reqAdmin: the repository's administration, which on a repository of another user the site administrator alone holds.
|
||||
if (!isAdmin(me) && me !== repo.owner) return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" });
|
||||
if (repoFaults.refuse.has(repo.full)) return json(res, 422, { message: "the forge would not have it" });
|
||||
repo.collaborators.set(user, (await read(req))?.permission ?? "write");
|
||||
return json(res, 204, null);
|
||||
}
|
||||
}
|
||||
return json(res, 404, { message: "no such route in the fake" });
|
||||
});
|
||||
return new Promise<{ url: string; users: typeof users; tokens: typeof tokens; teams: typeof teams; calls: string[]; mainRules: typeof mainRules;
|
||||
@@ -402,3 +469,226 @@ test("a mint with keep-main-rules false (the default) writes no branch protectio
|
||||
const writes = forge.calls.slice(from).filter((c: string) => /branch_protections/.test(c) && !/^GET /.test(c) && !/^(GET|HEAD) /.test(c));
|
||||
assert.deepEqual(writes, []);
|
||||
});
|
||||
|
||||
// The mesh's accounts read every repository (readall.ts; the operator, 2026-10-26): jschoubben/car-hunter and
|
||||
// jschoubben/snake, private repositories of a user, answered the tools 404 while that user's public ones were seen; and
|
||||
// once the tools' account read them, the build modules' fetch as nox-builder still could not.
|
||||
|
||||
test("the start's pass gives each of the mesh's accounts read, as a collaborator that reads only, on each repository it could not read, and on no other; an account the forge does not hold is said, not refused", async () => {
|
||||
const env = await environment();
|
||||
await fresh(env).listRepos(); // the account and its team exist, as after any mint
|
||||
forge.users.set(BUILDER_ACCOUNT, { admin: false, password: "unused", active: true }); // made by the forge-account provider
|
||||
forge.users.delete(AGENT_ACCOUNT); // no machine's key asked for it yet
|
||||
forgeRepos.splice(0);
|
||||
addRepo("novox/hq", true);
|
||||
addRepo("jschoubben/txt-game", false);
|
||||
addRepo("jschoubben/car-hunter", true);
|
||||
addRepo("jschoubben/snake", true);
|
||||
addRepo("shy/open", false, "limited");
|
||||
addRepo("elsewhere/secret", true);
|
||||
const { GiteaAdmin } = await import("../dist/client.js");
|
||||
const { ReadAll, readAllTools } = await import("../dist/readall.js");
|
||||
const lines: string[] = [];
|
||||
const readAll = new ReadAll(GiteaAdmin.fromEnv(env), meshAccounts(), (l: string) => lines.push(l));
|
||||
assert.deepEqual(readAll.accounts, [TOOLS_ACCOUNT, BUILDER_ACCOUNT, AGENT_ACCOUNT]);
|
||||
const [unreadableTool, grantTool] = readAllTools(() => readAll);
|
||||
|
||||
// What the tool says before, per account: the four the tools' account cannot read, each with why; the builders'
|
||||
// account, in no write team here, also not novox/hq; nox-agent not on the forge.
|
||||
const before = await unreadableTool.run({}) as any;
|
||||
assert.equal(before.checked, 6);
|
||||
const of = (account: string) => before.unreadable.filter((u: any) => u.account === account).map((u: any) => u.repository).sort();
|
||||
assert.deepEqual(of(TOOLS_ACCOUNT), ["elsewhere/secret", "jschoubben/car-hunter", "jschoubben/snake", "shy/open"]);
|
||||
assert.deepEqual(of(BUILDER_ACCOUNT), ["elsewhere/secret", "jschoubben/car-hunter", "jschoubben/snake", "novox/hq", "shy/open"]);
|
||||
assert.deepEqual(before.accounts, [{ account: TOOLS_ACCOUNT, unreadable: 4 }, { account: BUILDER_ACCOUNT, unreadable: 5 }]);
|
||||
assert.deepEqual(before.absent, [AGENT_ACCOUNT]);
|
||||
assert.match(before.unreadable.find((u: any) => u.account === BUILDER_ACCOUNT && u.repository === "jschoubben/car-hunter").why,
|
||||
/a private repository of jschoubben, on which nox-builder is no collaborator/);
|
||||
const builderOnly = await unreadableTool.run({ accounts: [BUILDER_ACCOUNT] }) as any;
|
||||
assert.deepEqual(builderOnly.accounts, [{ account: BUILDER_ACCOUNT, unreadable: 5 }]);
|
||||
await assert.rejects(unreadableTool.run({ accounts: ["someone"] }), /not one of the mesh's accounts/);
|
||||
assert.match(before.unreadable.find((u: any) => u.repository === "jschoubben/car-hunter").why,
|
||||
/a private repository of jschoubben, on which mesh-tools is no collaborator/);
|
||||
assert.match(before.unreadable.find((u: any) => u.repository === "shy/open").why, /visibility is limited/);
|
||||
assert.deepEqual((await fresh(env).listAllRepos()).map((r) => r.full_name).sort(), ["jschoubben/txt-game", "novox/hq"]);
|
||||
|
||||
// The account's own token cannot grant itself: the forge asks the repository's administration (the reason for the admin login).
|
||||
const own = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim();
|
||||
const self = await fetch(`${forge.url}/api/v1/repos/jschoubben/car-hunter/collaborators/${TOOLS_ACCOUNT}`,
|
||||
{ method: "PUT", headers: { Authorization: `token ${own}`, "Content-Type": "application/json" }, body: JSON.stringify({ permission: "read" }) });
|
||||
assert.equal(self.status, 404, "a repository the account cannot read is not even there for it");
|
||||
|
||||
const from = forge.calls.length;
|
||||
const r = await readAll.atStart();
|
||||
const grantedTo = (account: string) => r.granted.filter((x: any) => x.account === account).map((x: any) => x.repository).sort();
|
||||
assert.deepEqual(grantedTo(TOOLS_ACCOUNT), ["elsewhere/secret", "jschoubben/car-hunter", "jschoubben/snake", "shy/open"]);
|
||||
assert.deepEqual(grantedTo(BUILDER_ACCOUNT), ["elsewhere/secret", "jschoubben/car-hunter", "jschoubben/snake", "novox/hq", "shy/open"]);
|
||||
assert.deepEqual(r.refused, []);
|
||||
assert.deepEqual(r.absent, [AGENT_ACCOUNT]);
|
||||
const puts = forge.calls.slice(from).filter((c) => c.startsWith("PUT "));
|
||||
assert.equal(puts.length, 9, "the repositories each read already were left alone, and the absent account was asked nothing");
|
||||
assert.ok(!puts.some((c) => c.includes(AGENT_ACCOUNT)));
|
||||
assert.ok(puts.every((c) => c.endsWith(` as ${ADMIN}`)), "only the administrator's login grants");
|
||||
for (const repo of forgeRepos.filter((x) => x.collaborators.size)) {
|
||||
for (const [account, permission] of repo.collaborators) assert.equal(permission, "read", `${account} on ${repo.full} reads only`);
|
||||
}
|
||||
assert.equal(forgeRepos.find((x) => x.full === "jschoubben/car-hunter")!.collaborators.get(BUILDER_ACCOUNT), "read");
|
||||
assert.match(lines.join("\n"), /6 looked at; mesh-tools 4 granted read now .*; nox-builder 5 granted read now .*; not on the forge yet, so not granted: nox-agent/);
|
||||
|
||||
// The account made later is granted by hand (or at the next pass), its rows like any other.
|
||||
forge.users.set(AGENT_ACCOUNT, { admin: false, password: "unused", active: true });
|
||||
const agent = await grantTool.run({ accounts: [AGENT_ACCOUNT], repositories: ["jschoubben/car-hunter"] }) as any;
|
||||
assert.deepEqual(agent.granted, [{ account: AGENT_ACCOUNT, repository: "jschoubben/car-hunter" }]);
|
||||
assert.deepEqual(agent.absent, []);
|
||||
forge.users.delete(AGENT_ACCOUNT);
|
||||
|
||||
// After: the account's own token sees every repository, and the tool names none.
|
||||
assert.equal((await fresh(env).listAllRepos()).length, 6);
|
||||
assert.deepEqual(((await unreadableTool.run({})) as any).unreadable, []);
|
||||
// Granting again asks nothing of the forge but reads.
|
||||
const again = forge.calls.length;
|
||||
assert.deepEqual(((await grantTool.run({})) as any).granted, []);
|
||||
assert.ok(forge.calls.slice(again).every((c) => c.startsWith("GET ")));
|
||||
});
|
||||
|
||||
test("on the forge's events: a repository made is granted read at once, a known readable one asks nothing, a refusal is said, not thrown", async () => {
|
||||
const env = await environment();
|
||||
await fresh(env).listRepos();
|
||||
forgeRepos.splice(0);
|
||||
addRepo("jschoubben/txt-game", false);
|
||||
const { GiteaAdmin } = await import("../dist/client.js");
|
||||
const { ReadAll, readAllTools } = await import("../dist/readall.js");
|
||||
const lines: string[] = [];
|
||||
forge.users.set(BUILDER_ACCOUNT, { admin: false, password: "unused", active: true });
|
||||
const readAll = new ReadAll(GiteaAdmin.fromEnv(env), [TOOLS_ACCOUNT, BUILDER_ACCOUNT], (l: string) => lines.push(l));
|
||||
await readAll.atStart();
|
||||
|
||||
const cases: { name: string; event: string; repo: string; make?: boolean; refuse?: boolean; unanswered?: boolean; asks: boolean; reads: string }[] = [
|
||||
{ name: "a repository made", event: "forge.repository.created", repo: "jschoubben/new-private", make: true, asks: true, reads: "read" },
|
||||
{ name: "a fork made", event: "forge.fork", repo: "someone/fork", make: true, asks: true, reads: "read" },
|
||||
{ name: "a push to a repository seen readable", event: "forge.push", repo: "jschoubben/txt-game", asks: false, reads: "read" },
|
||||
{ name: "a push to a private repository never seen", event: "forge.push", repo: "jschoubben/quiet", make: true, asks: true, reads: "read" },
|
||||
{ name: "a repository the forge refuses the grant on", event: "forge.repository.created", repo: "jschoubben/stuck", make: true, refuse: true, asks: true, reads: "none" },
|
||||
{ name: "a repository whose permission the forge will not answer", event: "forge.repository.created", repo: "jschoubben/mute", make: true, unanswered: true, asks: true, reads: "none" },
|
||||
];
|
||||
for (const c of cases) {
|
||||
if (c.make) addRepo(c.repo, true);
|
||||
if (c.refuse) repoFaults.refuse.add(c.repo);
|
||||
if (c.unanswered) repoFaults.permission.add(c.repo);
|
||||
const from = forge.calls.length;
|
||||
const [owner, repo] = c.repo.split("/");
|
||||
await readAll.onEvent({ name: c.event, body: { owner, repo } });
|
||||
assert.equal(forge.calls.length > from, c.asks, `${c.name}: asks the forge`);
|
||||
for (const account of [TOOLS_ACCOUNT, BUILDER_ACCOUNT]) {
|
||||
assert.equal(forgeRepos.find((x) => x.full === c.repo)!.collaborators.get(account) ?? (c.make ? "none" : "read"), c.reads, `${c.name}: ${account}`);
|
||||
}
|
||||
}
|
||||
assert.match(lines.join("\n"), /mesh-tools could not be given read on jschoubben\/stuck — .*422/);
|
||||
assert.match(lines.join("\n"), /nox-builder was given read on someone\/fork, on the forge's forge.fork/);
|
||||
const named = await (readAllTools(() => readAll)[1].run({ repositories: ["jschoubben/stuck"] })) as any;
|
||||
assert.deepEqual(named.refused.map((x: any) => `${x.account} ${x.repository}`), [`${TOOLS_ACCOUNT} jschoubben/stuck`, `${BUILDER_ACCOUNT} jschoubben/stuck`]);
|
||||
await assert.rejects(readAllTools(() => readAll)[1].run({ repositories: ["no-slash"] }), /not a repository/);
|
||||
assert.match(lines.join("\n"), /could not be given read on jschoubben\/mute — its permission could not be read — .*500/);
|
||||
// A whole pass goes past the repository it cannot judge: every other one is still granted, and the listing names it.
|
||||
addRepo("jschoubben/after-mute", true);
|
||||
const pass = await readAll.grant();
|
||||
assert.ok(pass.granted.some((x: any) => x.repository === "jschoubben/after-mute" && x.account === BUILDER_ACCOUNT), "the pass did not stop at the repository it could not judge");
|
||||
assert.ok(pass.refused.some((x: any) => x.repository === "jschoubben/mute" && /permission could not be read/.test(x.why)));
|
||||
const listed = await readAllTools(() => readAll)[0].run({}) as any;
|
||||
assert.equal(listed.unreadable.find((u: any) => u.repository === "jschoubben/mute").permission, "unknown");
|
||||
repoFaults.refuse.clear();
|
||||
repoFaults.permission.clear();
|
||||
});
|
||||
|
||||
// The merging account writes every repository that builds a registered module, and no other (writes.ts; the operator,
|
||||
// 2026-10-26; issue 1945): car-hunter #2 was approved and its commit status refused 403, mesh-tools reading only there.
|
||||
|
||||
test("the merging account is given write on each repository the indexer's modules name, read back and said; the operator's other repositories stay read-only; the tools say both", async () => {
|
||||
const env = await environment();
|
||||
const client = fresh(env);
|
||||
await client.listRepos(); // the account and its team exist, as after any mint
|
||||
forgeRepos.splice(0);
|
||||
addRepo("novox/hq", true);
|
||||
for (const full of ["jschoubben/car-hunter", "jschoubben/snake", "jschoubben/dotfiles"]) addRepo(full, true).collaborators.set(TOOLS_ACCOUNT, "read");
|
||||
const { GiteaAdmin } = await import("../dist/client.js");
|
||||
const { DeliveredWrites, writeTools, WRITABLE_EVENT } = await import("../dist/writes.js");
|
||||
const lines: string[] = [];
|
||||
const said: { type: string; body: any }[] = [];
|
||||
const writes = new DeliveredWrites(GiteaAdmin.fromEnv(env), TOOLS_ACCOUNT, "novox", (l: string) => lines.push(l),
|
||||
async (type: string, body: unknown) => { said.push({ type, body }); });
|
||||
const [unwritableTool, grantTool] = writeTools(() => writes);
|
||||
await assert.rejects(grantTool.run({}), /not known yet/, "nothing is granted before the indexer's modules were read");
|
||||
assert.equal((await client.getRepo("jschoubben", "car-hunter")).writes, false, "the statuses' answer says the account cannot write");
|
||||
|
||||
// 1945/AC1: the watch hands the indexer's modules; a user's repository a module builds from is made writable, once.
|
||||
await writes.take({ key: "car-hunter", op: "put", value: { v: 1, repository: "jschoubben/car-hunter", commit: "c" }, current: true });
|
||||
await writes.take({ key: "gitea", op: "put", value: { v: 1, repository: "novox/hq", commit: "c" }, current: true });
|
||||
await writes.take({ key: "broken", op: "put", value: { v: 1, repository: "no-slash" }, current: true });
|
||||
const from = forge.calls.length;
|
||||
await writes.take({ key: "car-hunter-2", op: "put", value: { v: 1, repository: "jschoubben/car-hunter" } });
|
||||
assert.equal(forge.calls.length, from, "a repository already written asks nothing more");
|
||||
const repo = (full: string) => forgeRepos.find((x) => x.full === full)!;
|
||||
assert.equal(repo("jschoubben/car-hunter").collaborators.get(TOOLS_ACCOUNT), "write", "a user's repository is written, not administered");
|
||||
assert.equal(userRules.has("jschoubben/car-hunter"), false, "a repository with no rule for main is given none");
|
||||
assert.equal(repo("novox/hq").collaborators.size, 0, "the organisation's repository is written through its team, no collaborator");
|
||||
assert.equal((await client.getRepo("jschoubben", "car-hunter")).writes, true);
|
||||
assert.deepEqual(said, [{ type: WRITABLE_EVENT, body: { owner: "jschoubben", repo: "car-hunter", account: TOOLS_ACCOUNT } }]);
|
||||
assert.match(lines.join("\n"), /mesh-tools was given write on jschoubben\/car-hunter, which builds the registered module car-hunter/);
|
||||
assert.match(lines.join("\n"), /broken names no repository/);
|
||||
|
||||
// 1945/AC2: a delivered repository the forge refuses is said, in its words, by the listing and the grant; the operator's
|
||||
// repository no module builds from is left read-only, named with why, and a grant naming it asks the forge nothing.
|
||||
repoFaults.refuse.add("jschoubben/snake");
|
||||
await writes.take({ key: "snake", op: "put", value: { v: 1, repository: "jschoubben/snake" } });
|
||||
assert.match(lines.join("\n"), /could not be given write on jschoubben\/snake, which builds snake — .*422/);
|
||||
const listed = await unwritableTool.run({}) as any;
|
||||
assert.deepEqual(listed.unwritable.map((u: any) => [u.repository, u.permission]), [["jschoubben/snake", "read"]]);
|
||||
assert.match(listed.unwritable[0].why, /builds snake.*gitea_grant_write/);
|
||||
const all = await grantTool.run({}) as any;
|
||||
assert.deepEqual(all.delivered, ["jschoubben/car-hunter", "jschoubben/snake", "novox/hq"]);
|
||||
assert.deepEqual(all.already, ["jschoubben/car-hunter", "novox/hq"]);
|
||||
assert.deepEqual(all.granted, []);
|
||||
assert.match(all.refused.find((x: any) => x.repository === "jschoubben/snake").why, /422.*would not have it/);
|
||||
assert.deepEqual(all.left_read_only.map((x: any) => x.repository), ["jschoubben/dotfiles"]);
|
||||
assert.match(all.left_read_only[0].why, /builds no registered module/);
|
||||
const puts = forge.calls.length;
|
||||
const named = await grantTool.run({ repositories: ["jschoubben/dotfiles"] }) as any;
|
||||
assert.deepEqual([named.granted, named.left_read_only.map((x: any) => x.repository)], [[], ["jschoubben/dotfiles"]]);
|
||||
assert.ok(!forge.calls.slice(puts).some((c) => c.startsWith("PUT ")), "no write is asked on a repository no module builds from");
|
||||
assert.equal(repo("jschoubben/dotfiles").collaborators.get(TOOLS_ACCOUNT), "read");
|
||||
repoFaults.refuse.clear();
|
||||
// The operator's rule is left as set (the operator, 2026-10-26: "Write, rule untouched"): only mesh-tools is added to its
|
||||
// merge whitelist; a status the mesh does not set and required approvals are said as blocking the mesh's merge, unchanged.
|
||||
const operators = { rule_name: "main", enable_push: true, enable_merge_whitelist: true, merge_whitelist_usernames: ["jschoubben"],
|
||||
enable_status_check: true, status_check_contexts: ["mesh/check", "mesh/merge-gate", "ci/external"], required_approvals: 2,
|
||||
block_admin_merge_override: false, require_signed_commits: true, block_on_outdated_branch: true, block_on_rejected_reviews: true };
|
||||
userRules.set("jschoubben/snake", structuredClone(operators));
|
||||
const after = await grantTool.run({ repositories: ["jschoubben/snake"] }) as any;
|
||||
assert.deepEqual(after.granted, ["jschoubben/snake"]);
|
||||
assert.equal(repo("jschoubben/snake").collaborators.get(TOOLS_ACCOUNT), "write");
|
||||
assert.deepEqual(userRules.get("jschoubben/snake"), { ...operators, merge_whitelist_usernames: ["jschoubben", TOOLS_ACCOUNT] }, "only the whitelist changed");
|
||||
assert.deepEqual(after.merge_whitelist, [{ repository: "jschoubben/snake", changed: true, listed: ["jschoubben", TOOLS_ACCOUNT] }]);
|
||||
const blockedWhy = "writable, but main's rule blocks the mesh's merge: it requires the status ci/external, which the mesh does not set; " +
|
||||
"it requires 2 approving reviews on the forge, which the gate does not give; it requires signed commits, which the mesh does not make; " +
|
||||
"it refuses a branch behind main, which the mesh does not bring up to date; " +
|
||||
"it refuses a merge after a review asked for changes, which the gate does not clear";
|
||||
assert.deepEqual(after.merge_blocked, [{ repository: "jschoubben/snake", why: blockedWhy }]);
|
||||
const again = await unwritableTool.run({}) as any;
|
||||
assert.deepEqual([again.unwritable, again.merge_blocked], [[], [{ repository: "jschoubben/snake", why: blockedWhy }]]);
|
||||
const writesBefore = forge.calls.filter((c) => /^PATCH .*branch_protections/.test(c)).length;
|
||||
assert.deepEqual(((await grantTool.run({ repositories: ["jschoubben/snake"] })) as any).merge_whitelist[0].changed, false);
|
||||
assert.equal(forge.calls.filter((c) => /^PATCH .*branch_protections/.test(c)).length, writesBefore, "a listed account is not written again");
|
||||
// The mesh's own statuses never block: a rule requiring only them, with mesh-tools listed, is not reported.
|
||||
userRules.set("jschoubben/snake", { rule_name: "main", enable_merge_whitelist: true, merge_whitelist_usernames: [TOOLS_ACCOUNT],
|
||||
enable_status_check: true, status_check_contexts: ["mesh/check", "mesh/merge-gate", "mesh/repo-check"] });
|
||||
assert.deepEqual(((await unwritableTool.run({})) as any).merge_blocked, []);
|
||||
assert.deepEqual(((await grantTool.run({ repositories: ["jschoubben/snake"] })) as any).merge_blocked, []);
|
||||
// The forge drops a whitelist login that cannot write and still answers 200: the whitelist is read back and that fails loud.
|
||||
userRules.set("jschoubben/dotfiles", { rule_name: "main", enable_merge_whitelist: true, merge_whitelist_usernames: ["jschoubben"] });
|
||||
await assert.rejects(GiteaAdmin.fromEnv(env).ensureMergerListed("jschoubben", "dotfiles", TOOLS_ACCOUNT),
|
||||
/mesh-tools was added to the merge whitelist of main's rule main of jschoubben\/dotfiles, but the forge holds \[jschoubben\]: it drops a login that cannot write/);
|
||||
userRules.delete("jschoubben/dotfiles");
|
||||
// A module gone from the indexer leaves its repository out of the delivered list.
|
||||
await writes.take({ key: "snake", op: "delete" });
|
||||
assert.deepEqual(writes.delivered(), ["jschoubben/car-hunter", "novox/hq"]);
|
||||
});
|
||||
|
||||
@@ -34,6 +34,9 @@ test("both statuses say the head and the trunk commit the verdict judged, first,
|
||||
const [old] = statusesFor({ ...c, trunk: undefined, gate: { verdict: "pass", summary: "every machine composes" } });
|
||||
assert.equal(old.description, "pass: every machine composes");
|
||||
assert.equal(trunkNamed(old.description), undefined);
|
||||
// ADR 0548 §8: a carried verdict's line names the head on the new trunk first, so the merge tool merges it on that trunk.
|
||||
assert.equal(trunkNamed(`pass ${HEAD.slice(0, 12)} on ${MOVED.slice(0, 12)} | carried from ${TRUNK.slice(0, 12)}, mesh-catalog 0123456789ab | unit 0 run, 4 kept`),
|
||||
MOVED.slice(0, 12));
|
||||
});
|
||||
|
||||
test("a trunk's move puts an open head's merge check out of date, naming both commits, unless a check is on its way", () => {
|
||||
|
||||
@@ -22,6 +22,15 @@
|
||||
import { keepMainRules } from "./mainrules.js";
|
||||
|
||||
export const TOOLS_ACCOUNT = "mesh-tools";
|
||||
/** The build modules' account, whose keys fetch over ssh (internal/forgeaccount BuilderUser, ADR 0466 §3). */
|
||||
export const BUILDER_ACCOUNT = "nox-builder";
|
||||
/** The machines' agents' account, whose keys ssh-client reports (internal/forgeaccount AgentUser, ADR 0410 §1). */
|
||||
export const AGENT_ACCOUNT = "nox-agent";
|
||||
/** Every forge account the mesh's own parts use: each reads every repository on the forge (readall.ts; the operator,
|
||||
* 2026-10-26). internal/forgeaccount's test holds the two Go names to its own constants. */
|
||||
export function meshAccounts(tools: string = TOOLS_ACCOUNT): string[] {
|
||||
return [tools, BUILDER_ACCOUNT, AGENT_ACCOUNT];
|
||||
}
|
||||
/** The organisation whose repositories are the mesh's, and the team through which the tools' account writes there. */
|
||||
export const TOOLS_ORG = "novox";
|
||||
export const TOOLS_TEAM = "mesh-tools";
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import { emit } from "@novox/mesh-sdk/events";
|
||||
import { GiteaAdmin, GiteaClient, type BranchProtection } from "../client.js";
|
||||
import { readTeamRights } from "../teamrights.js";
|
||||
import { appendNote, run, viewBody, viewComment } from "../delivery.js";
|
||||
import { containedFrom, isCommit, mergeableUnder, newestByContext } from "../checks.js";
|
||||
import {
|
||||
@@ -76,6 +77,17 @@ function namedPaths(raw: unknown): Record<string, string[]> | undefined {
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The `listed` argument of gitea_commit_trees: [directory], each plain (no `.` or `..` segment, not absolute, the root
|
||||
* the empty name); anything else is refused before the forge is asked. Absent is absent. */
|
||||
function listedDirs(raw: unknown): string[] | undefined {
|
||||
if (raw === undefined || raw === null) return undefined;
|
||||
if (!Array.isArray(raw) || raw.some((d) => typeof d !== "string")) throw new Error("listed is [directory, ...]");
|
||||
for (const d of raw as string[]) {
|
||||
if (d !== "" && d.split("/").some((seg) => seg === "" || seg === "." || seg === "..")) throw new Error(`listed: ${JSON.stringify(d)} is not a plain relative directory`);
|
||||
}
|
||||
return raw as string[];
|
||||
}
|
||||
|
||||
export function getGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] {
|
||||
return namesChecked(allGiteaTools(gitea, admin));
|
||||
}
|
||||
@@ -374,7 +386,7 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
// ---- Contents and branches ----
|
||||
{
|
||||
name: "gitea_get_file",
|
||||
description: "One file's contents from a repository, decoded, at a branch, tag or commit (default the repository's default branch).",
|
||||
description: "One file's contents from a repository, decoded, at a branch, tag or commit (default the repository's default branch). Refuses a file over 512 KiB, which a tool answer cannot carry: read it from a clone.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
@@ -427,6 +439,26 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
return { branch: String(args.branch), rule: rule ? summarised(rule) : null };
|
||||
},
|
||||
},
|
||||
// ---- Forge teams (issue 1563) ----
|
||||
{
|
||||
name: "gitea_team_rights",
|
||||
description: "Read-only: the forge's teams of an organisation, who is in each (members) and what each may do — its permission, its units read, write or admin per unit (code, issues, pull requests), whether it covers all repositories, and whether it is read-only. Give a user to see which teams it is in and its effective right (read, write, admin) on every repository, with those it can write listed: the check that a build key's forge user (nox-builder, team builders) reads and pushes nothing. Forge team members and rights; GET only, no secret answered.",
|
||||
input: {
|
||||
org: { type: "string", description: "the organisation; novox when not given" },
|
||||
team: { type: "string", description: "one team by name; every team of the organisation when not given" },
|
||||
user: { type: "string", description: "a forge user: its teams and effective right on each repository" },
|
||||
repositories: { type: "array", items: { type: "string" }, description: "owner/name of the repositories to look at for the user; every repository when not given" },
|
||||
list: { type: "boolean", description: "with a user, list the right on every repository, not only those above read" },
|
||||
},
|
||||
run: async (args) => {
|
||||
const strings = Array.isArray(args.repositories) ? args.repositories.map(String) : undefined;
|
||||
// The second read made as the forge's administrator (ADR 0375's precedent, issue 1563): the tools' token lacks
|
||||
// read:organization, so a team is not theirs to read. GET only.
|
||||
return readTeamRights(admin ? admin() : (() => { throw new Error("the forge's administrator login is not set: a team is read with it"); })(),
|
||||
String(args.org || "novox"), { team: args.team ? String(args.team) : undefined, user: args.user ? String(args.user) : undefined,
|
||||
repositories: strings, list: args.list === true });
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "gitea_branch_protection_set",
|
||||
description: "Protect a branch. Never weaker (novox/hq ADR 0266): a change that lets a person push directly or lets an administrator merge past the rule is refused, and on main so is a change that drops the merge whitelist (only mesh-tools merges there) or widens it (novox/hq ADR 0398 §2, issue 948) — that is the operator's, at the forge. The forge requires no status to merge (ADR 0534): the rule written requires none, and one the rule required is taken off; the checks are judged by mesh-delivery and the merge tool. Edits the branch's rule (everything else kept unless given) or creates one, which refuses direct pushes unless push=true. Answers the rule before and after.",
|
||||
@@ -492,7 +524,7 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
},
|
||||
{
|
||||
name: "gitea_commit_statuses",
|
||||
description: "Read a commit's statuses — or a pull request's head's — as the forge keeps them: the newest of each context (mesh/check, mesh/delivery, …) with its state, description, link, when it was set and by whom; and whether the branch it merges into would let it merge, every status that branch's protection requires being a success. Reads only.",
|
||||
description: "Read a commit's statuses — or a pull request's head's — as the forge keeps them: the newest of each context (mesh/check, mesh/delivery, …) with its state, description, link, when it was set and by whom; and whether the branch it merges into would let it merge, every status that branch's protection requires being a success; for a pull request, also the forge's own mergeable (false while it is in conflict with its base or the forge is still working that out); and merge.writes, whether the mesh's merging account may write the repository at all (false: its statuses and its merge are refused until gitea_grant_write). Reads only.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
@@ -508,11 +540,12 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
if (!owner || !repo) throw new Error("name the repository: owner and repo");
|
||||
if (!Number.isInteger(number) || number < 0) throw new Error(`${args.number} is not a pull request's number`);
|
||||
let pull: { number: number; title: string; state: string; merged: boolean; base?: string; head?: string;
|
||||
head_sha?: string; merge_commit_sha?: string; html_url: string } | undefined;
|
||||
head_sha?: string; merge_commit_sha?: string; merged_at?: string; html_url: string; mergeable?: boolean } | undefined;
|
||||
if (number > 0) {
|
||||
const p = await gitea.getPullRequest(owner, repo, number);
|
||||
pull = { number: p.number, title: p.title, state: p.state, merged: p.merged, base: p.base, head: p.head,
|
||||
head_sha: p.head_sha, merge_commit_sha: p.merge_commit_sha, html_url: p.html_url };
|
||||
head_sha: p.head_sha, merge_commit_sha: p.merge_commit_sha, merged_at: p.merged_at, html_url: p.html_url,
|
||||
...(p.mergeable === undefined ? {} : { mergeable: p.mergeable }) };
|
||||
if (!ref) ref = p.head_sha ?? "";
|
||||
if (!base) base = p.base ?? "";
|
||||
if (!ref) throw new Error(`${owner}/${repo}#${number} names no head commit`);
|
||||
@@ -527,8 +560,18 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
} catch (err) {
|
||||
unreadable = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
// Whether this account may write the repository at all (issue 1945): a commit status and a merge need it, and a
|
||||
// repository of a user it only reads refuses both 403. Not said when the forge would not answer.
|
||||
let writes: boolean | undefined;
|
||||
try {
|
||||
writes = (await gitea.getRepo(owner, repo)).writes;
|
||||
} catch {
|
||||
writes = undefined;
|
||||
}
|
||||
const merge = mergeableUnder(base, rule, statuses, unreadable);
|
||||
return { commit: combined.sha || ref, pull, combined: combined.state, statuses,
|
||||
merge: mergeableUnder(base, rule, statuses, unreadable) };
|
||||
merge: writes === undefined ? merge : { ...merge, writes,
|
||||
...(writes ? {} : { cannot_write: `the forge account cannot write ${owner}/${repo}: gitea_grant_write` }) } };
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -539,19 +582,20 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
},
|
||||
{
|
||||
name: "gitea_commit_trees",
|
||||
description: "A commit's trees from the forge's one recursive listing: the root tree, and the tree of every directory that holds a module.json (the root as an empty name), each by its git hash. What the forge bridge puts in a merge's message (novox/hq ADR 0437 §5). With paths, {directory: [path relative to it]}, the answer also carries paths, {directory: {path: git object id}} of each named file or directory at the commit (novox/hq ADR 0499 §3); a path the commit lacks fails naming the directory and the path. The commit is a full 40-hex sha. A listing the forge cut is refused, never answered in part. Reads only.",
|
||||
description: "A commit's trees from the forge's one recursive listing: the root tree, and the tree of every directory that holds a module.json (the root as an empty name), each by its git hash. What the forge bridge puts in a merge's message (novox/hq ADR 0437 §5). With paths, {directory: [path relative to it]}, the answer also carries paths, {directory: {path: git object id}} of each named file or directory at the commit (novox/hq ADR 0499 §3); a path the commit lacks fails naming the directory and the path. The commit is a full 40-hex sha. With listed, [directory], the answer also carries listed, {directory: {name: git blob id}} of the files directly under each (the catalogue's placement files, novox/hq ADR 0549 §2). A listing the forge cut is refused, never answered in part. Reads only.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
sha: { type: "string", description: "the commit, its full 40-hex sha" },
|
||||
paths: { type: "object", description: "optional, {directory: [path relative to it]}: the git object id of each named file or directory is answered under paths; the root directory is the empty name" },
|
||||
listed: { type: "array", items: { type: "string" }, description: "optional, [directory]: the files directly under each, {directory: {name: git blob id}}, are answered under listed; a directory the commit lacks lists none" },
|
||||
},
|
||||
run: async (args) => {
|
||||
const owner = String(args.owner ?? "").trim(), repo = String(args.repo ?? "").trim();
|
||||
const sha = String(args.sha ?? "").trim();
|
||||
if (!owner || !repo) throw new Error("name the repository: owner and repo");
|
||||
if (!/^[0-9a-f]{40}$/i.test(sha)) throw new Error("sha is a commit's full 40-hex sha");
|
||||
return gitea.commitTrees(owner, repo, sha, namedPaths(args.paths));
|
||||
return gitea.commitTrees(owner, repo, sha, namedPaths(args.paths), listedDirs(args.listed));
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,321 @@
|
||||
// The account that sets commit statuses and merges writes every repository that builds a registered module, and no other
|
||||
// (the operator, 2026-10-26: "yes the mesh should get write access", on the repositories the mesh delivers; novox/hq issue
|
||||
// 1945). car-hunter #2 (a repository of a user, not of the organisation novox) was approved, and mesh-delivery's first
|
||||
// act, its commit status, was refused 403: the tools' account was only a collaborator that reads there (readall.ts).
|
||||
//
|
||||
// **Which account.** The tools' account, `mesh-tools`: the gitea module's own tools set every commit status and merge
|
||||
// (gitea_commit_status, gitea_merge_pull_request) with its token. The build modules' `nox-builder` and the machines'
|
||||
// `nox-agent` only fetch, and keep read (readall.ts).
|
||||
//
|
||||
// **Which repositories: the indexer's word, read over the bus.** mesh-delivery-indexer keeps every registered module in its
|
||||
// state `modules` (mesh-sdk go/delivery ModulesBucket), each value naming the repository it builds from. This module reads
|
||||
// that state (its manifest's `reads`) with a watch: the watch hands the latest value of every key when it starts, and then
|
||||
// every change, so a repository is granted at the module's start and when a module of it is first registered. Nothing
|
||||
// polls. A repository of the organisation needs nothing: the organisation's team writes it (tools-account.ts). Every other
|
||||
// repository of a user (the operator's dotfiles, say) is left as it is: read only.
|
||||
//
|
||||
// **Every grant is read back** from the forge (its permission for the account), and said on the bus as
|
||||
// `repo.writable` {owner, repo, account}: mesh-delivery-gate, which does not ask the operator's Approve of a pull request
|
||||
// whose merge the forge would refuse, reads that repository again on it.
|
||||
//
|
||||
// **A repository of a user is written, and main's rule is left as the operator set it** (the operator, 2026-10-26, asked
|
||||
// "Write, rule untouched"): the account is a write collaborator there, and only added to main's merge whitelist when that
|
||||
// rule keeps one (client.ts ensureMergerListed). What else in main's rule stops the mesh's merge (a status the mesh does not
|
||||
// set, approvals the gate does not give) is changed by nobody but said: both tools list such a repository as writable, but
|
||||
// blocked, with why (merge_blocked).
|
||||
//
|
||||
// Not taken away: a repository whose last module goes from the indexer keeps its collaborator until a hand removes it;
|
||||
// gitea_unwritable_repos and gitea_grant_write say the delivered list as it is now.
|
||||
|
||||
import type { ForgeRepository } from "./client.js";
|
||||
import type { BranchProtection } from "./protection.js";
|
||||
import { CHECK_CONTEXT, CHECK_STATUS, REPO_CHECK_CONTEXT } from "./pulls.js";
|
||||
|
||||
/** The indexer's state this module reads: one key per registered module, its value naming the repository. */
|
||||
export const INDEXER_MODULES = "mesh-delivery-indexer.modules";
|
||||
/** The event said once an account writes a repository it could not write before. */
|
||||
export const WRITABLE_EVENT = "repo.writable";
|
||||
|
||||
/** What the module asks of the forge for this, with the site administrator's login (GiteaAdmin, client.ts). */
|
||||
export interface WriteForge {
|
||||
everyRepository(): Promise<ForgeRepository[]>;
|
||||
repository(owner: string, repo: string): Promise<ForgeRepository | null>;
|
||||
repositoryPermission(owner: string, repo: string, user: string): Promise<string>;
|
||||
addWriteCollaborator(owner: string, repo: string, user: string): Promise<void>;
|
||||
mainRule(owner: string, repo: string): Promise<BranchProtection | null>;
|
||||
ensureMergerListed(owner: string, repo: string, account: string): Promise<{ changed: boolean; rule: BranchProtection | null }>;
|
||||
}
|
||||
|
||||
/** The statuses every pull request the mesh delivers carries (mesh-delivery's checks); a rule requiring another blocks. */
|
||||
const MESH_CONTEXTS = new Set([CHECK_STATUS, CHECK_CONTEXT, REPO_CHECK_CONTEXT]);
|
||||
|
||||
/** What in main's rule stops the mesh's merge, in words; empty when nothing does. Changed by nobody here: the operator's. */
|
||||
export function mergeBlocks(rule: BranchProtection | null, account: string): string[] {
|
||||
if (!rule) return [];
|
||||
const blocks: string[] = [];
|
||||
if (rule.enable_merge_whitelist === true && !(rule.merge_whitelist_usernames ?? []).includes(account)) {
|
||||
blocks.push(`its merge whitelist [${(rule.merge_whitelist_usernames ?? []).join(", ")}] leaves out ${account} (gitea_grant_write adds it)`);
|
||||
}
|
||||
if (rule.enable_status_check === true) {
|
||||
for (const c of rule.status_check_contexts ?? []) {
|
||||
if (!MESH_CONTEXTS.has(c)) blocks.push(`it requires the status ${c}, which the mesh does not set`);
|
||||
}
|
||||
}
|
||||
const approvals = Number(rule.required_approvals ?? 0);
|
||||
if (approvals > 0) blocks.push(`it requires ${approvals} approving review${approvals === 1 ? "" : "s"} on the forge, which the gate does not give`);
|
||||
if (rule.require_signed_commits === true) blocks.push("it requires signed commits, which the mesh does not make");
|
||||
if (rule.block_on_outdated_branch === true) blocks.push("it refuses a branch behind main, which the mesh does not bring up to date");
|
||||
if (rule.block_on_rejected_reviews === true) blocks.push("it refuses a merge after a review asked for changes, which the gate does not clear");
|
||||
return blocks;
|
||||
}
|
||||
|
||||
const blockedWhy = (blocks: string[]): string => `writable, but main's rule blocks the mesh's merge: ${blocks.join("; ")}`;
|
||||
|
||||
/** One change of the indexer's modules state, as the SDK's state watch hands it. */
|
||||
export interface ModulesChange {
|
||||
key: string;
|
||||
op: "put" | "delete";
|
||||
value?: unknown;
|
||||
revision?: number;
|
||||
current?: boolean;
|
||||
}
|
||||
|
||||
export interface Written {
|
||||
account: string;
|
||||
/** The repositories that build a registered module, as the indexer says now. */
|
||||
delivered: string[];
|
||||
/** Made writable now, each read back from the forge. */
|
||||
granted: string[];
|
||||
/** Written already (a collaborator of a user's repository), or written through the organisation's team. */
|
||||
already: string[];
|
||||
/** Each repository of a user: whether the account was added to main's merge whitelist now, and the whitelist read back
|
||||
* (null: main's rule keeps no whitelist, or there is no rule, and whoever writes may merge). */
|
||||
merge_whitelist: { repository: string; changed: boolean; listed: string[] | null }[];
|
||||
/** Writable, but main's rule (the operator's, left as set) stops the mesh's merge, and why. */
|
||||
merge_blocked: { repository: string; why: string }[];
|
||||
/** The forge refused, in its own words; or the repository is the organisation's, whose team should write it. */
|
||||
refused: { repository: string; why: string }[];
|
||||
/** Repositories the account cannot write that are left so, and why: they build no registered module. */
|
||||
left_read_only: { repository: string; permission: string; why: string }[];
|
||||
}
|
||||
|
||||
const WRITES = new Set(["write", "admin", "owner"]);
|
||||
const said = (err: unknown): string => (err instanceof Error ? err.message : String(err));
|
||||
|
||||
function split(full: string): [string, string] {
|
||||
const [owner, name, ...rest] = String(full ?? "").split("/");
|
||||
if (!owner || !name || rest.length) throw new Error(`"${full}" is not a repository: say owner/name`);
|
||||
return [owner, name];
|
||||
}
|
||||
|
||||
export class DeliveredWrites {
|
||||
/** Module to the repository it builds from, as the indexer's state says. */
|
||||
readonly modules = new Map<string, string>();
|
||||
/** Whether the watch has handed the indexer's state at all: until then, which repositories are delivered is not known. */
|
||||
heard = false;
|
||||
/** Repositories seen written by the account in this process: a module registered again asks nothing more. */
|
||||
readonly writable = new Set<string>();
|
||||
|
||||
constructor(private readonly forge: WriteForge, readonly account: string, readonly org: string,
|
||||
private readonly log: (line: string) => void = (l) => console.log(`[gitea] ${l}`),
|
||||
private readonly emit: (type: string, body: unknown) => Promise<void> = async () => {}) {}
|
||||
|
||||
/** The repositories that build a registered module, sorted. */
|
||||
delivered(): string[] {
|
||||
return [...new Set(this.modules.values())].sort();
|
||||
}
|
||||
|
||||
/** One change of the indexer's modules: a repository newly delivered is made writable now. Never throws. */
|
||||
async take(change: ModulesChange): Promise<void> {
|
||||
this.heard = true;
|
||||
if (change.op === "delete") {
|
||||
this.modules.delete(change.key);
|
||||
return;
|
||||
}
|
||||
const repository = String((change.value as { repository?: unknown } | undefined)?.repository ?? "").trim();
|
||||
try {
|
||||
split(repository);
|
||||
} catch {
|
||||
this.log(`${INDEXER_MODULES} ${change.key} names no repository (${JSON.stringify(repository)}): no write is granted for it`);
|
||||
return;
|
||||
}
|
||||
this.modules.set(change.key, repository);
|
||||
if (this.writable.has(repository)) return;
|
||||
try {
|
||||
const r = await this.grant([repository]);
|
||||
for (const g of r.granted) this.log(`${this.account} was given write on ${g}, which builds the registered module ${change.key}`);
|
||||
for (const x of r.refused) this.log(`${this.account} could not be given write on ${x.repository}, which builds ${change.key} — ${x.why}`);
|
||||
} catch (err) {
|
||||
this.log(`whether ${this.account} writes ${repository} (module ${change.key}) could not be settled — ${said(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Grant write on every delivered repository the account cannot write (or on those named, each of which must be
|
||||
* delivered); with none named, also say which repositories it cannot write are left so, and why. */
|
||||
async grant(only?: string[]): Promise<Written> {
|
||||
if (!this.heard) {
|
||||
throw new Error(`which repositories build a registered module is not known yet: ${INDEXER_MODULES} has not been read ` +
|
||||
"(the watch is not granted or not answered); nothing is granted");
|
||||
}
|
||||
const delivered = this.delivered();
|
||||
const result: Written = { account: this.account, delivered, granted: [], already: [], merge_whitelist: [], merge_blocked: [], refused: [], left_read_only: [] };
|
||||
const named = only && only.length ? only.map((x) => String(x).trim()) : null;
|
||||
const targets: string[] = [];
|
||||
for (const full of named ?? delivered) {
|
||||
split(full);
|
||||
if (delivered.includes(full)) targets.push(full);
|
||||
else result.left_read_only.push({ repository: full, permission: "unchanged",
|
||||
why: "builds no registered module (the indexer names none from it), so the account is given no write there" });
|
||||
}
|
||||
for (const full of targets) {
|
||||
const [owner, name] = split(full);
|
||||
let permission: string;
|
||||
try {
|
||||
if (!(await this.forge.repository(owner, name))) {
|
||||
result.refused.push({ repository: full, why: "the forge holds no repository by that name" });
|
||||
continue;
|
||||
}
|
||||
permission = await this.forge.repositoryPermission(owner, name, this.account);
|
||||
} catch (err) {
|
||||
result.refused.push({ repository: full, why: `its permission could not be read — ${said(err)}` });
|
||||
continue;
|
||||
}
|
||||
if (owner === this.org) {
|
||||
if (WRITES.has(permission)) {
|
||||
this.writable.add(full);
|
||||
result.already.push(full);
|
||||
continue;
|
||||
}
|
||||
// The organisation's repositories are written through its team (tools-account.ts), never a collaborator.
|
||||
result.refused.push({ repository: full, why: `a repository of ${this.org}, which ${this.account} writes through the ` +
|
||||
`team ${this.org}/${this.account}; the forge gives it "${permission}", so the team is not as it should be (it is ` +
|
||||
"set again at the next mint of the tools' token)" });
|
||||
continue;
|
||||
}
|
||||
// A repository of a user: the account writes it, and is put on main's merge whitelist when the rule keeps one.
|
||||
let changed = false;
|
||||
if (!WRITES.has(permission)) {
|
||||
try {
|
||||
await this.forge.addWriteCollaborator(owner, name, this.account);
|
||||
} catch (err) {
|
||||
result.refused.push({ repository: full, why: said(err) });
|
||||
continue;
|
||||
}
|
||||
let after: string;
|
||||
try {
|
||||
after = await this.forge.repositoryPermission(owner, name, this.account);
|
||||
} catch (err) {
|
||||
result.refused.push({ repository: full, why: `granted, but its permission could not be read back — ${said(err)}` });
|
||||
continue;
|
||||
}
|
||||
if (!WRITES.has(after)) {
|
||||
result.refused.push({ repository: full, why: `the grant was answered, but the forge still gives ${this.account} "${after}"` });
|
||||
continue;
|
||||
}
|
||||
changed = true;
|
||||
}
|
||||
try {
|
||||
const { changed: listedNow, rule } = await this.forge.ensureMergerListed(owner, name, this.account);
|
||||
result.merge_whitelist.push({ repository: full, changed: listedNow,
|
||||
listed: rule?.enable_merge_whitelist === true ? (rule.merge_whitelist_usernames ?? []) : null });
|
||||
const blocks = mergeBlocks(rule, this.account);
|
||||
if (blocks.length > 0) result.merge_blocked.push({ repository: full, why: blockedWhy(blocks) });
|
||||
} catch (err) {
|
||||
result.refused.push({ repository: full, why: `${this.account} writes it, but could not be put on main's merge whitelist — ${said(err)}` });
|
||||
continue;
|
||||
}
|
||||
this.writable.add(full);
|
||||
if (!changed) {
|
||||
result.already.push(full);
|
||||
continue;
|
||||
}
|
||||
result.granted.push(full);
|
||||
await this.emit(WRITABLE_EVENT, { owner, repo: name, account: this.account }).catch((err) =>
|
||||
this.log(`${this.account} writes ${full} now, and that could not be said on the bus — ${said(err)}`));
|
||||
}
|
||||
if (!named) {
|
||||
for (const repo of await this.forge.everyRepository()) {
|
||||
if (repo.owner === this.org || delivered.includes(repo.full_name)) continue;
|
||||
let permission: string;
|
||||
try {
|
||||
permission = await this.forge.repositoryPermission(repo.owner, repo.name, this.account);
|
||||
} catch (err) {
|
||||
permission = `unknown — ${said(err)}`;
|
||||
}
|
||||
if (WRITES.has(permission)) continue;
|
||||
result.left_read_only.push({ repository: repo.full_name, permission,
|
||||
why: "builds no registered module (the indexer names none from it), so the account is given no write there" });
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Every delivered repository the account cannot write, as the forge says now, and why. */
|
||||
async unwritable(): Promise<{ account: string; delivered: number; unwritable: { repository: string; permission: string; why: string }[];
|
||||
merge_blocked: { repository: string; why: string }[] }> {
|
||||
if (!this.heard) throw new Error(`which repositories build a registered module is not known yet: ${INDEXER_MODULES} has not been read`);
|
||||
const delivered = this.delivered();
|
||||
const out: { repository: string; permission: string; why: string }[] = [];
|
||||
const blocked: { repository: string; why: string }[] = [];
|
||||
for (const full of delivered) {
|
||||
const [owner, name] = split(full);
|
||||
let permission: string;
|
||||
try {
|
||||
permission = await this.forge.repositoryPermission(owner, name, this.account);
|
||||
} catch (err) {
|
||||
out.push({ repository: full, permission: "unknown", why: `the forge did not answer ${this.account}'s permission on it — ${said(err)}` });
|
||||
continue;
|
||||
}
|
||||
if (WRITES.has(permission)) {
|
||||
if (owner === this.org) continue; // the organisation's main rules are the mesh's own (mainrules.ts)
|
||||
try {
|
||||
const blocks = mergeBlocks(await this.forge.mainRule(owner, name), this.account);
|
||||
if (blocks.length > 0) blocked.push({ repository: full, why: blockedWhy(blocks) });
|
||||
} catch (err) {
|
||||
blocked.push({ repository: full, why: `writable, but main's rule could not be read — ${said(err)}` });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const modules = [...this.modules].filter(([, r]) => r === full).map(([m]) => m).sort();
|
||||
out.push({ repository: full, permission, why: `builds ${modules.join(", ")}, and the forge gives ${this.account} ` +
|
||||
`"${permission}" on it: its commit status and its merge are refused until gitea_grant_write grants it` });
|
||||
}
|
||||
return { account: this.account, delivered: delivered.length, unwritable: out, merge_blocked: blocked };
|
||||
}
|
||||
}
|
||||
|
||||
type Tool = { name: string; description: string; input: Record<string, unknown>; run: (args: Readonly<Record<string, unknown>>) => Promise<unknown> };
|
||||
|
||||
/** The two tools: which delivered repositories the merging account cannot write, and the grant now. */
|
||||
export function writeTools(writes: () => DeliveredWrites): Tool[] {
|
||||
return [
|
||||
{
|
||||
name: "gitea_unwritable_repos",
|
||||
description: "Every repository that builds a registered module (the indexer's modules, read over the bus) on which " +
|
||||
"the mesh's merging forge account (mesh-tools, which sets commit statuses and merges) cannot write, with the " +
|
||||
"permission the forge gives it there and why: such a pull request can be checked but never merged, and the " +
|
||||
"delivery gate does not ask its Approve. Empty when it writes every one. Its merge_blocked lists each repository of " +
|
||||
"a user it writes whose main's rule (the operator's, left as set) still stops the mesh's merge: a merge whitelist " +
|
||||
"without it, a required status the mesh does not set, required approving reviews. Read with the site administrator's login.",
|
||||
input: {},
|
||||
run: async () => writes().unwritable(),
|
||||
},
|
||||
{
|
||||
name: "gitea_grant_write",
|
||||
description: "Give the mesh's merging forge account (mesh-tools) write, as a collaborator, on every repository of a " +
|
||||
"user that builds a registered module, or on the repositories named (each must build one). Main's rule there is left " +
|
||||
"as the operator set it; only when it keeps a merge whitelist is mesh-tools added to it, read back (merge_whitelist). " +
|
||||
"What else in that rule stops the mesh's merge (a required status the mesh does not set, required approving reviews) " +
|
||||
"is changed by nobody and said in merge_blocked. " +
|
||||
"The module does this itself at its start and when a module of a new repository is registered; this is the same " +
|
||||
"act by hand, such as when a merge or a commit status was refused 403. A repository of the organisation novox " +
|
||||
"is written through its team and needs nothing. Answers what was granted (each read back from the forge), what " +
|
||||
"was written already, what the forge refused in its own words, and which other repositories it left read-only " +
|
||||
"and why (they build no registered module). Needs the site administrator's login, used for this grant alone.",
|
||||
input: {
|
||||
repositories: { type: "array", items: { type: "string" }, description: "owner/name of each repository; empty or absent, every delivered one" },
|
||||
},
|
||||
run: async (args) => writes().grant(Array.isArray(args.repositories) ? args.repositories.map(String) : undefined),
|
||||
},
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# the copies of this package in every module that carries one.
|
||||
modules/xorg/internal/desktop/*.go
|
||||
modules/lemurs/internal/desktop/*.go
|
||||
modules/i3/internal/desktop/*.go
|
||||
modules/xterm/internal/desktop/*.go
|
||||
modules/adwaita/internal/desktop/*.go
|
||||
@@ -0,0 +1,9 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole;
|
||||
# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept
|
||||
# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing.
|
||||
# the copies of this package in every module that carries one.
|
||||
modules/xorg/internal/desktop/*.go
|
||||
modules/lemurs/internal/desktop/*.go
|
||||
modules/i3/internal/desktop/*.go
|
||||
modules/xterm/internal/desktop/*.go
|
||||
modules/adwaita/internal/desktop/*.go
|
||||
@@ -102,7 +102,11 @@ integration level, the postgres major and the nats-server version; and, when the
|
||||
workspace (`beside`), each one's repository and commit, which the answer echoes as `beside` (the checker keeps a verdict only
|
||||
for the siblings it was judged with). The text's form is `DigestVersion` 3. The Go version is not in it: the toolchain enters a
|
||||
fingerprint once, as the version the build nodes agree on (ADR 0477 §3), and the standard library's packages are that
|
||||
version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors
|
||||
version's. A package with no tests at the level has no digest and is named in `without_tests`. A catalogue module's packages also hash the module's non-Go files (the nearest ancestor directory holding `module.json`: its manifest, README, `files/` and the rest, so `../../module.json` is covered; digest form 5). A package whose directory holds a file named `reads-outside` (any content; say what the tests read) is never kept (issue 1935): its tests read
|
||||
files outside it, which a digest from `go list` cannot name, so its digest is a fresh random one on every call, the ask finds no kept
|
||||
verdict for it and runs it, and the answer names it in `reads_outside` and the module's log says why. The checker still writes a
|
||||
verdict under that digest, which nothing ever reads again. The marker is the plain way to say it until each verdict keeps the files
|
||||
its tests opened (Go's test log). A directory that vendors
|
||||
is listed and tested from its vendor directory (`-mod=vendor`).
|
||||
|
||||
## A change to this module is judged by its candidate
|
||||
|
||||
@@ -14,6 +14,7 @@ var Kept = []announce.State{
|
||||
{Name: "toolchains", Readable: true},
|
||||
{Name: "keys", Readable: true},
|
||||
{Name: "steps", Readable: true},
|
||||
{Name: "pauses", Readable: true},
|
||||
}
|
||||
|
||||
// keepStates announces Kept as this module's (stdio.Keeps): main calls it before it serves.
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
@@ -43,6 +44,7 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps {
|
||||
return delivery.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()}, nil
|
||||
},
|
||||
Kept: m.KeptLookup,
|
||||
Has: m.Has,
|
||||
Release: func(ws string) error { return m.spaces().Release(ws) },
|
||||
Logf: m.Say,
|
||||
}
|
||||
@@ -80,19 +82,31 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
|
||||
source = buildstep.SeatAsks{Language: delivery.AskGo}
|
||||
}
|
||||
capacity := m.Capacity
|
||||
// The capacity's own change wakes the puller (issue 1887): the runtime's watch of the config value as mesh-config
|
||||
// resolved it; the settings file is read only when that watch cannot start. A test's Capacity is its own.
|
||||
var watch func(string, func(stdio.ProvisionChange) error) error
|
||||
// And the CPU bounds it (issue 1806's follow-up): the configured capacity is an upper bound, the node's build slice
|
||||
// gives floor(its quota's cores / the module's cores-per-step) at once, read live; a test judges by its own Capacity.
|
||||
// And the node's build hours (the operator, 2026-10-26: g14 builds only from 18:00 to 06:00): outside them it takes
|
||||
// nothing new.
|
||||
var cores func() float64
|
||||
var hours func() (*buildstep.BuildWindow, string)
|
||||
if capacity == nil {
|
||||
hours = buildstep.NodeBuildHours(m.Settings, m.Node, m.Say)
|
||||
capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say)
|
||||
watch = buildstep.WatchConfigServed
|
||||
cores = buildstep.NodeCoresPerStep(m.Settings, buildstep.DefaultCoresPerStep[delivery.AskGo], m.Say)
|
||||
}
|
||||
ask := m.askSteps(steps)
|
||||
var load func(string) ([]byte, error)
|
||||
if m.ReadSteps.Get != nil {
|
||||
if m.ReadPauses.Get != nil {
|
||||
load = func(key string) ([]byte, error) {
|
||||
raw, err := m.ReadSteps.Get(key)
|
||||
raw, err := m.ReadPauses.Get(key)
|
||||
return raw, err
|
||||
}
|
||||
}
|
||||
return &buildstep.Puller{Source: source, Language: delivery.AskGo, Node: m.Node, Put: m.Steps, Load: load,
|
||||
Capacity: capacity, Logf: m.Say,
|
||||
return &buildstep.Puller{Source: source, Language: delivery.AskGo, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load,
|
||||
Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Logf: m.Say, Has: m.Has,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
if err := m.Check(ctx); err != nil {
|
||||
return nil, err
|
||||
@@ -100,6 +114,9 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.Compose != nil {
|
||||
return m.compose(ctx, ask.Fetch, a)
|
||||
}
|
||||
return buildstep.RunAsk(ctx, ask, a)
|
||||
}}
|
||||
}
|
||||
@@ -108,3 +125,18 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
|
||||
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
|
||||
return buildstep.SeatTools(Seat, "Go", p)
|
||||
}
|
||||
|
||||
// StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes.
|
||||
const StepEndTool = "build_go_step_end"
|
||||
|
||||
// endLeftRunning is the module's start sweep (issue 1890): it ends the steps of its node that a previous process left
|
||||
// running (buildstep.EndStaleAtStart; every step names its owner). main runs it once at the start, and the module's test
|
||||
// runs it through the conformance kit's left-running case.
|
||||
func (m *Module) endLeftRunning(ctx context.Context, scan buildstep.Scan, booted time.Time, logf func(string, ...any)) {
|
||||
buildstep.EndStaleAtStart(ctx, scan, m.Node, booted, logf)
|
||||
}
|
||||
|
||||
// stepScan is the module's steps state as buildstep reads it.
|
||||
func (m *Module) stepScan() buildstep.Scan {
|
||||
return buildstep.ScanOf(stdio.State(delivery.StepsState))
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -14,16 +13,25 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
// checkCommit is the commit the check ask fetches, apart from the delivery ask's, so the fetch order names which ran first.
|
||||
const checkCommit = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
|
||||
// 1887/AC-order, 1887/AC-pull against the real bus: a check ask published before a delivery ask runs after it, and at
|
||||
// capacity 1 the second waits in the stream until the first ends.
|
||||
//
|
||||
// **A bus of its own.** The integration level runs every directory's tests at once on one bus, and this test names the
|
||||
// build seat's stream and its fixed workers (SEAT_NODE_BUILD_GO and its workers) and deletes the stream at its end, so
|
||||
// anything else on that bus, a puller or a leftover consumer, could hold or take its delivery ask and let the check
|
||||
// ask run first. It starts from testbus (mesh-sdk, novox/hq ADR 0495 clause 5): a bus for this test alone, gone after it.
|
||||
//
|
||||
// **The order is read from the record.** At capacity 1 the puller starts the check the instant the delivery ask's step
|
||||
// ends, well before any poll, so "the check key holds a state" is true on correct behaviour (issue 1887). The test
|
||||
// waits for both asks to end and compares the step records' own Started and Ended times, and the fetch order by commit.
|
||||
func TestTheModuleTakesADeliveryAskBeforeAnEarlierCheck(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS names no bus: the integration level starts one")
|
||||
}
|
||||
nc, err := nats.Connect(url)
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -40,6 +48,7 @@ func TestTheModuleTakesADeliveryAskBeforeAnEarlierCheck(t *testing.T) {
|
||||
t.Cleanup(func() { _ = js.DeleteStream(context.Background(), "SEAT_NODE_BUILD_GO") })
|
||||
check := deliveryAsk()
|
||||
check.Kind = delivery.AskCheck
|
||||
check.Build.Fetch["commit"] = checkCommit
|
||||
first := deliveryAsk()
|
||||
for _, a := range []delivery.StepAsk{check, first} {
|
||||
raw, _ := delivery.EncodeStepAsk(a)
|
||||
@@ -68,30 +77,36 @@ func TestTheModuleTakesADeliveryAskBeforeAnEarlierCheck(t *testing.T) {
|
||||
p := m.Puller(steps, &buildstep.JetStreamAsks{JS: js, Language: delivery.AskGo})
|
||||
p.PullWait = 200 * time.Millisecond
|
||||
go p.Pull(ctx)
|
||||
for _, a := range []delivery.StepAsk{first, check} {
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(10 * time.Millisecond) {
|
||||
states.mu.Lock()
|
||||
v := states.steps[key]
|
||||
states.mu.Unlock()
|
||||
if len(v) > 0 && v[len(v)-1].State != delivery.StepRunning {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the ask %s (%s) never ended", a.ID, a.Kind)
|
||||
}
|
||||
}
|
||||
var ran map[string][]delivery.Step
|
||||
for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(10 * time.Millisecond) {
|
||||
ran = map[string][]delivery.Step{}
|
||||
ended := 0
|
||||
states.mu.Lock()
|
||||
checkKey, _ := delivery.AskStepKey("ace", check.ID)
|
||||
if a.ID == first.ID && len(states.steps[checkKey]) > 0 {
|
||||
states.mu.Unlock()
|
||||
t.Fatal("the check ask started before the delivery ask ended at capacity 1")
|
||||
for _, a := range []delivery.StepAsk{first, check} {
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
v := append([]delivery.Step(nil), states.steps[key]...)
|
||||
ran[a.ID] = v
|
||||
if len(v) > 0 && v[len(v)-1].State != delivery.StepRunning {
|
||||
ended++
|
||||
}
|
||||
}
|
||||
states.mu.Unlock()
|
||||
if ended == 2 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the asks never both ended: %v", ran)
|
||||
}
|
||||
}
|
||||
// The check's step may start the moment the delivery's ends, so the order is read from the records' own times.
|
||||
deliveryEnded := ran[first.ID][len(ran[first.ID])-1].Ended
|
||||
checkStarted := ran[check.ID][0].Started
|
||||
if checkStarted.Before(deliveryEnded) {
|
||||
t.Fatalf("the check ask started at %v, before the delivery ask ended at %v at capacity 1", checkStarted, deliveryEnded)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(order) != 2 {
|
||||
t.Fatalf("fetched %v", order)
|
||||
if len(order) != 2 || order[0] != askCommit || order[1] != checkCommit {
|
||||
t.Fatalf("fetched %v, want the delivery ask's commit %s first, then the check's %s", order, askCommit, checkCommit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -125,12 +126,28 @@ func TestManifestClaimsTheGoBuildSeatAndReadsItsCapacity(t *testing.T) {
|
||||
raw, _ := os.ReadFile(moduleDir + "/module.json")
|
||||
var doc struct {
|
||||
Config map[string]struct {
|
||||
Default string `json:"default"`
|
||||
Default json.RawMessage `json:"default"`
|
||||
} `json:"config"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &doc)
|
||||
if doc.Config[buildstep.CapacitySetting].Default != buildstep.DefaultCapacities {
|
||||
t.Errorf("the capacity's default = %q, want %q", doc.Config[buildstep.CapacitySetting].Default, buildstep.DefaultCapacities)
|
||||
// Issue 1806's follow-up: the cores a step needs is a setting of its own, in the settings file, with the SDK's default.
|
||||
if cores := string(doc.Config[buildstep.CoresPerStepSetting].Default); cores != fmt.Sprint(buildstep.DefaultCoresPerStep[delivery.AskGo]) {
|
||||
t.Errorf("the cores-per-step default = %s, want %g", cores, buildstep.DefaultCoresPerStep[delivery.AskGo])
|
||||
}
|
||||
if got := buildstep.NodeCoresPerStep(moduleDir+"/files/settings.json", 0, nil)(); got != buildstep.DefaultCoresPerStep[delivery.AskGo] {
|
||||
t.Errorf("the settings template holds cores-per-step %g", got)
|
||||
}
|
||||
// And the build hours (the operator, 2026-10-26): a setting of their own, empty by default (any hour), in the template.
|
||||
if hours := string(doc.Config[buildstep.BuildHoursSetting].Default); hours != `""` {
|
||||
t.Errorf("the build-hours default = %s, want empty", hours)
|
||||
}
|
||||
if w, fault := buildstep.NodeBuildHours(moduleDir+"/files/settings.json", "g14", t.Errorf)(); w != nil || fault != "" {
|
||||
t.Errorf("the settings template holds build hours %v (%s)", w, fault)
|
||||
}
|
||||
var capacity string
|
||||
_ = json.Unmarshal(doc.Config[buildstep.CapacitySetting].Default, &capacity)
|
||||
if capacity != buildstep.DefaultCapacities {
|
||||
t.Errorf("the capacity's default = %q, want %q", capacity, buildstep.DefaultCapacities)
|
||||
}
|
||||
settings := resource(t, m, "settings")
|
||||
if settings["path"] != "${dir:state}/settings.json" {
|
||||
@@ -174,7 +191,7 @@ func TestACheckAsksDigestIsTheToolsAnswerInTheCall(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if l := out.(delivery.CheckAnswer).Levels; len(l) != 1 || !strings.Contains(string(l[0].Digest), `"x/a":"d1"`) {
|
||||
if l := out.(delivery.CheckAnswer).Levels; len(l) != 1 || l[0].Digest.Map()["x/a"] != "d1" {
|
||||
t.Fatalf("levels = %+v, want the digest's packages", l)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,7 +116,7 @@ func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) {
|
||||
t.Errorf("the integration digest listed %q", s.command)
|
||||
}
|
||||
}
|
||||
if DigestVersion != "4" {
|
||||
if DigestVersion != "6" {
|
||||
t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,12 +183,12 @@ func (m *Module) buildEnv(dir string) []string {
|
||||
} else if modfile, ok := m.pinModfile(dir); ok {
|
||||
mod += " -modfile=" + modfile // a pin on a pull request's head the fetch placed beside the workspace (pins.go)
|
||||
}
|
||||
return []string{
|
||||
return append([]string{
|
||||
"CGO_ENABLED=0",
|
||||
"GOFLAGS=" + recipeFlags + " " + mod,
|
||||
"GOTOOLCHAIN=local",
|
||||
"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
|
||||
"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
|
||||
"GOPROXY=off",
|
||||
}
|
||||
}, m.threadEnv()...)
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// The runners of a digest or test step: this installed program, or the candidate's compiled from the workspace.
|
||||
@@ -108,7 +110,7 @@ func (m *Module) mergeCommit(ctx context.Context, workspace string) (string, err
|
||||
r := m.Run(ctx, nil, "git", "-C", filepath.Join(m.Work, workspace, "tree"), "rev-parse", "HEAD")
|
||||
commit := strings.TrimSpace(r.Stdout)
|
||||
if r.Status != 0 || r.Err != "" || commit == "" {
|
||||
return "", fmt.Errorf("the workspace's commit: git rev-parse: %s", firstLine(r.Err+"\n"+r.Stderr))
|
||||
return "", fmt.Errorf("the workspace's commit: git rev-parse: %s", forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
return commit, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
package main
|
||||
|
||||
// **A compose ask runs mesh-sdk's composer at a commit** (novox/hq ADR 0482 clause 4, issues 1621 and 1622): the
|
||||
// assigner asks it, to compose a group with its mesh-sdk member's head, and to read the group's manifests with the
|
||||
// mesh-sdk the running controller and node-engine pin (ADR 0482 clause 5, issue 1623). The module fetches the commit
|
||||
// (its fetch step: the forge key and the network, none of the change's code), builds the composer package in the Go
|
||||
// module the ask names, in a run step shut off with nothing fetched, and keeps the program under its key
|
||||
// (delivery.ComposerKey), so each commit is built once on a node. It fills the request's manifests from the delivery
|
||||
// indexer's manifests state, by digest, and runs the program shut off: no network, none of the account, the request a
|
||||
// file of a workspace of its own. The answer is the composer's verdict whole (delivery.ComposeAnswer); the module
|
||||
// judges nothing of it.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ManifestsState is the delivery indexer's manifests by digest, which a compose ask's request names (the module reads
|
||||
// it, module.json reads).
|
||||
const ManifestsState = delivery.IndexerModule + ".manifests"
|
||||
|
||||
// composerLocks hold one mutex per composer key, so a composer two asks need at once is built once.
|
||||
var composerLocks sync.Map
|
||||
|
||||
// composerPath is where the composer of key is kept: the cache directory, beside Go's own caches.
|
||||
func (m *Module) composerPath(key string) string {
|
||||
return filepath.Join(m.Cache, "composers", key, "mesh-compose")
|
||||
}
|
||||
|
||||
// compose runs a compose ask: the composer of its commit, built once, over its request with the manifests filled.
|
||||
func (m *Module) compose(ctx context.Context, fetch buildstep.StepFunc, a delivery.StepAsk) (any, error) {
|
||||
job := *a.Compose
|
||||
repository, _ := job.Fetch["repository"].(string)
|
||||
commit, _ := job.Fetch["commit"].(string)
|
||||
installed, err := m.Probe(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := delivery.ComposerKey(repository, commit, job.Dir, job.Package, delivery.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()})
|
||||
exe, built, err := m.composerOf(ctx, fetch, job, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := composer.Decode(job.Request)
|
||||
if err != nil {
|
||||
return nil, buildstep.Refuse("the compose ask %s: %v", a.ID, err)
|
||||
}
|
||||
if err := composer.Fill(&req, m.manifestOf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
verdict, err := m.runComposer(ctx, exe, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return delivery.ComposeAnswer{Built: built, Key: key, Verdict: verdict}, nil
|
||||
}
|
||||
|
||||
// manifestOf is the manifest the indexer keeps under digest, as its repository holds it.
|
||||
func (m *Module) manifestOf(digest string) ([]byte, error) {
|
||||
if m.ReadManifest == nil {
|
||||
return nil, errors.New("this module reads no manifests state")
|
||||
}
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := m.ReadManifest(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if raw == nil {
|
||||
return nil, fmt.Errorf("the indexer keeps no manifest of digest %s", digest)
|
||||
}
|
||||
man, err := delivery.DecodeManifest(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []byte(man.JSON), nil
|
||||
}
|
||||
|
||||
// composerOf is the composer kept under key, or, when none is, the one built now from the fetched commit (built true).
|
||||
func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job delivery.ComposeJob, key string) (string, bool, error) {
|
||||
exe := m.composerPath(key)
|
||||
mu, _ := composerLocks.LoadOrStore(exe, &sync.Mutex{})
|
||||
mu.(*sync.Mutex).Lock()
|
||||
defer mu.(*sync.Mutex).Unlock()
|
||||
if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() {
|
||||
return exe, false, nil
|
||||
}
|
||||
if fetch == nil {
|
||||
return "", false, buildstep.Refuse("this build module fetches nothing")
|
||||
}
|
||||
out, err := fetch(ctx, job.Fetch)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
raw, err := json.Marshal(out)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
var f struct {
|
||||
Workspace string `json:"workspace"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" {
|
||||
return "", false, fmt.Errorf("the fetch answered no workspace: %s", raw)
|
||||
}
|
||||
defer m.releaseSaid(f.Workspace)
|
||||
done := m.spaces().Hold(f.Workspace)
|
||||
defer done()
|
||||
dir := filepath.Join(m.Work, f.Workspace, "tree", job.Dir)
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil {
|
||||
return "", false, fmt.Errorf("%s at %s holds no go.mod in %s, so no composer can be built there", job.Fetch["repository"], job.Fetch["commit"], job.Dir)
|
||||
}
|
||||
into := filepath.Join(m.Work, f.Workspace, "composer", "mesh-compose")
|
||||
if err := os.MkdirAll(filepath.Dir(into), 0o700); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
run, err := m.stepRunner(f.Workspace, false)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
env := append(m.goEnvFor(dir), "GOPROXY=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0")
|
||||
r := run(ctx, env, "go", "-C", dir, "build", "-trimpath", "-o", into, job.Package)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
// The words Go says when the module holds no such package (a mesh-sdk from before the composer) are kept whole,
|
||||
// so the asker tells that apart from a fault.
|
||||
return "", false, fmt.Errorf("the composer %s does not build at %s: %s", job.Package, job.Fetch["commit"], compileError(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
if err := copyExecutable(into, exe); err != nil {
|
||||
return "", false, fmt.Errorf("the composer could not be kept: %w", err)
|
||||
}
|
||||
return exe, true, nil
|
||||
}
|
||||
|
||||
// copyExecutable copies a built program into place through a file beside it, so a half-written program is never kept.
|
||||
func copyExecutable(from, to string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(to), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
src, err := os.Open(from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
tmp := to + ".new"
|
||||
dst, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(dst, src); err != nil {
|
||||
dst.Close()
|
||||
return err
|
||||
}
|
||||
if err := dst.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, to)
|
||||
}
|
||||
|
||||
// releaseSaid removes a workspace once its steps ended, saying a failure: the sweep removes it later.
|
||||
func (m *Module) releaseSaid(workspace string) {
|
||||
if err := m.spaces().Release(workspace); err != nil && m.Say != nil {
|
||||
m.Say("the compose ask's workspace %s: %v", workspace, err)
|
||||
}
|
||||
}
|
||||
|
||||
// runComposer runs the composer over req in a workspace of its own, shut off, and answers its verdict: the last line of
|
||||
// its standard output, a composer.Verdict of the version this module reads.
|
||||
func (m *Module) runComposer(ctx context.Context, exe string, req composer.Request) (json.RawMessage, error) {
|
||||
id := buildstep.NewID()
|
||||
done := m.spaces().Hold(id)
|
||||
defer m.releaseSaid(id)
|
||||
defer done()
|
||||
ws := filepath.Join(m.Work, id)
|
||||
if err := os.MkdirAll(ws, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file := filepath.Join(ws, "request.json")
|
||||
if err := os.WriteFile(file, raw, 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
run := m.Run
|
||||
if !m.Inside {
|
||||
iso := m.isolation(id, false)
|
||||
iso.ReadOnly = append(iso.ReadOnly, exe)
|
||||
run = buildstep.Isolated(m.Run, iso)
|
||||
}
|
||||
r := run(ctx, nil, exe, "-request", file)
|
||||
line := strings.TrimSpace(lastLine(r.Stdout))
|
||||
var v composer.Verdict
|
||||
if r.Status != 0 || r.Err != "" || json.Unmarshal([]byte(line), &v) != nil || v.V != composer.Version {
|
||||
return nil, fmt.Errorf("the composer answered no verdict (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
return json.RawMessage(line), nil
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
// sdkPin is the mesh-sdk version this module's go.mod requires: the composer the test builds is that version's.
|
||||
func sdkPin(t *testing.T) string {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("../../go.mod")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, l := range strings.Split(string(raw), "\n") {
|
||||
if f := strings.Fields(l); len(f) >= 2 && f[0] == "git.novox.be/novox/mesh-sdk/go" {
|
||||
return f[1]
|
||||
}
|
||||
}
|
||||
t.Fatal("go.mod requires no mesh-sdk")
|
||||
return ""
|
||||
}
|
||||
|
||||
// goEnvOf is one value of this machine's go env.
|
||||
func goEnvOf(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
out, err := exec.Command("go", "env", name).Output()
|
||||
if err != nil {
|
||||
t.Fatalf("go env %s: %v", name, err)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
// 1622/AC1, AC3 against the real bus: a compose ask put on the Go build seat's check stream is pulled by the module
|
||||
// started whole on that bus, which fetches the commit (here a tree of a module requiring mesh-sdk at this module's pin,
|
||||
// as a reader repository is), builds mesh-sdk's composer there with nothing fetched, fills the request's manifests from
|
||||
// the indexer's state and runs the composer; the step key <node>.<ask id> passes with the composer's verdict. A second
|
||||
// ask of the same commit runs the kept composer: fetched and built once.
|
||||
func TestAComposeAskIsBuiltOnceAndAnsweredOnTheBus(t *testing.T) {
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
if err := buildstep.DeclareAskWorkers(ctx, js, delivery.AskGo, 2*time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.DeleteStream(context.Background(), "SEAT_NODE_BUILD_GO") })
|
||||
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.26.0")
|
||||
m.Work, m.Cache, m.Inside = t.TempDir(), t.TempDir(), true
|
||||
m.Run = StepRunner(t.TempDir())
|
||||
m.GoModCache = goEnvOf(t, "GOMODCACHE") // the module cache this module's own build filled, read and not fetched
|
||||
run, gocache := m.Run, goEnvOf(t, "GOCACHE") // this machine's build cache, so the composer's packages compile once
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
return run(ctx, append(env, "GOCACHE="+gocache), name, args...)
|
||||
}
|
||||
read, digests := indexerHolds(t, `{"module":"demo","version":"1"}`)
|
||||
m.ReadManifest = read
|
||||
|
||||
gosum, err := os.ReadFile("../../go.sum")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fetches := 0
|
||||
steps := fakeSteps(new([]string))
|
||||
for i := range steps {
|
||||
if steps[i].Name == "build_go_fetch" {
|
||||
steps[i].Run = func(ctx context.Context, args map[string]any) (any, error) {
|
||||
fetches++
|
||||
ws := buildstep.NewID()
|
||||
tree := filepath.Join(m.Work, ws, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mod := "module reader\n\ngo 1.26.0\n\nrequire git.novox.be/novox/mesh-sdk/go " + sdkPin(t) + "\n"
|
||||
if err := os.WriteFile(filepath.Join(tree, "go.mod"), []byte(mod), 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"workspace": ws}, os.WriteFile(filepath.Join(tree, "go.sum"), gosum, 0o600)
|
||||
}
|
||||
}
|
||||
}
|
||||
p := m.Puller(steps, &buildstep.JetStreamAsks{JS: js, Language: delivery.AskGo})
|
||||
p.PullWait = 200 * time.Millisecond
|
||||
go p.Pull(ctx)
|
||||
|
||||
ask := func() delivery.Step {
|
||||
a := composeAskOf(t, digests[0])
|
||||
a.Compose.Fetch["repository"], a.Compose.Dir = "novox/mesh-controller", delivery.RootPath
|
||||
raw, err := delivery.EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck)
|
||||
if _, err := js.Publish(ctx, subject, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
for deadline := time.Now().Add(90 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
states.mu.Lock()
|
||||
v := states.steps[key]
|
||||
states.mu.Unlock()
|
||||
if len(v) > 0 && v[len(v)-1].State != delivery.StepRunning {
|
||||
return v[len(v)-1]
|
||||
}
|
||||
}
|
||||
t.Fatalf("the compose ask %s never ended", a.ID)
|
||||
return delivery.Step{}
|
||||
}
|
||||
for n, wantBuilt := range []bool{true, false} {
|
||||
step := ask()
|
||||
if step.State != delivery.StepPassed {
|
||||
t.Fatalf("ask %d: %s: %s", n+1, step.State, step.Reason)
|
||||
}
|
||||
var ans delivery.ComposeAnswer
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(step.Result, &ans); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(ans.Verdict, &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ans.Built != wantBuilt || !strings.HasPrefix(ans.Key, "composer-") || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted ||
|
||||
!strings.Contains(v.SDK, sdkPin(t)) {
|
||||
t.Fatalf("ask %d: answer built %v key %s, verdict %+v; want built %v and demo accepted by mesh-sdk %s", n+1, ans.Built, ans.Key, v, wantBuilt, sdkPin(t))
|
||||
}
|
||||
}
|
||||
if fetches != 1 {
|
||||
t.Errorf("two asks of one commit fetched %d times, want once", fetches)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const composeCommit = "4444444444444444444444444444444444444444"
|
||||
|
||||
// indexerHolds is the delivery indexer's manifests state holding each body under its digest, as the indexer writes it.
|
||||
func indexerHolds(t *testing.T, bodies ...string) (func(string) (json.RawMessage, error), []string) {
|
||||
t.Helper()
|
||||
kept, digests := map[string]json.RawMessage{}, []string{}
|
||||
for _, b := range bodies {
|
||||
d := delivery.ManifestDigest([]byte(b))
|
||||
key, _ := delivery.ManifestsKey(d)
|
||||
raw, err := delivery.EncodeManifest(delivery.Manifest{Module: "demo", JSON: b})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept[key], digests = raw, append(digests, d)
|
||||
}
|
||||
return func(key string) (json.RawMessage, error) { return kept[key], nil }, digests
|
||||
}
|
||||
|
||||
func composeAskOf(t *testing.T, digest string) delivery.StepAsk {
|
||||
t.Helper()
|
||||
req, _ := json.Marshal(composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "change",
|
||||
Manifests: map[string]composer.Manifest{"demo": {Digest: digest}}}}})
|
||||
return delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-assigner",
|
||||
At: time.Now(), Compose: &delivery.ComposeJob{Fetch: map[string]any{"repository": "novox/mesh-sdk", "commit": composeCommit},
|
||||
Dir: "go", Package: composer.Package, Request: req}}
|
||||
}
|
||||
|
||||
// composeModule is a module with work and cache directories of its own, running its commands as they are (Inside: a
|
||||
// unit test has no bubblewrap to shut them off in).
|
||||
func composeModule(t *testing.T) *Module {
|
||||
t.Helper()
|
||||
m := moduleWith(newStates(), "go1.26.0")
|
||||
m.Work, m.Cache, m.Inside, m.Run = t.TempDir(), t.TempDir(), true, StepRunner(t.TempDir())
|
||||
return m
|
||||
}
|
||||
|
||||
// 1622/AC3: a composer kept under the ask's key is run again, nothing fetched or built: the answer says it was not
|
||||
// built, names the key, and carries the composer's verdict whole; the request it read had its manifests filled from the
|
||||
// indexer's state.
|
||||
func TestAKeptComposerIsRunWithTheManifestsFilled(t *testing.T) {
|
||||
m := composeModule(t)
|
||||
read, digests := indexerHolds(t, `{"module":"demo","version":"1"}`)
|
||||
m.ReadManifest = read
|
||||
a := composeAskOf(t, digests[0])
|
||||
key := delivery.ComposerKey("novox/mesh-sdk", composeCommit, "go", composer.Package, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()})
|
||||
exe := m.composerPath(key)
|
||||
if err := os.MkdirAll(filepath.Dir(exe), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A composer that answers which manifests its request held, filled or not, as a verdict.
|
||||
script := "#!/bin/sh\n" +
|
||||
`if grep -q '"digest"' "$2"; then echo '{"v":1,"sdk":"stand-in","cases":[{"name":"unfilled","manifests":{}}]}'; ` +
|
||||
`else echo '{"v":1,"sdk":"stand-in","cases":[{"name":"change","manifests":{"demo":{"accepted":true}}}]}'; fi` + "\n"
|
||||
if err := os.WriteFile(exe, []byte(script), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fetched := 0
|
||||
fetch := func(context.Context, map[string]any) (any, error) { fetched++; return nil, nil }
|
||||
out, err := m.compose(context.Background(), fetch, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ans := out.(delivery.ComposeAnswer)
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(ans.Verdict, &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ans.Built || ans.Key != key || fetched != 0 || len(v.Cases) != 1 || v.Cases[0].Name != "change" || !v.Cases[0].Manifests["demo"].Accepted {
|
||||
t.Fatalf("answer = %+v (fetched %d), verdict %+v", ans, fetched, v)
|
||||
}
|
||||
if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
|
||||
t.Errorf("the run's workspace was left: %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622: a manifest the indexer does not keep fails the ask, naming its digest; a fetched commit with no go.mod where
|
||||
// the ask builds fails it naming the directory, and its workspace is released.
|
||||
func TestAComposeAskFailsLoudly(t *testing.T) {
|
||||
m := composeModule(t)
|
||||
m.ReadManifest, _ = indexerHolds(t)
|
||||
a := composeAskOf(t, "feedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeed")
|
||||
key := delivery.ComposerKey("novox/mesh-sdk", composeCommit, "go", composer.Package, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()})
|
||||
os.MkdirAll(filepath.Dir(m.composerPath(key)), 0o700)
|
||||
os.WriteFile(m.composerPath(key), []byte("#!/bin/sh\nexit 0\n"), 0o700)
|
||||
if _, err := m.compose(context.Background(), nil, a); err == nil || !strings.Contains(err.Error(), "feedfeed") {
|
||||
t.Errorf("an unkept manifest: %v", err)
|
||||
}
|
||||
m2 := composeModule(t)
|
||||
fetch := func(context.Context, map[string]any) (any, error) {
|
||||
if err := os.MkdirAll(filepath.Join(m2.Work, "ws0000000000a", "tree", "go"), 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"workspace": "ws0000000000a"}, nil
|
||||
}
|
||||
if _, err := m2.compose(context.Background(), fetch, composeAskOf(t, "x")); err == nil || !strings.Contains(err.Error(), "no go.mod in go") {
|
||||
t.Errorf("a commit with no go.mod: %v", err)
|
||||
}
|
||||
if entries, _ := os.ReadDir(m2.Work); len(entries) != 0 {
|
||||
t.Errorf("the fetched workspace was left: %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
// 1622: the module reads the delivery indexer's manifests, which a compose ask names by digest.
|
||||
func TestTheModuleReadsTheIndexersManifests(t *testing.T) {
|
||||
if m := readManifest(t); !slices.Contains(m.Reads, ManifestsState) {
|
||||
t.Errorf("reads = %v, lacks %s", m.Reads, ManifestsState)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"git.novox.be/novox/mesh-sdk/go/buildcpu"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// The processors a step may take (novox/hq ADR 0553, issue 1806). The node-engine holds this module's runtime, with
|
||||
// every other build module's of the node, in one slice at idle priority under the node's one share of its processors:
|
||||
// that quota bounds the total. Threads only size each step: the share the composer gives (buildstep.NodeLimits) over
|
||||
// this module's own capacity, so a step does not wait in the kernel's throttle. Without the composer's share there is
|
||||
// no slice to size a step to, and no thread limit is set.
|
||||
|
||||
// cpuLimits is what one step is held to now.
|
||||
func (m *Module) cpuLimits() buildcpu.Limits {
|
||||
slots := 1
|
||||
switch {
|
||||
case m.Held != nil:
|
||||
slots = m.Held.CapacityNow() // the capacity the puller judges by, bounded by the node's build slice
|
||||
case m.Capacity != nil:
|
||||
slots = m.Capacity()
|
||||
case m.Settings != "":
|
||||
slots = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, nil)()
|
||||
}
|
||||
return buildstep.NodeLimits(slots)
|
||||
}
|
||||
|
||||
// threadEnv is the environment (nothing without a share) every go command of a step is given for its threads: GOMAXPROCS sets its -p, the compiler's
|
||||
// concurrency, a test binary's processors and -parallel.
|
||||
func (m *Module) threadEnv() []string {
|
||||
return buildcpu.Env(m.cpuLimits().Threads)
|
||||
}
|
||||
|
||||
// cpuStatus is what the status tool says of it: the limits a step is given, and what the kernel holds this runtime
|
||||
// to (its unit and its slice), read from its cgroup rather than assumed.
|
||||
func (m *Module) cpuStatus() map[string]any {
|
||||
out := map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()}
|
||||
if m.Held != nil {
|
||||
out["capacity"] = m.Held.CapacityDetail() // the capacity judged by and why (issue 1806's follow-up)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -7,29 +7,77 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind.
|
||||
// 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3). 3: the siblings'
|
||||
// commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement).
|
||||
// 4: the siblings' commits enter only the digest of a package whose tests reach a file naming MESH_CHECK_BESIDE (ADR 0536 §1).
|
||||
const DigestVersion = "4"
|
||||
// 5: a catalogue module's non-Go files enter the digest of its packages (issue 1935).
|
||||
// 6: a reads-outside marker that lists what its tests read enters the digest with those files' contents (issue 1936).
|
||||
const DigestVersion = "6"
|
||||
|
||||
// besideVariable is the name a file bears when it reads where the siblings lie; a package whose tests reach such a file
|
||||
// depends on the siblings' commits.
|
||||
const besideVariable = "MESH_CHECK_BESIDE"
|
||||
|
||||
// ReadsOutsideMarker is the name of a file in a package's directory that says its tests read files outside it (a
|
||||
// repository's documents, a sibling directory), which the Go toolchain does not list (issue 1935). It lists, one per
|
||||
// line, what those tests read, as paths from the repository's root; a path may be a glob (filepath.Match), and a
|
||||
// directory it names is read whole (issue 1936). Blank lines and lines starting with # are not read. A line starting
|
||||
// with ? names a path whose absence is what the tests read (a file that must not be there): it may match nothing.
|
||||
//
|
||||
// The digest then hashes, besides what the toolchain lists, every line of the marker, every file it names (under its
|
||||
// path in the tree, by content) and every file of the package's own directory, Go files included (a file its build tags
|
||||
// exclude is one go list does not hash). A marker that lists nothing, a line that is not a clean relative path, or a
|
||||
// line without ? that matches nothing makes the package one that is never kept (its digest a fresh random hex): a list
|
||||
// nobody can trust keeps nothing. The exact fix, the files a test opened kept
|
||||
// from the Go test log with each verdict, replaces the lists (issue 1936).
|
||||
const ReadsOutsideMarker = "reads-outside"
|
||||
|
||||
// NeedsMarker is the name of a file in a package's directory that names, one per line, what its integration tests need
|
||||
// of the node they run on (issue 890; mesh-sdk delivery.AskNeeds): `node-build-node` for tests that read the Node build
|
||||
// module's installed toolchain. The digest reports it (DigestResult.Needs); a check ask leaves such a package untested on
|
||||
// a node that lacks it, and the checker asks it again of a node that has it. A marker beside the tests, as
|
||||
// ReadsOutsideMarker is, and not a manifest field: it is a property of the package's tests, read only where they are
|
||||
// run, and the manifest describes the module on the machines. The unit level is never given anything of the node (ADR
|
||||
// 0533 §1), so a marker binds the integration level alone. Blank lines and lines starting with # are not read.
|
||||
const NeedsMarker = "check-needs"
|
||||
|
||||
// readNeeds is what the marker in dir names; nil when there is none.
|
||||
func readNeeds(dir string) ([]string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(dir, NeedsMarker))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var needs []string
|
||||
for _, l := range strings.Split(string(raw), "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") {
|
||||
needs = append(needs, l)
|
||||
}
|
||||
}
|
||||
return needs, nil
|
||||
}
|
||||
|
||||
// integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation
|
||||
// for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run.
|
||||
var integrationTags = []string{"integration", "isolation"}
|
||||
@@ -89,6 +137,15 @@ type DigestResult struct {
|
||||
// only when they are the siblings it gave the fetch.
|
||||
Beside []string `json:"beside,omitempty"`
|
||||
WithoutTests []string `json:"without_tests,omitempty"`
|
||||
// ReadsOutside names the packages whose marker lists nothing it can trust (ReadsOutsideMarker): each has a digest
|
||||
// that no kept verdict can have (a fresh random one), so the check runs it every time.
|
||||
ReadsOutside []string `json:"reads_outside,omitempty"`
|
||||
// neverKept is, per package of ReadsOutside, why its marker cannot be trusted; hashedFiles caches the content lines
|
||||
// of the files markers name, which several packages share (the catalogue's manifests).
|
||||
neverKept map[string]string
|
||||
hashedFiles map[string]string
|
||||
// Needs are the packages whose tests at this level need something of the node, as their NeedsMarker names it.
|
||||
Needs []delivery.PackageNeeds `json:"needs,omitempty"`
|
||||
// JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
|
||||
// runner workspace.
|
||||
JudgedBy string `json:"judged_by,omitempty"`
|
||||
@@ -194,11 +251,18 @@ func (m *Module) digestHere(ctx context.Context, workspace, level string) (Diges
|
||||
if err != nil {
|
||||
return DigestResult{}, err
|
||||
}
|
||||
if err := digestModule(&res, pkgs, tree, header, besideHeader); err != nil {
|
||||
if err := digestModule(&res, pkgs, tree, header, besideHeader, level == LevelIntegration); err != nil {
|
||||
return DigestResult{}, err
|
||||
}
|
||||
}
|
||||
sort.Strings(res.WithoutTests)
|
||||
sort.Strings(res.ReadsOutside)
|
||||
sort.Slice(res.Needs, func(i, j int) bool { return res.Needs[i].Package < res.Needs[j].Package })
|
||||
for _, p := range res.ReadsOutside {
|
||||
if m.Say != nil {
|
||||
m.Say("the %s level of %s always runs: its %s %s, so its tests read files outside it that no digest names", level, p, ReadsOutsideMarker, res.neverKept[p])
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
@@ -238,7 +302,7 @@ func parsePackages(out string) (map[string]*goPackage, error) {
|
||||
|
||||
// digestModule adds to res the digest of each root package of one go.mod directory. A root is a package of the main
|
||||
// module that is listed, not only a dependency, and not a variant of another.
|
||||
func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header, beside []string) error {
|
||||
func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header, beside []string, needs bool) error {
|
||||
roots := []string{}
|
||||
for path, p := range pkgs {
|
||||
if p.ForTest == "" && !p.DepOnly && !p.Standard && !strings.HasSuffix(path, ".test") && p.Module != nil && p.Module.Main {
|
||||
@@ -256,7 +320,36 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he
|
||||
if testMain == nil {
|
||||
return fmt.Errorf("go list gave no test binary for %s", path)
|
||||
}
|
||||
d, err := packageDigest(root, testMain, pkgs, tree, header, beside)
|
||||
if needs {
|
||||
n, err := readNeeds(root.Dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: its %s: %w", path, NeedsMarker, err)
|
||||
}
|
||||
if len(n) > 0 {
|
||||
res.Needs = append(res.Needs, delivery.PackageNeeds{Package: path, Needs: n})
|
||||
}
|
||||
}
|
||||
declared, why, err := declaredReads(tree, root, res.hashed())
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: its %s: %w", path, ReadsOutsideMarker, err)
|
||||
}
|
||||
if why != "" {
|
||||
// The digest of a package whose reads outside itself are not listed is one nothing can have kept: the key is a
|
||||
// fresh random hex, so the ask finds no kept verdict and runs it, and what the checker writes under it is never
|
||||
// read again.
|
||||
var nonce [32]byte
|
||||
if _, err := rand.Read(nonce[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
res.Packages[path] = hex.EncodeToString(nonce[:])
|
||||
res.ReadsOutside = append(res.ReadsOutside, path)
|
||||
if res.neverKept == nil {
|
||||
res.neverKept = map[string]string{}
|
||||
}
|
||||
res.neverKept[path] = why
|
||||
continue
|
||||
}
|
||||
d, err := packageDigest(root, testMain, pkgs, tree, header, beside, declared)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
@@ -269,14 +362,15 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he
|
||||
// tests reach in the module's own tree (by their content, under their path in the tree), the version and checksum of
|
||||
// every module outside it, and the package's own test data. The beside lines (the siblings' commits) join the header
|
||||
// only when a file the tests reach names MESH_CHECK_BESIDE. A package of the standard library is the Go version's.
|
||||
func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header, beside []string) (string, error) {
|
||||
// The lines a package's ReadsOutsideMarker declares join them (declaredReads).
|
||||
func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header, beside, declared []string) (string, error) {
|
||||
reached := map[string]*goPackage{root.ImportPath: root}
|
||||
for _, dep := range testMain.Deps {
|
||||
if p := pkgs[dep]; p != nil {
|
||||
reached[dep] = p
|
||||
}
|
||||
}
|
||||
lines := []string{}
|
||||
lines := append([]string{}, declared...)
|
||||
readsBeside := false
|
||||
for _, p := range reached {
|
||||
if p.Standard || p.Module == nil {
|
||||
@@ -303,6 +397,15 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s
|
||||
}
|
||||
}
|
||||
}
|
||||
// A catalogue module's tests read its manifest, its README and its files from the package's directory (../../module.json):
|
||||
// every non-Go file of the module's directory is an input (issue 1935). The module is the nearest ancestor with a module.json.
|
||||
if modDir := enclosingModule(tree, root.Dir); modDir != "" {
|
||||
more, err := moduleFileLines(tree, root, modDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
lines = append(lines, more...)
|
||||
}
|
||||
data := filepath.Join(root.Dir, "testdata")
|
||||
if _, err := os.Stat(data); err == nil {
|
||||
err := filepath.WalkDir(data, func(path string, e fs.DirEntry, err error) error {
|
||||
@@ -328,6 +431,41 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// enclosingModule is the nearest ancestor of dir, inside tree, that holds a module.json, or "" when there is none.
|
||||
func enclosingModule(tree, dir string) string {
|
||||
tree = filepath.Clean(tree)
|
||||
for d := filepath.Clean(dir); d != tree && strings.HasPrefix(d, tree+string(filepath.Separator)); d = filepath.Dir(d) {
|
||||
if st, err := os.Stat(filepath.Join(d, "module.json")); err == nil && !st.IsDir() {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// moduleFileLines are the content lines of every file of a module's directory that is not Go source (Go source is the
|
||||
// packages' own, listed by go list), skipping .git and node_modules.
|
||||
func moduleFileLines(tree string, p *goPackage, modDir string) ([]string, error) {
|
||||
var lines []string
|
||||
err := filepath.WalkDir(modDir, func(path string, e fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if e.IsDir() {
|
||||
if n := e.Name(); n == ".git" || n == "node_modules" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.HasSuffix(path, ".go") {
|
||||
return nil
|
||||
}
|
||||
l, err := contentLine(tree, p, path)
|
||||
lines = append(lines, l)
|
||||
return err
|
||||
})
|
||||
return lines, err
|
||||
}
|
||||
|
||||
func fileLine(tree string, p *goPackage, name string) (string, error) {
|
||||
return contentLine(tree, p, filepath.Join(p.Dir, name))
|
||||
}
|
||||
@@ -354,3 +492,104 @@ func contentLine(tree string, p *goPackage, path string) (string, error) {
|
||||
sum := sha256.Sum256(raw)
|
||||
return "file " + label + " " + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// hashed is the cache of content lines a digest step shares across its packages.
|
||||
func (r *DigestResult) hashed() map[string]string {
|
||||
if r.hashedFiles == nil {
|
||||
r.hashedFiles = map[string]string{}
|
||||
}
|
||||
return r.hashedFiles
|
||||
}
|
||||
|
||||
// declaredReads are the digest lines a package's ReadsOutsideMarker adds: each of its lines, the content line of every
|
||||
// file it names, and those of every file of the package's own directory. Without a marker it answers nothing. why is
|
||||
// set, and the lines nil, when the marker cannot be trusted: it lists nothing, a line is not a clean relative path, or
|
||||
// a line without ? matches nothing.
|
||||
func declaredReads(tree string, root *goPackage, cache map[string]string) (lines []string, why string, err error) {
|
||||
raw, err := os.ReadFile(filepath.Join(root.Dir, ReadsOutsideMarker))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
var patterns []string
|
||||
for _, l := range strings.Split(string(raw), "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") {
|
||||
patterns = append(patterns, l)
|
||||
}
|
||||
}
|
||||
if len(patterns) == 0 {
|
||||
return nil, "lists nothing", nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
add := func(path string) error {
|
||||
if seen[path] {
|
||||
return nil
|
||||
}
|
||||
seen[path] = true
|
||||
l, ok := cache[path]
|
||||
if !ok {
|
||||
var err error
|
||||
if l, err = contentLine(tree, root, path); err != nil {
|
||||
return err
|
||||
}
|
||||
cache[path] = l
|
||||
}
|
||||
lines = append(lines, "read "+l)
|
||||
return nil
|
||||
}
|
||||
for _, l := range patterns {
|
||||
pattern, absent := strings.CutPrefix(l, "?")
|
||||
if !cleanTreePattern(pattern) {
|
||||
return nil, fmt.Sprintf("names %q, which is not a clean path from the repository's root", l), nil
|
||||
}
|
||||
matches, err := filepath.Glob(filepath.Join(tree, filepath.FromSlash(pattern)))
|
||||
if err != nil {
|
||||
return nil, fmt.Sprintf("names %q, which is not a glob", l), nil
|
||||
}
|
||||
if len(matches) == 0 && !absent {
|
||||
return nil, fmt.Sprintf("names %q, which matches nothing (a path whose absence is read starts with ?)", l), nil
|
||||
}
|
||||
lines = append(lines, "reads "+l)
|
||||
for _, match := range matches {
|
||||
if err := walkFiles(match, add); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
// Every file of the package's own directory: its lists, its marker, and its Go files too, since go list leaves a
|
||||
// file its build tags exclude (an integration test a unit test globs) out of what the digest hashes. A file the
|
||||
// toolchain lists is hashed twice, which changes nothing.
|
||||
err = walkFiles(root.Dir, add)
|
||||
return lines, "", err
|
||||
}
|
||||
|
||||
// cleanTreePattern says a marker's line is a path from the repository's root that stays inside it.
|
||||
func cleanTreePattern(p string) bool {
|
||||
if p == "" || strings.ContainsAny(p, " \t\\") || strings.HasPrefix(p, "/") || path.Clean(p) != p {
|
||||
return false
|
||||
}
|
||||
for _, e := range strings.Split(p, "/") {
|
||||
if e == ".." || e == "." {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// walkFiles calls each with every file at or under path, skipping .git and node_modules, in lexical order.
|
||||
func walkFiles(path string, each func(string) error) error {
|
||||
return filepath.WalkDir(path, func(p string, e fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if e.IsDir() {
|
||||
if n := e.Name(); p != path && (n == ".git" || n == "node_modules") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return each(p)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -173,7 +173,7 @@ func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) {
|
||||
testMain := &goPackage{ImportPath: "x/r.test", Deps: []string{ext.ImportPath}}
|
||||
pkgs[root.ImportPath] = root
|
||||
digest := func() string {
|
||||
d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"}, nil)
|
||||
d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"}, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -189,3 +189,127 @@ func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) {
|
||||
t.Errorf("a new version left the digest as it was")
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1935, 1936: a package whose reads-outside marker lists nothing it can trust is never kept: its digest is one no
|
||||
// kept verdict can have (it differs on every call, though nothing changed), and the answer names it. A marker that lists
|
||||
// what the tests read is kept: a change to a file it names changes the package's digest, a change elsewhere does not.
|
||||
func TestAPackageWithTheReadsOutsideMarkerIsKeptOnlyByWhatItLists(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, tree := digestFixture(t)
|
||||
plain := digests(t, m, "unit")
|
||||
if len(plain.ReadsOutside) != 0 {
|
||||
t.Fatalf("a package carries the marker in the fixture: %v", plain.ReadsOutside)
|
||||
}
|
||||
c, a := "fixture/digest/c", "fixture/digest/a"
|
||||
if err := os.MkdirAll(filepath.Join(tree, "docs"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
edit(t, tree, "docs/one.md", "one\n")
|
||||
edit(t, tree, "other.md", "other\n")
|
||||
for name, marker := range map[string]string{
|
||||
"an empty marker": "# nothing listed\n\n",
|
||||
"a sentence": "the tests read ../docs\n",
|
||||
"a path climbing out": "../docs\n",
|
||||
"a path matching nothing": "docs/none.md\n",
|
||||
"an absolute path": "/etc/passwd\n",
|
||||
"a path that is not clean": "docs/./one.md\n",
|
||||
"a good line and a bad line": "docs\nnope/*.md\n",
|
||||
} {
|
||||
edit(t, tree, "c/"+ReadsOutsideMarker, marker)
|
||||
first, second := digests(t, m, "unit"), digests(t, m, "unit")
|
||||
if len(first.Packages[c]) != 64 || first.Packages[c] == second.Packages[c] || first.Packages[c] == plain.Packages[c] {
|
||||
t.Errorf("%s: c's digest is stable: %s, %s, before %s", name, first.Packages[c], second.Packages[c], plain.Packages[c])
|
||||
}
|
||||
if strings.Join(first.ReadsOutside, ",") != c {
|
||||
t.Errorf("%s: the answer does not name the package that reads outside: %v", name, first.ReadsOutside)
|
||||
}
|
||||
if first.Packages[a] != plain.Packages[a] || first.Packages[a] != second.Packages[a] {
|
||||
t.Errorf("%s: a has no marker and its digest moved: %s, %s, before %s", name, first.Packages[a], second.Packages[a], plain.Packages[a])
|
||||
}
|
||||
}
|
||||
|
||||
// A list: a glob, a directory read whole, and a path whose absence is read.
|
||||
edit(t, tree, "c/"+ReadsOutsideMarker, "# what c's tests read\ndocs/*.md\ndocs\n?gone\n")
|
||||
before := digests(t, m, "unit")
|
||||
if len(before.ReadsOutside) != 0 || len(before.Packages[c]) != 64 || before.Packages[c] == plain.Packages[c] {
|
||||
t.Fatalf("a listed marker is not kept: %v, %s (no marker %s)", before.ReadsOutside, before.Packages[c], plain.Packages[c])
|
||||
}
|
||||
if again := digests(t, m, "unit"); again.Packages[c] != before.Packages[c] {
|
||||
t.Errorf("nothing changed and c's digest moved")
|
||||
}
|
||||
for _, step := range []struct {
|
||||
file, text string
|
||||
moves bool
|
||||
}{
|
||||
{"other.md", "changed\n", false}, // outside the list
|
||||
{"d/d.go", "package d\n\n// D.\n", false}, // a package c's tests do not reach
|
||||
{"docs/one.md", "changed\n", true}, // a listed file
|
||||
{"docs/two.md", "new\n", true}, // a new file the glob matches
|
||||
{"gone", "here\n", true}, // a path whose absence was read
|
||||
{"c/note.txt", "changed\n", true}, // the package's own file that is not Go
|
||||
{"c/x_integration_test.go", "//go:build integration\n\npackage c\n", true}, // a Go file its build tags leave out (go list's IgnoredGoFiles)
|
||||
} {
|
||||
edit(t, tree, step.file, step.text)
|
||||
after := digests(t, m, "unit")
|
||||
if moved := after.Packages[c] != before.Packages[c]; moved != step.moves {
|
||||
t.Errorf("a change to %s: c's digest moved %v, want %v", step.file, moved, step.moves)
|
||||
}
|
||||
if after.Packages[a] != plain.Packages[a] {
|
||||
t.Errorf("a change to %s moved a's digest", step.file)
|
||||
}
|
||||
before = after
|
||||
}
|
||||
|
||||
// Issue 890/AC3: a package whose directory carries the needs marker is named with its needs at the integration level,
|
||||
// comments and blank lines not read, and at the unit level not at all.
|
||||
edit(t, tree, "a/"+NeedsMarker, "# its integration tests copy the installed Node toolchain\n\nnode-build-node\n")
|
||||
if n := digests(t, m, LevelIntegration).Needs; len(n) != 1 || n[0].Package != a || strings.Join(n[0].Needs, ",") != "node-build-node" {
|
||||
t.Errorf("the integration level's needs = %+v, want %s needing node-build-node", n, a)
|
||||
}
|
||||
if n := digests(t, m, LevelUnit).Needs; len(n) != 0 {
|
||||
t.Errorf("the unit level names needs: %+v", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1935: a catalogue module's packages read ../../module.json and the module's files: a change to them changes the
|
||||
// digest of the module's package, and a change to another module's files does not.
|
||||
func TestACatalogueModulesFilesChangeTheDigestOfItsPackagesOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, tree := digestFixture(t)
|
||||
for _, mod := range []string{"mod1", "mod2"} {
|
||||
dir := filepath.Join(tree, mod, "cmd", "p")
|
||||
if err := os.MkdirAll(filepath.Join(tree, mod, "files"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
edit(t, tree, mod+"/module.json", `{"module":"`+mod+`"}`)
|
||||
edit(t, tree, mod+"/README.md", "# "+mod+"\n")
|
||||
edit(t, tree, mod+"/files/f.env", "A=1\n")
|
||||
edit(t, tree, mod+"/cmd/p/p.go", "package p\n")
|
||||
edit(t, tree, mod+"/cmd/p/p_test.go", "package p\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestReads(t *testing.T) {\n\tif _, err := os.ReadFile(\"../../module.json\"); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n")
|
||||
}
|
||||
p1, p2 := "fixture/digest/mod1/cmd/p", "fixture/digest/mod2/cmd/p"
|
||||
before := digests(t, m, "unit")
|
||||
if len(before.Packages[p1]) != 64 || len(before.Packages[p2]) != 64 {
|
||||
t.Fatalf("no digest for the module packages: %+v", before.Packages)
|
||||
}
|
||||
if before.Packages[p1] == before.Packages[p2] {
|
||||
t.Errorf("two modules, one digest")
|
||||
}
|
||||
for _, file := range []string{"mod1/module.json", "mod1/README.md", "mod1/files/f.env"} {
|
||||
edit(t, tree, file, "changed "+file+"\n")
|
||||
after := digests(t, m, "unit")
|
||||
if after.Packages[p1] == before.Packages[p1] {
|
||||
t.Errorf("a change to %s did not change the module's digest", file)
|
||||
}
|
||||
if after.Packages[p2] != before.Packages[p2] {
|
||||
t.Errorf("a change to %s changed the other module's digest", file)
|
||||
}
|
||||
before = after
|
||||
}
|
||||
if got := digests(t, m, "unit"); got.Packages["fixture/digest/c"] == "" {
|
||||
t.Errorf("a package outside any module lost its digest")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A failed test keeps the lines that say why it failed, not the lines it logged first (issue 1901). The fixture is
|
||||
// the real `go test -json` of a test that logs a line and then fails with a multi-line t.Fatalf.
|
||||
func TestAFailedTestKeepsItsFatalMessageAfterItsLogLines(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/captured/go-test-json-log-then-fatal.jsonl")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res := &TestResult{}
|
||||
rd := newReader(res, ".")
|
||||
rd.read(string(raw))
|
||||
rd.finish()
|
||||
if len(res.Failed) != 1 {
|
||||
t.Fatalf("failures = %+v, want one", res.Failed)
|
||||
}
|
||||
f := res.Failed[0]
|
||||
for _, want := range []string{"3 subscription(s) derive to a subject no emitter owns:", "a.b.c", "d.e.f", "g.h.i"} {
|
||||
if !strings.Contains(f.Output, want) {
|
||||
t.Errorf("1901/AC1: output lacks %q: %q", want, f.Output)
|
||||
}
|
||||
}
|
||||
if want := "3 subscription(s) derive to a subject no emitter owns:"; f.Cause != want && !strings.HasSuffix(f.Cause, want) {
|
||||
t.Errorf("1901/AC2: cause = %q, want the t.Fatalf line, not the t.Logf line", f.Cause)
|
||||
}
|
||||
}
|
||||
|
||||
// A failure longer than the bound keeps its last lines, where the cause is, and says how many earlier lines it dropped.
|
||||
func TestAFailureLongerThanTheBoundKeepsItsLastLines(t *testing.T) {
|
||||
var lines []string
|
||||
for i := 0; i < 400; i++ {
|
||||
lines = append(lines, " x_test.go:1: log line "+strings.Repeat("y", 40))
|
||||
}
|
||||
lines = append(lines, " x_test.go:9: the real cause", "--- FAIL: TestX (0.00s)")
|
||||
out := clip(lines)
|
||||
if len(out) > outputBytes+100 {
|
||||
t.Errorf("1901/AC1: kept %d bytes, bound is %d", len(out), outputBytes)
|
||||
}
|
||||
if !strings.HasSuffix(out, "the real cause\n--- FAIL: TestX (0.00s)") || !strings.HasPrefix(out, "... (") {
|
||||
t.Errorf("1901/AC1: out = %q...%q", out[:40], out[len(out)-60:])
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
@@ -41,11 +42,13 @@ type FetchResult struct {
|
||||
// head onto it, the paths the merge changes, and the tree of each module directory (a directory holding a
|
||||
// module.json) in the merge. A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the
|
||||
// files, and the workspace is left at the trunk commit.
|
||||
Base string `json:"base,omitempty"`
|
||||
Merge string `json:"merge,omitempty"`
|
||||
Conflicts []string `json:"conflicts,omitempty"`
|
||||
Changed []string `json:"changed,omitempty"`
|
||||
Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"`
|
||||
Base string `json:"base,omitempty"`
|
||||
Merge string `json:"merge,omitempty"`
|
||||
Conflicts []string `json:"conflicts,omitempty"`
|
||||
Changed []string `json:"changed,omitempty"`
|
||||
// Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge.
|
||||
Programs *[]buildstep.Program `json:"programs,omitempty"`
|
||||
Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"`
|
||||
// Beside are the sibling repositories placed as plain directories under the workspace's beside directory, which
|
||||
// build_go_test hands to the tests as MESH_CHECK_BESIDE (ADR 0484 clause 4).
|
||||
Beside []Sibling `json:"beside,omitempty"`
|
||||
@@ -255,7 +258,7 @@ func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error {
|
||||
func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
|
||||
r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "go.mod", "*/go.mod")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
return nil, fmt.Errorf("git ls-files: %s", forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
dirs := []string{}
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
@@ -288,15 +291,19 @@ func inTestdata(path string) bool {
|
||||
// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory
|
||||
// that vendors, see goEnvFor.
|
||||
func (m *Module) goEnv() []string {
|
||||
return []string{
|
||||
modcache := filepath.Join(m.Cache, "gomod")
|
||||
if m.GoModCache != "" {
|
||||
modcache = m.GoModCache
|
||||
}
|
||||
return append([]string{
|
||||
"GOTOOLCHAIN=local",
|
||||
"GOFLAGS=-mod=mod",
|
||||
"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
|
||||
"GOMODCACHE=" + modcache,
|
||||
"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
|
||||
"GOPRIVATE=git.novox.be",
|
||||
"GIT_TERMINAL_PROMPT=0",
|
||||
"GIT_SSH_COMMAND=ssh -o BatchMode=yes",
|
||||
}
|
||||
}, m.threadEnv()...)
|
||||
}
|
||||
|
||||
// goEnvFor is goEnv for a command run in dir: a directory that vendors (vendor/modules.txt) is read from its vendor
|
||||
@@ -327,10 +334,11 @@ func (m *Module) clock() time.Time {
|
||||
func (m *Module) now() time.Time { return m.clock() }
|
||||
|
||||
// causeLine is the first line of a go command's output that says something other than Go's own progress: `go:
|
||||
// downloading ...` comes first when Go fetches a module, and the cause of a failure on a later line.
|
||||
// downloading ...` comes first when Go fetches a module, and `# <package>` heads a compile error of go build or go vet
|
||||
// (issue 1703: the reason said only the package), and the cause of a failure on a later line.
|
||||
func causeLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "go: downloading ") {
|
||||
if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "go: downloading ") && !strings.HasPrefix(l, "# ") {
|
||||
return l
|
||||
}
|
||||
}
|
||||
|
||||
@@ -302,3 +302,16 @@ func TestWorkAndCacheAreTheDirectoriesTheManifestPlacesOutsideTheHome(t *testing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// issue 1703: a go command's reason is its cause, not Go's progress nor go build's `# <package>` heading.
|
||||
func TestCauseLineSkipsProgressAndThePackageHeading(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"go: downloading x v1\n# git.example/m/cmd\n./main.go:3:2: undefined: y\n": "./main.go:3:2: undefined: y",
|
||||
"\ngo: example.com/x@v1: reading: 404\n": "go: example.com/x@v1: reading: 404",
|
||||
"go: downloading x v1\n": "go: downloading x v1",
|
||||
} {
|
||||
if got := causeLine(in); got != want {
|
||||
t.Errorf("causeLine(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,10 +5,13 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildcpu"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
@@ -28,6 +31,7 @@ func toolchainValues(env []string) []string {
|
||||
// 1572/AC1 (ADR 0477 §1): every environment a step gives its go commands says GOTOOLCHAIN=local, once, and names no
|
||||
// release.
|
||||
func TestEveryStepEnvironmentSaysGotoolchainLocal(t *testing.T) {
|
||||
t.Setenv("MESH_BUILD_CPU_SHARE", "50")
|
||||
m := &Module{Cache: t.TempDir()}
|
||||
vend := t.TempDir()
|
||||
os.MkdirAll(filepath.Join(vend, "vendor"), 0o755)
|
||||
@@ -41,6 +45,24 @@ func TestEveryStepEnvironmentSaysGotoolchainLocal(t *testing.T) {
|
||||
if got := toolchainValues(env); len(got) != 1 || got[0] != "GOTOOLCHAIN=local" {
|
||||
t.Errorf("%s: GOTOOLCHAIN settings %v, want exactly GOTOOLCHAIN=local", name, got)
|
||||
}
|
||||
// Issue 1806: and its threads, its part of the node's processor cap (probeEnv runs no build).
|
||||
if procs := strings.Count(strings.Join(env, "\n"), "GOMAXPROCS="); name != "probeEnv" && procs != 1 {
|
||||
t.Errorf("%s: GOMAXPROCS said %d times, want once: %v", name, procs, env)
|
||||
}
|
||||
}
|
||||
// Without the composer's share no thread limit; with it, the share over its own three slots (ADR 0553 §4).
|
||||
t.Setenv("MESH_BUILD_CPU_SHARE", "")
|
||||
m = &Module{Cache: t.TempDir(), Node: "shanks", Capacity: func() int { return 3 }}
|
||||
if l := m.cpuLimits(); l.Threads != 0 || strings.Contains(strings.Join(m.goEnv(), " "), "GOMAXPROCS=") {
|
||||
t.Errorf("without the share: %+v, goEnv %v", l, m.goEnv())
|
||||
}
|
||||
t.Setenv("MESH_BUILD_CPU_SHARE", "50")
|
||||
want := buildcpu.Threads(50, runtime.NumCPU(), 3)
|
||||
if l := m.cpuLimits(); l.Slots != 3 || !strings.Contains(strings.Join(m.goEnv(), " "), "GOMAXPROCS="+strconv.Itoa(want)) {
|
||||
t.Errorf("limits %+v, goEnv %v; want %d threads", l, m.goEnv(), want)
|
||||
}
|
||||
if st := m.cpuStatus(); st["limits"].(buildcpu.Limits) != m.cpuLimits() {
|
||||
t.Errorf("the status says %v", st)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa
|
||||
return nil, fmt.Errorf("the integration level's bus: %w", err)
|
||||
}
|
||||
defer stopBus()
|
||||
env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...)
|
||||
env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.checkEnv(workspace)...)
|
||||
// Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run
|
||||
// below finds them built. Whether they compile is for the run to say; its failure is not read here.
|
||||
// Only the directories the selected packages lie in compile (ADR 0536 §3); no selection is every directory.
|
||||
|
||||
@@ -40,7 +40,7 @@ func (m *Module) isolation(workspace string, write bool) buildstep.Isolation {
|
||||
Workspace: filepath.Join(m.Work, workspace),
|
||||
Caches: m.caches(),
|
||||
WriteCaches: write,
|
||||
Env: []string{"PATH=" + path, "LC_ALL=C"},
|
||||
Env: append([]string{"PATH=" + path, "LC_ALL=C"}, m.threadEnv()...),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep/conformance"
|
||||
)
|
||||
|
||||
// 1901/AC7: the module's own parse path (readGoTest, then the level's judging) keeps the failure contract on every captured
|
||||
// go test -json case of the SDK's conformance kit, and its result names the module that sent it.
|
||||
func TestTheTestStepKeepsTheFailureContract(t *testing.T) {
|
||||
conformance.Run(t, conformance.GoTest, func(o conformance.Output) []buildstep.Failure {
|
||||
res := &TestResult{Level: LevelUnit, By: buildstep.By("mesh-build-go")}
|
||||
readGoTest(res, ".", o.Stdout)
|
||||
res.judge(Bounds{Run: UnitBound})
|
||||
if res.Passed {
|
||||
t.Errorf("a failed run judged passed: %+v", res)
|
||||
}
|
||||
return res.Failed
|
||||
})
|
||||
}
|
||||
|
||||
// 1890/AC7: the module's own start sweep (endLeftRunning, as main runs it) keeps the conformance kit's left-running
|
||||
// case: a step a previous process left running is ended at once and said, whatever its age, and a step of this process
|
||||
// is kept.
|
||||
func TestTheStartSweepEndsWhatAPreviousProcessLeft(t *testing.T) {
|
||||
conformance.LeftRunning(t, func(ctx context.Context, scan buildstep.Scan, node string, before time.Time, logf func(string, ...any)) {
|
||||
(&Module{Node: node}).endLeftRunning(ctx, scan, before, logf)
|
||||
})
|
||||
}
|
||||
@@ -11,11 +11,13 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// levelArg is the first argument that makes this binary a step's shut-off process: the integration level's, and, when
|
||||
@@ -140,6 +142,49 @@ func (m *Module) nodeToolchains() []string {
|
||||
return []string{dir}
|
||||
}
|
||||
|
||||
// nodeToolchainMark is the file the Node build module writes in a toolchain directory once it is installed whole
|
||||
// (mesh-build-node toolchain.go installedMark).
|
||||
const nodeToolchainMark = ".installed"
|
||||
|
||||
// Has says whether this node has a need a check ask or a package's tests name (issue 890, ADR 0552; mesh-sdk
|
||||
// delivery.AskNeeds), read from the node itself at every ask: node-build-node when the Node build module's toolchains
|
||||
// directory, given to the integration level as nodeToolchains gives it, holds a toolchain installed whole (its mark and
|
||||
// its package-lock.json readable by this account), and node, npm and bwrap are installed: what mesh-build-node's
|
||||
// integration tests (realMachine) look for before they run. Any other need it lacks.
|
||||
func (m *Module) Has(need string) bool {
|
||||
if m.HasNeed != nil {
|
||||
return m.HasNeed(need)
|
||||
}
|
||||
switch need {
|
||||
case delivery.NeedNodeBuildNode:
|
||||
dirs := m.nodeToolchains()
|
||||
if len(dirs) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, p := range []string{"node", "npm", "bwrap"} {
|
||||
if _, err := exec.LookPath(p); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
entries, err := os.ReadDir(dirs[0])
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, e := range entries {
|
||||
dir := filepath.Join(dirs[0], e.Name())
|
||||
if _, err := os.Stat(filepath.Join(dir, nodeToolchainMark)); err != nil {
|
||||
continue
|
||||
}
|
||||
if f, err := os.Open(filepath.Join(dir, "package-lock.json")); err == nil {
|
||||
f.Close()
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func lastLine(s string) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
return strings.TrimSpace(lines[len(lines)-1])
|
||||
@@ -183,7 +228,7 @@ func (m *Module) answer(ctx context.Context, spec levelSpec) levelAnswer {
|
||||
return levelAnswer{Digest: &d}
|
||||
case stepUnit:
|
||||
res, err := m.runLevel(ctx, tree, levelOpts{workspace: spec.Workspace, level: LevelUnit, packages: spec.Packages, run: spec.Run,
|
||||
bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(spec.Workspace)})
|
||||
bounds: Bounds{Run: m.unitBound()}, env: m.checkEnv(spec.Workspace)})
|
||||
if err != nil {
|
||||
return levelAnswer{Error: err.Error()}
|
||||
}
|
||||
|
||||
@@ -6,13 +6,12 @@ package main
|
||||
// step that could not run.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// The two levels (ADR 0445 §2).
|
||||
@@ -31,33 +30,14 @@ type Bounds struct {
|
||||
Setup, Run time.Duration
|
||||
}
|
||||
|
||||
// PackageResult is one package's part of a level.
|
||||
type PackageResult struct {
|
||||
Dir string `json:"dir"`
|
||||
Package string `json:"package"`
|
||||
Passed bool `json:"passed"`
|
||||
Elapsed float64 `json:"elapsed"`
|
||||
Tests int `json:"tests"`
|
||||
// NoTests is a package with no test files (at this level: a file with the other level's build tag is none).
|
||||
NoTests bool `json:"no_tests,omitempty"`
|
||||
}
|
||||
|
||||
// TestTime is one test and the time it took.
|
||||
type TestTime struct {
|
||||
Package string `json:"package"`
|
||||
Test string `json:"test"`
|
||||
Elapsed float64 `json:"elapsed"`
|
||||
}
|
||||
|
||||
// Failure is a test that failed, or a package that failed with no test to name (a build that failed, a TestMain that
|
||||
// panicked); Output is the last lines it wrote, up to outputBytes (issue 1901), and Cause the first line of what the
|
||||
// test reported as its error (t.Error, t.Fatal), without its file and line, when go test said which lines those were.
|
||||
type Failure struct {
|
||||
Package string `json:"package"`
|
||||
Test string `json:"test,omitempty"`
|
||||
Output string `json:"output,omitempty"`
|
||||
Cause string `json:"cause,omitempty"`
|
||||
}
|
||||
// PackageResult, TestTime and Failure are the SDK's failure contract (buildstep, issue 1901), with the JSON names this
|
||||
// module always sent: a Failure names its test, its file, its cause (never empty, never a passing line) and its own
|
||||
// output up to 4096 bytes, cause first.
|
||||
type (
|
||||
PackageResult = buildstep.GoPackage
|
||||
TestTime = buildstep.TestTime
|
||||
Failure = buildstep.Failure
|
||||
)
|
||||
|
||||
// TestResult is what a test step passes with.
|
||||
type TestResult struct {
|
||||
@@ -91,179 +71,23 @@ type TestResult struct {
|
||||
// JudgedBy is the program that ran the level (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
|
||||
// runner workspace.
|
||||
JudgedBy string `json:"judged_by,omitempty"`
|
||||
// By is the build module and the SDK it was built with (buildstep.By): the checker names it when a failure hides
|
||||
// its cause.
|
||||
By string `json:"by,omitempty"`
|
||||
}
|
||||
|
||||
// event is one line of `go test -json` (test2json, and the build events go test adds).
|
||||
type event struct {
|
||||
Action string
|
||||
Package string
|
||||
Test string
|
||||
Output string
|
||||
OutputType string // "error" for a t.Error or t.Fatal message, "error-continue" for its further lines (go test -json)
|
||||
ImportPath string
|
||||
FailedBuild string
|
||||
Elapsed float64
|
||||
}
|
||||
|
||||
// outputBytes is the most a failure keeps of its output, and outputLineBytes the most of one line. **A failure keeps its
|
||||
// last lines** (issue 1901): a test logs before it fails, so the lines that say why are at the end, and the first
|
||||
// fifteen lines kept before lost them (the verdict of group r4g kept a t.Logf line, not the t.Fatalf after it). Four
|
||||
// kilobytes holds a long assertion message and stays far under a bus message's or a verdict's bound.
|
||||
const (
|
||||
outputBytes = 4096
|
||||
outputLineBytes = 1024
|
||||
)
|
||||
|
||||
// reader collects the events of one go.mod directory's `go test -json` into a result.
|
||||
type reader struct {
|
||||
res *TestResult
|
||||
dir string
|
||||
pkgs map[string]*PackageResult
|
||||
order []string
|
||||
builds map[string][]string // build output by import path
|
||||
pkgOut map[string][]string
|
||||
testOut map[string][]string // by package + "\x00" + test
|
||||
causes map[string]string // the first error line of a test, by the same key
|
||||
times []TestTime
|
||||
failedIn map[string]bool // packages with a failed test
|
||||
}
|
||||
|
||||
func newReader(res *TestResult, dir string) *reader {
|
||||
return &reader{res: res, dir: dir, pkgs: map[string]*PackageResult{}, builds: map[string][]string{},
|
||||
pkgOut: map[string][]string{}, testOut: map[string][]string{}, causes: map[string]string{}, failedIn: map[string]bool{}}
|
||||
}
|
||||
|
||||
// read takes go test's output. Lines that are not events are kept for a failure that has nothing else to say.
|
||||
func (r *reader) read(out string) (noise []string) {
|
||||
sc := bufio.NewScanner(strings.NewReader(out))
|
||||
sc.Buffer(make([]byte, 1<<20), 1<<26)
|
||||
for sc.Scan() {
|
||||
var ev event
|
||||
if err := json.Unmarshal(sc.Bytes(), &ev); err != nil {
|
||||
if l := strings.TrimSpace(sc.Text()); l != "" {
|
||||
noise = append(noise, l)
|
||||
}
|
||||
continue
|
||||
}
|
||||
r.event(ev)
|
||||
}
|
||||
return noise
|
||||
}
|
||||
|
||||
func (r *reader) pkg(name string) *PackageResult {
|
||||
p := r.pkgs[name]
|
||||
if p == nil {
|
||||
p = &PackageResult{Dir: r.dir, Package: name}
|
||||
r.pkgs[name] = p
|
||||
r.order = append(r.order, name)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func keep(lines []string, out string) []string {
|
||||
if t := strings.TrimRight(out, "\r\n"); t != "" && !strings.HasPrefix(t, "=== ") {
|
||||
lines = append(lines, t)
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func (r *reader) event(ev event) {
|
||||
switch {
|
||||
case ev.Package == "" && ev.ImportPath != "":
|
||||
if ev.Action == "build-output" {
|
||||
r.builds[ev.ImportPath] = keep(r.builds[ev.ImportPath], ev.Output)
|
||||
}
|
||||
case ev.Package == "":
|
||||
case ev.Test == "":
|
||||
p := r.pkg(ev.Package)
|
||||
switch ev.Action {
|
||||
case "output":
|
||||
r.pkgOut[ev.Package] = keep(r.pkgOut[ev.Package], ev.Output)
|
||||
case "pass":
|
||||
p.Passed, p.Elapsed = true, ev.Elapsed
|
||||
case "fail":
|
||||
p.Passed, p.Elapsed = false, ev.Elapsed
|
||||
if ev.FailedBuild != "" {
|
||||
r.pkgOut[ev.Package] = append(r.builds[ev.FailedBuild], r.pkgOut[ev.Package]...)
|
||||
}
|
||||
case "skip":
|
||||
p.Passed, p.NoTests = true, true
|
||||
}
|
||||
default:
|
||||
key := ev.Package + "\x00" + ev.Test
|
||||
switch ev.Action {
|
||||
case "output":
|
||||
r.testOut[key] = keep(r.testOut[key], ev.Output)
|
||||
if ev.OutputType == "error" && r.causes[key] == "" {
|
||||
r.causes[key] = errorWords(ev.Output)
|
||||
}
|
||||
case "pass", "fail", "skip":
|
||||
p := r.pkg(ev.Package)
|
||||
p.Tests++
|
||||
r.res.Tests++
|
||||
if ev.Action == "skip" {
|
||||
r.res.Skipped++
|
||||
}
|
||||
r.times = append(r.times, TestTime{Package: ev.Package, Test: ev.Test, Elapsed: ev.Elapsed})
|
||||
if ev.Action == "fail" {
|
||||
r.failedIn[ev.Package] = true
|
||||
r.res.Failed = append(r.res.Failed, Failure{Package: ev.Package, Test: ev.Test, Output: clip(r.testOut[key]), Cause: r.causes[key]})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// clip joins the last lines of a failure's output that fit outputBytes, each cut to outputLineBytes, and says how many
|
||||
// earlier lines it left out.
|
||||
func clip(lines []string) string {
|
||||
size, from := 0, len(lines)
|
||||
for from > 0 {
|
||||
n := len(cutLine(lines[from-1])) + 1
|
||||
if size+n > outputBytes && from < len(lines) {
|
||||
break
|
||||
}
|
||||
size += n
|
||||
from--
|
||||
}
|
||||
kept := make([]string, 0, len(lines)-from+1)
|
||||
if from > 0 {
|
||||
kept = append(kept, fmt.Sprintf("... (%d earlier lines)", from))
|
||||
}
|
||||
for _, l := range lines[from:] {
|
||||
kept = append(kept, cutLine(l))
|
||||
}
|
||||
return strings.Join(kept, "\n")
|
||||
}
|
||||
|
||||
// cutLine cuts a line to outputLineBytes, on a character boundary.
|
||||
func cutLine(l string) string {
|
||||
if len(l) <= outputLineBytes {
|
||||
return l
|
||||
}
|
||||
return strings.ToValidUTF8(l[:outputLineBytes], "") + "..."
|
||||
}
|
||||
|
||||
var fileAndLine = regexp.MustCompile(`^\S+\.go:\d+: `)
|
||||
|
||||
// errorWords is the words of an error line of a test: trimmed, without the file and line go test put before them.
|
||||
func errorWords(out string) string {
|
||||
return strings.TrimSpace(fileAndLine.ReplaceAllString(strings.TrimSpace(out), ""))
|
||||
}
|
||||
|
||||
// finish moves what was read into the result: the packages in order, a failure for each failed package that has no
|
||||
// failed test to name it, and the tests' times.
|
||||
func (r *reader) finish() {
|
||||
for _, name := range r.order {
|
||||
p := r.pkgs[name]
|
||||
if !p.Passed && !r.failedIn[name] {
|
||||
r.res.Failed = append(r.res.Failed, Failure{Package: name, Output: clip(r.pkgOut[name])})
|
||||
}
|
||||
if !p.NoTests && p.Elapsed > r.res.Elapsed {
|
||||
r.res.Elapsed = p.Elapsed
|
||||
}
|
||||
r.res.Packages = append(r.res.Packages, *p)
|
||||
}
|
||||
r.res.Slowest = append(r.res.Slowest, r.times...)
|
||||
// readGoTest reads one go.mod directory's `go test -json` into res through the SDK's reader (issue 1901: one failure
|
||||
// contract for every build module, buildstep.Failure), and answers the lines that were no event and how many packages
|
||||
// it named.
|
||||
func readGoTest(res *TestResult, dir, out string) (noise []string, packages int) {
|
||||
read := buildstep.ReadGoTest(out, dir)
|
||||
res.Tests += read.Tests
|
||||
res.Skipped += read.Skipped
|
||||
res.Failed = append(res.Failed, read.Failed...)
|
||||
res.Packages = append(res.Packages, read.Packages...)
|
||||
res.Slowest = append(res.Slowest, read.Times...)
|
||||
res.Elapsed = max(res.Elapsed, read.Elapsed)
|
||||
return read.Noise, len(read.Packages)
|
||||
}
|
||||
|
||||
// judge sets Passed and Why from what was read and the bounds. A level with no failure, over no bound, passes; with no
|
||||
|
||||
@@ -12,7 +12,9 @@ import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
@@ -102,8 +104,7 @@ func (m *Module) lint(ctx context.Context, run forgekey.Runner, tree, d string,
|
||||
}
|
||||
var failed []Failure
|
||||
if len(files) > 0 {
|
||||
failed = append(failed, Failure{Package: d, Test: "gofmt",
|
||||
Output: "not as gofmt writes them (run gofmt -w): " + strings.Join(files, ", ")})
|
||||
failed = append(failed, buildstep.ConsoleFailure(d, "gofmt", "not as gofmt writes them (run gofmt -w): "+strings.Join(files, ", ")))
|
||||
}
|
||||
sets := [][]string{nil}
|
||||
if len(tags) > 0 {
|
||||
@@ -122,8 +123,53 @@ func (m *Module) lint(ctx context.Context, run forgekey.Runner, tree, d string,
|
||||
return nil, fmt.Errorf("%s in %s could not be run: %s", name, d, firstLine(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
if r.Status != 0 {
|
||||
failed = append(failed, Failure{Package: d, Test: name, Output: strings.TrimSpace(r.Stderr + "\n" + r.Stdout)})
|
||||
failed = append(failed, buildstep.ConsoleFailure(d, name, r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
}
|
||||
return failed, nil
|
||||
}
|
||||
|
||||
// lintAll lints each directory, up to m.parallel() at once like the level's tests; the failures join in directory order
|
||||
// and the error is the first directory's, so the answer does not depend on which ended first (issue 1930).
|
||||
func (m *Module) lintAll(ctx context.Context, run forgekey.Runner, tree string, dirs []string, env func(d string) []string) ([]Failure, error) {
|
||||
// The ask's phases say it (issue 1890): lint of the directories, then their tests.
|
||||
buildstep.Phase(ctx, fmt.Sprintf("unit: lint of %d directories", len(dirs)))
|
||||
defer buildstep.Phase(ctx, fmt.Sprintf("unit: tests of %d directories", len(dirs)))
|
||||
type answer struct {
|
||||
failed []Failure
|
||||
err error
|
||||
}
|
||||
answers := make([]answer, len(dirs))
|
||||
slots := make(chan struct{}, m.parallel())
|
||||
var wg sync.WaitGroup
|
||||
var crash any // a panic of a lint goes on in the caller, as a panic of a test run does
|
||||
var crashMu sync.Mutex
|
||||
for i, d := range dirs {
|
||||
wg.Add(1)
|
||||
slots <- struct{}{}
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-slots }()
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
crashMu.Lock()
|
||||
crash = p
|
||||
crashMu.Unlock()
|
||||
}
|
||||
}()
|
||||
answers[i].failed, answers[i].err = m.lint(ctx, run, tree, d, env(d))
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if crash != nil {
|
||||
panic(crash)
|
||||
}
|
||||
var failed []Failure
|
||||
for _, a := range answers {
|
||||
if a.err != nil {
|
||||
return nil, a.err
|
||||
}
|
||||
failed = append(failed, a.failed...)
|
||||
}
|
||||
return failed, nil
|
||||
}
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
@@ -90,3 +92,43 @@ func TestAnUnformattedFileFailsTheUnitLevel(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1930: the lint of many directories runs side by side up to the level's bound, not one after another, and the
|
||||
// failures still join in directory order.
|
||||
func TestTheLintOfManyDirectoriesRunsAtOnceUpToTheBound(t *testing.T) {
|
||||
files := map[string]string{}
|
||||
var dirs []string
|
||||
for _, d := range []string{"a", "b", "c", "d", "e", "f"} {
|
||||
files[d+"/go.mod"] = "module x/" + d + "\n"
|
||||
files[d+"/x.go"] = "package x\n"
|
||||
dirs = append(dirs, d)
|
||||
}
|
||||
tree := writeTree(t, files)
|
||||
var mu sync.Mutex
|
||||
running, most := 0, 0
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
mu.Lock()
|
||||
running++
|
||||
most = max(most, running)
|
||||
mu.Unlock()
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
mu.Lock()
|
||||
running--
|
||||
mu.Unlock()
|
||||
if strings.HasSuffix(args[1], "b") || strings.HasSuffix(args[1], "e") {
|
||||
return forgekey.Ran{Status: 1, Stderr: "bad " + filepath.Base(args[1])}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
m := &Module{Parallel: 3}
|
||||
failed, err := m.lintAll(context.Background(), run, tree, dirs, func(string) []string { return nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if most < 2 || most > 3 {
|
||||
t.Errorf("%d lint commands ran at once, want 2 or 3 (the bound)", most)
|
||||
}
|
||||
if len(failed) != 2 || failed[0].Package != "b" || failed[1].Package != "e" {
|
||||
t.Errorf("failed = %+v, want b's then e's", failed)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"os"
|
||||
"os/user"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -40,6 +41,8 @@ func main() {
|
||||
Emit: func(t string, b any) error { return stdio.Emit(t, b) },
|
||||
Steps: stdio.State(delivery.StepsState).Put,
|
||||
ReadSteps: readOf(stdio.State(delivery.StepsState)),
|
||||
Pauses: stdio.State(delivery.PausesState).Put,
|
||||
ReadPauses: readOf(stdio.State(delivery.PausesState)),
|
||||
ReadToolchainsState: readOf(stdio.State(delivery.ToolchainsState)),
|
||||
Ctx: ctx,
|
||||
Say: logf,
|
||||
@@ -54,6 +57,7 @@ func main() {
|
||||
m.RegistryFile = m.Home + "/registry"
|
||||
m.Settings = m.Home + "/settings.json"
|
||||
m.ReadKept = readOf(stdio.State(KeptState)).Get
|
||||
m.ReadManifest = readOf(stdio.State(ManifestsState)).Get
|
||||
m.Probe = ProbeGo(m.Home, m.Cache)
|
||||
m.Bubblewrap = ProbeBubblewrap
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
@@ -67,7 +71,14 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
steps := m.StepTools()
|
||||
booted := time.Now()
|
||||
puller := m.Puller(steps, nil)
|
||||
m.Held = puller
|
||||
if m.Node != "" {
|
||||
// A step a previous process left running is ended (issue 1890): one that names another owner, or none and
|
||||
// started before booted.
|
||||
go m.endLeftRunning(ctx, m.stepScan(), booted, logf)
|
||||
}
|
||||
if m.Node != "" {
|
||||
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
|
||||
go func() { _ = puller.Pull(ctx) }()
|
||||
|
||||
@@ -92,9 +92,13 @@ func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *tes
|
||||
states := []string{}
|
||||
for _, s := range m.State {
|
||||
states = append(states, s.Name)
|
||||
// A step's key is per step and dropped after its age, so the bucket cannot fill; a pause has no age.
|
||||
if (s.Name == "steps") != (s.TTLSeconds > 0) {
|
||||
t.Errorf("state %s has ttl-seconds %d: only steps has an age", s.Name, s.TTLSeconds)
|
||||
}
|
||||
}
|
||||
slices.Sort(states)
|
||||
if want := []string{"keys", "steps", "toolchains"}; !slices.Equal(states, want) {
|
||||
if want := []string{"keys", "pauses", "steps", "toolchains"}; !slices.Equal(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != Seat {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"syscall"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// mirrorLocks hold one mutex per mirror of this process; the lock file beside the mirror holds off another process.
|
||||
@@ -67,7 +68,7 @@ func (m *Module) dissociate(ctx context.Context, dir string) error {
|
||||
return nil // no mirror was used
|
||||
}
|
||||
if r := m.Run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-C", dir, "repack", "-a", "-d", "-q"); r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("git repack in %s: %s", dir, firstLine(r.Err+" "+r.Stderr))
|
||||
return fmt.Errorf("git repack in %s: %s", dir, forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
return os.Remove(alternates)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -121,20 +122,35 @@ func TestTwoUpdatesOfOneMirrorNeverRunAtOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A mirror that cannot be updated does not fail the fetch: the fetch goes on without it.
|
||||
// A mirror that cannot be updated does not fail the fetch: the fetch goes on without it, and the line saying so carries
|
||||
// git's own words. The forge's refusal of a repository its account cannot read came out as "git -C: error:" (ace,
|
||||
// 2026-10-26: git writes a bare "error:" line before the forge's message).
|
||||
func TestAMirrorThatCannotBeUpdatedLetsTheFetchGoOnWithoutIt(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n"}
|
||||
f.rule = func(line string) (r runAnswer, ok bool) {
|
||||
if strings.Contains(line, "mirror") && strings.Contains(line, " fetch ") {
|
||||
return runAnswer{Status: 1, Stderr: "fatal: no space left\n"}, true
|
||||
return runAnswer{Status: 128, Stderr: "error: \nerror: User: 22:nox-builder with Key: 30:mesh:ace.mesh-build-go is not authorized to read jschoubben/car-hunter.\nerror: \nfatal: Could not read from remote repository.\n"}, true
|
||||
}
|
||||
return r, false
|
||||
}
|
||||
m, states := fetchModule(t, f)
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
m.Say = func(format string, a ...any) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
said = append(said, fmt.Sprintf(format, a...))
|
||||
}
|
||||
ws := fetchOnce(t, m, states, fetchArgs).Workspace
|
||||
if _, err := os.Stat(filepath.Join(m.Work, ws, "tree", ".git", "objects", "info", "alternates")); err == nil {
|
||||
t.Errorf("the workspace borrows from a mirror that is not there")
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
all := strings.Join(said, "\n")
|
||||
if !strings.Contains(all, "is not used: git fetch: error: User: 22:nox-builder with Key: 30:mesh:ace.mesh-build-go is not authorized to read jschoubben/car-hunter.; fatal: Could not read from remote repository.") {
|
||||
t.Errorf("the mirror's line lost git's words:\n%s", all)
|
||||
}
|
||||
}
|
||||
|
||||
func sleepABit() { time.Sleep(30 * time.Millisecond) }
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user