diff --git a/.gitignore b/.gitignore index a95e38a3..9552fa38 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,8 @@ modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image /checks/paths/paths /modules/mesh-build-go/mesh-build-go /modules/mesh-delivery-gate/mesh-delivery-gate +modules/mesh-config/cmd/mesh-config/mesh-config +checks/placement/placement # Built by their modules' build step (issue 1577), never committed. modules/forticlient/cmd/forticlient-tools/forticlient-tools diff --git a/checks/agenthome/reads-outside b/checks/agenthome/reads-outside index f6b2d871..8fc7689d 100644 --- a/checks/agenthome/reads-outside +++ b/checks/agenthome/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest. +modules/*/module.json diff --git a/checks/config/reads-outside b/checks/config/reads-outside index f6b2d871..cc3c0a39 100644 --- a/checks/config/reads-outside +++ b/checks/config/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest and the files it names. +modules diff --git a/checks/failurekit/reads-outside b/checks/failurekit/reads-outside index f6b2d871..f5fef2fa 100644 --- a/checks/failurekit/reads-outside +++ b/checks/failurekit/reads-outside @@ -1 +1,7 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Judge("../.."): every manifest, and the directory of each module claiming a node-build-* seat, walked for a test that +# calls the kit. The build modules are the mesh-build-* directories; a build module named otherwise needs its line here. +modules/*/module.json +modules/mesh-build-* diff --git a/checks/health/reads-outside b/checks/health/reads-outside index f6b2d871..f6ce1405 100644 --- a/checks/health/reads-outside +++ b/checks/health/reads-outside @@ -1 +1,10 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# every manifest through mesh-sdk's ReadTreeManifest: the root's predefined keys and each templated module's +# templates (files/ by default, or the directory its manifest names); and the root's kept count of resources that do not say how they are ready. +modules/*/module.json +predefined.json +modules/*/files +modules/*/templates +health-undeclared diff --git a/checks/imagebuild/reads-outside b/checks/imagebuild/reads-outside index f6b2d871..8fc7689d 100644 --- a/checks/imagebuild/reads-outside +++ b/checks/imagebuild/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest. +modules/*/module.json diff --git a/checks/imageplace/reads-outside b/checks/imageplace/reads-outside index f6b2d871..8a080f79 100644 --- a/checks/imageplace/reads-outside +++ b/checks/imageplace/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest, and whether the path each image artifact builds from is there. +modules diff --git a/checks/locks/reads-outside b/checks/locks/reads-outside index f6b2d871..4f2c2396 100644 --- a/checks/locks/reads-outside +++ b/checks/locks/reads-outside @@ -1 +1,7 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest, a TypeScript module's package.json and whether its lock is there. +modules/*/module.json +modules/*/package.json +modules/*/package-lock.json diff --git a/checks/mailstore/reads-outside b/checks/mailstore/reads-outside index f6b2d871..8fc7689d 100644 --- a/checks/mailstore/reads-outside +++ b/checks/mailstore/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest. +modules/*/module.json diff --git a/checks/nobinaries/reads-outside b/checks/nobinaries/reads-outside index f6b2d871..8c970936 100644 --- a/checks/nobinaries/reads-outside +++ b/checks/nobinaries/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# every file under modules/ (git ls-files and a walk). +modules diff --git a/checks/noscripts/reads-outside b/checks/noscripts/reads-outside index f6b2d871..a161a850 100644 --- a/checks/noscripts/reads-outside +++ b/checks/noscripts/reads-outside @@ -1 +1,7 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# that none of the old check scripts stands at the root. +?merge-check.sh +?merge-check-typescript.sh +?check-nested-modules.sh diff --git a/checks/owned/reads-outside b/checks/owned/reads-outside index f6b2d871..bc374f72 100644 --- a/checks/owned/reads-outside +++ b/checks/owned/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest; its lists are this package's own files. +modules/*/module.json diff --git a/checks/particulars/reads-outside b/checks/particulars/reads-outside index f6b2d871..c2a9b667 100644 --- a/checks/particulars/reads-outside +++ b/checks/particulars/reads-outside @@ -1 +1,6 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# check("../.."): the root's list and every file of the modules it names. +particulars +modules diff --git a/checks/paths/reads-outside b/checks/paths/reads-outside index f6b2d871..cfaaf0f8 100644 --- a/checks/paths/reads-outside +++ b/checks/paths/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest and every file of a module. +modules diff --git a/checks/placement/go.mod b/checks/placement/go.mod new file mode 100644 index 00000000..6085d760 --- /dev/null +++ b/checks/placement/go.mod @@ -0,0 +1,5 @@ +module checks/placement + +go 1.26.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 diff --git a/checks/placement/go.sum b/checks/placement/go.sum new file mode 100644 index 00000000..05a0bf8b --- /dev/null +++ b/checks/placement/go.sum @@ -0,0 +1,14 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/checks/placement/main.go b/checks/placement/main.go new file mode 100644 index 00000000..659edcc6 --- /dev/null +++ b/checks/placement/main.go @@ -0,0 +1,80 @@ +// Command placement is the catalogue's check of its placement files (novox/hq ADR 0549 §1, ADR 0555; issue 1126): every +// `placement/.json` reads as a placement of the node its name says, and the files together are refused, every +// finding named at once, for two modules of one node on one host port, a pin, binding or holding naming a provider not +// placed where it says, a seat held once for the mesh held twice, two nodes with one address on the private network, or +// more than one hub (mesh-sdk delivery.CheckPlacements). +// +// A module the catalogue does not hold and a node not enrolled are the composition check's to refuse: placement names +// modules of every repository the indexer reads and the nodes the mesh enrolled, neither of which this tree holds. +// +// go run . +// +// Exits 1 on a finding. +package main + +import ( + "fmt" + "os" + "path/filepath" + "sort" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Check reads every placement file under root and answers each finding; none is a catalogue whose placement holds. A +// catalogue with no placement directory has none to check. +func Check(root string) ([]string, error) { + dir := filepath.Join(root, delivery.PlacementDir) + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + var found []string + placements := map[string]delivery.Placement{} + for _, e := range entries { + rel := delivery.PlacementDir + "/" + e.Name() + node, ok := delivery.PlacementFileNode(rel) + if e.IsDir() || !ok { + found = append(found, fmt.Sprintf("%s is not a placement file: the directory holds only .json, one per node", rel)) + continue + } + raw, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + return nil, err + } + p, err := delivery.DecodePlacement(raw) + if err != nil { + found = append(found, fmt.Sprintf("%s does not read as a placement: %v", rel, err)) + continue + } + if p.Revision != "" { + found = append(found, fmt.Sprintf("%s names a revision, which only the indexer writes into the key", rel)) + } + placements[node] = p + } + all := func(string) bool { return true } + found = append(found, delivery.CheckPlacements(placements, all, all)...) + sort.Strings(found) + return found, nil +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: placement ") + os.Exit(2) + } + found, err := Check(os.Args[1]) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + for _, f := range found { + fmt.Println(f) + } + if len(found) > 0 { + os.Exit(1) + } +} diff --git a/checks/placement/main_test.go b/checks/placement/main_test.go new file mode 100644 index 00000000..88e45aca --- /dev/null +++ b/checks/placement/main_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The catalogue's placement files as the merge check reads them (ADR 0549 §1, ADR 0555): the real tree holds every +// rule, and a tree that breaks them is refused with every finding named at once. +func TestPlacementFiles(t *testing.T) { + found, err := Check("../..") + if err != nil { + t.Fatal(err) + } + for _, f := range found { + t.Errorf("%s (novox/hq ADR 0549 §1)", f) + } + + root := t.TempDir() + write := func(name, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(root, "placement"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "placement", name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + if found, err := Check(root); err != nil || len(found) != 0 { + t.Fatalf("a catalogue with no placement directory: %v, %v", found, err) + } + write("ace.json", `{"v":1,"node":"ace","decided":{"hub":true,"address":"10.77.0.1"},"modules":[{"module":"a","ports":{"80":8080}},{"module":"b","ports":{"81":8080}}]}`) + write("g14.json", `{"v":1,"node":"g14","decided":{"hub":true,"address":"10.77.0.1"},"modules":[{"module":"c","bindings":{"pg":{"node":"ace","module":"pg"}}}]}`) + write("novox.json", `{"v":1,"node":"shanks","modules":[]}`) + write("bad.json", `{"v":1,"node":"bad","modules":{}}`) + write("dated.json", `{"v":1,"node":"dated","revision":"abc","modules":[]}`) + write("notes.md", "x") + found, err = Check(root) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"a and b are both published on host port 8080", "bound to ace/pg", "both have the address 10.77.0.1", + "ace, g14 are each placed as it", "placement file of novox names the node", "bad.json does not read", "dated.json names a revision", + "notes.md is not a placement file"} { + hit := false + for _, f := range found { + hit = hit || strings.Contains(f, want) + } + if !hit { + t.Errorf("no finding says %q; found %v", want, found) + } + } +} diff --git a/checks/deps/reads-outside b/checks/placement/reads-outside similarity index 100% rename from checks/deps/reads-outside rename to checks/placement/reads-outside diff --git a/checks/predefined/reads-outside b/checks/predefined/reads-outside index f6b2d871..81a14658 100644 --- a/checks/predefined/reads-outside +++ b/checks/predefined/reads-outside @@ -1 +1,6 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the root's list of predefined keys and every manifest that names one. +predefined.json +modules/*/module.json diff --git a/checks/secrets/reads-outside b/checks/secrets/reads-outside index f6b2d871..cc3c0a39 100644 --- a/checks/secrets/reads-outside +++ b/checks/secrets/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest and the files it names. +modules diff --git a/checks/sshdir/reads-outside b/checks/sshdir/reads-outside index f6b2d871..8fc7689d 100644 --- a/checks/sshdir/reads-outside +++ b/checks/sshdir/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest. +modules/*/module.json diff --git a/checks/templates/reads-outside b/checks/templates/reads-outside index f6b2d871..e854eebe 100644 --- a/checks/templates/reads-outside +++ b/checks/templates/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest and its templates. +modules diff --git a/checks/toolchains/reads-outside b/checks/toolchains/reads-outside index f6b2d871..8fc7689d 100644 --- a/checks/toolchains/reads-outside +++ b/checks/toolchains/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# Check("../.."): every manifest. +modules/*/module.json diff --git a/checks/trunk/reads-outside b/checks/trunk/reads-outside deleted file mode 100644 index f6b2d871..00000000 --- a/checks/trunk/reads-outside +++ /dev/null @@ -1 +0,0 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). diff --git a/checks/words/reads-outside b/checks/words/reads-outside index f6b2d871..035d1d78 100644 --- a/checks/words/reads-outside +++ b/checks/words/reads-outside @@ -1 +1,7 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# run("../.."): the root's lists of retired words and lines, and every module's texts. +retired-words +retired-lines +modules diff --git a/modules/adwaita/internal/desktop/reads-outside b/modules/adwaita/internal/desktop/reads-outside index f6b2d871..7ef55c53 100644 --- a/modules/adwaita/internal/desktop/reads-outside +++ b/modules/adwaita/internal/desktop/reads-outside @@ -1 +1,9 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of this package in every module that carries one. +modules/xorg/internal/desktop/*.go +modules/lemurs/internal/desktop/*.go +modules/i3/internal/desktop/*.go +modules/xterm/internal/desktop/*.go +modules/adwaita/internal/desktop/*.go diff --git a/modules/blueman/cmd/blueman-tools/reads-outside b/modules/blueman/cmd/blueman-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/blueman/cmd/blueman-tools/reads-outside +++ b/modules/blueman/cmd/blueman-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/brother/cmd/brother/reads-outside b/modules/brother/cmd/brother/reads-outside index f6b2d871..95611a5c 100644 --- a/modules/brother/cmd/brother/reads-outside +++ b/modules/brother/cmd/brother/reads-outside @@ -1 +1,6 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# mesh-mail's store.json is the same as brother's; no module scan is left. +modules/mesh-mail/files/store.json +?modules/scan diff --git a/modules/forticlient/cmd/forticlient-tools/reads-outside b/modules/forticlient/cmd/forticlient-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/forticlient/cmd/forticlient-tools/reads-outside +++ b/modules/forticlient/cmd/forticlient-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index a36e113f..6628bfd7 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -174,8 +174,27 @@ function ignoreSecretSetting(cfg: Record, key: string): void { console.error(`the setting ${key} is ignored: it comes from the vault`); } +/** How long the forge may take to answer one request before it is cut and said (novox/hq issue 1577: a request for a 5 MB + * file never ended, and every read through the module timed out behind it). */ +export const FORGE_REQUEST_TIMEOUT_MS = 20_000; + +/** The most a tool's answer carries of one file: the bus's message bound is 1 MiB and a file travels as text (issue 1577). */ +export const MAX_FILE_BYTES = 512 * 1024; + +/** One request to the forge, cut at the limit and failing loud naming the request. */ +async function timedFetch(url: string, init: RequestInit, label: string, timeoutMs: number): Promise { + try { + return await fetch(url, { ...init, signal: init.signal ? AbortSignal.any([init.signal, AbortSignal.timeout(timeoutMs)]) : AbortSignal.timeout(timeoutMs) }); + } catch (err) { + if (err instanceof Error && err.name === "TimeoutError") throw new Error(`Gitea API ${label}: no answer in ${timeoutMs / 1000} s, the request was cut`); + throw err; + } +} + export class GiteaClient { readonly baseUrl: string; + /** The limit of one request; a test sets it short. */ + requestTimeoutMs = FORGE_REQUEST_TIMEOUT_MS; private readonly tokens: TokenSource; /** A token given as a string is one somebody configured; a source decides for itself (token.ts). */ @@ -230,14 +249,14 @@ export class GiteaClient { private send(path: string, options: RequestInit, token: string): Promise { checkSentPath(path); - return fetch(`${this.baseUrl}/api/v1${path}`, { + return timedFetch(`${this.baseUrl}/api/v1${path}`, { ...options, headers: { "Content-Type": "application/json", Authorization: `token ${token}`, ...(options.headers as Record | undefined), }, - }); + }, `${options.method ?? "GET"} ${path}`, this.requestTimeoutMs); } /** Generic authenticated API call — the escape hatch for endpoints without a dedicated method. @@ -540,7 +559,7 @@ export class GiteaClient { * **A listing the forge cut is no listing** (it says `truncated` on a page with nothing more to read, or the pages * run past a bound): the call fails naming the repository and the commit, never answering the directories it did * see, since a directory left out would be taken for one without a module. Reads only. */ - async commitTrees(owner: string, repo: string, sha: string, paths?: Record): Promise<{ root: string; trees: Record; paths?: Record> }> { + async commitTrees(owner: string, repo: string, sha: string, paths?: Record, listed?: string[]): Promise<{ root: string; trees: Record; paths?: Record>; listed?: Record> }> { const base = `${repoPath(owner, repo)}/git/trees/${commitSegment(sha)}?recursive=true`; const cut = (why: string) => new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} was cut (${why}); no trees are answered from a part`); const entries: any[] = []; @@ -567,7 +586,23 @@ export class GiteaClient { if (!tree) throw new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} holds ${e.path} but not the tree of ${dir}`); trees[dir] = tree; } - if (paths === undefined) return { root, trees }; + // **Listed directories** (novox/hq ADR 0549 §2, issue 1126): each asked directory's files directly under it, by name + // to blob id, from the same listing: how the forge bridge finds the catalogue's placement files, one per node, at a + // merge. A directory the commit lacks lists nothing (the catalogue before its first placement file). + let files: Record> | undefined; + if (listed !== undefined) { + files = {}; + for (const dir of listed) { + files[dir] = {}; + const prefix = dir === "" ? "" : `${dir}/`; + for (const e of entries) { + if (e?.type !== "blob" || typeof e.path !== "string" || !e.path.startsWith(prefix)) continue; + const name = e.path.slice(prefix.length); + if (name !== "" && !name.includes("/")) files[dir][name] = String(e.sha); + } + } + } + if (paths === undefined) return files === undefined ? { root, trees } : { root, trees, listed: files }; // **Named paths** (novox/hq ADR 0499 §3, issue 1758): each asked path's git object id at the commit, a file's or a // directory's, from the same listing, by directory and by path relative to it. A path the listing lacks fails // naming both, and nothing is answered in part. @@ -582,7 +617,7 @@ export class GiteaClient { named[dir][p] = id; } } - return { root, trees, paths: named }; + return files === undefined ? { root, trees, paths: named } : { root, trees, paths: named, listed: files }; } /** Whether a repository holds a file at a commit: true, false for a 404, thrown when the forge cannot say. */ @@ -771,6 +806,22 @@ export class GiteaClient { /** One file's contents at a ref (default the repository's default branch), decoded. */ async getFile(owner: string, repo: string, path: string, ref?: string): Promise<{ path: string; ref?: string; sha: string; size: number; content: string }> { const qs = ref ? `?ref=${encodeURIComponent(ref)}` : ""; + // The size first, from the file's directory listing, which carries no content: a file over what an answer can carry is + // refused without its body being fetched (issue 1577). A file the listing does not show is asked for as before. + const clean = path.replace(/^\/+|\/+$/g, ""); + pathSegments("file path", clean); // a path that is another path or user is refused before anything reaches the forge + const slash = clean.lastIndexOf("/"); + const dir = slash < 0 ? "" : clean.slice(0, slash); + let listing: any[] | null = null; + try { + listing = await this.request(`${repoPath(owner, repo)}/contents${dir ? "/" + pathSegments("file path", dir) : ""}${qs}`); + } catch (err) { + if (!(err instanceof Error) || !/: 404 /.test(err.message)) throw err; + } + const entry = Array.isArray(listing) ? listing.find((e) => e?.path === clean) : undefined; + if (entry && Number(entry.size) > MAX_FILE_BYTES) { + throw new Error(`the file is ${Number(entry.size)} bytes, over the ${MAX_FILE_BYTES / 1024} KiB a tool answer can carry; read it from a clone`); + } const f = await this.request(`${repoPath(owner, repo)}/contents/${pathSegments("file path", path.replace(/^\/+|\/+$/g, ""))}${qs}`); if (!f || f.type !== "file") throw new Error(`Gitea API: ${path} is not a file`); const content = f.encoding === "base64" ? Buffer.from(String(f.content ?? ""), "base64").toString("utf8") : String(f.content ?? ""); @@ -940,14 +991,14 @@ export class GiteaAdmin { * throwing on it — the caller decides which non-2xx codes are idempotent successes. Only an * unexpected status becomes an error, and only where the caller says so. */ private async request(path: string, options: RequestInit = {}): Promise { - const res = await fetch(`${this.baseUrl}/api/v1${path}`, { + const res = await timedFetch(`${this.baseUrl}/api/v1${path}`, { ...options, headers: { "Content-Type": "application/json", Authorization: this.authorization, ...(options.headers as Record | undefined), }, - }); + }, `${options.method ?? "GET"} ${path}`, FORGE_REQUEST_TIMEOUT_MS); const text = await res.text(); let body: any = null; if (text) { @@ -1047,12 +1098,19 @@ export class GiteaAdmin { return true; } - /** What the forge itself says an account may do on a repository: none, read, write, admin or owner. */ + /** What the forge gives an account on a repository, effective: none, read, write, admin or owner. It is the repository as + * that account sees it (the site administrator acts as the account with `sudo`), so access through an organisation's team + * counts. The collaborator endpoint is not used: it names only a collaborator's own permission, and says "read" of an + * account that writes through a team (novox/hq issue 1945). */ async repositoryPermission(owner: string, repo: string, user: string): Promise { - const path = `${repoPath(owner, repo)}/collaborators/${encodeURIComponent(user)}/permission`; + const path = `${repoPath(owner, repo)}?sudo=${encodeURIComponent(user)}`; const res = await this.request(path); + if (res.status === 404) return "none"; if (res.status !== 200) GiteaAdmin.fail(path, res); - return String(res.body?.permission ?? "none"); + const p = res.body?.permissions; + if (p?.admin === true) return res.body?.owner?.login === user ? "owner" : "admin"; + if (p?.push === true) return "write"; + return p?.pull === true ? "read" : "none"; } /** Make an account a collaborator of a repository that reads only. The forge asks a repository's administrator for @@ -1067,6 +1125,38 @@ export class GiteaAdmin { await this.addCollaborator(owner, repo, user, "write"); } + /** The rule that applies to main in a repository, as its administrator reads it: a rule named main, else the first glob + * rule covering main; null when none does. With the administrator's login. */ + async mainRule(owner: string, repo: string): Promise { + const rules = await this.branchProtections(owner, repo); + return rules.find((r) => (r.rule_name ?? r.branch_name) === MAIN_BRANCH) + ?? rules.find((r) => ruleCovers(String(r.rule_name ?? r.branch_name ?? ""), MAIN_BRANCH)) ?? null; + } + + /** Put an account on main's merge whitelist in a repository of a user, and nothing else (the operator, 2026-10-26: + * write, the rule untouched). Only an existing rule whose merge whitelist is on is changed, and only its whitelist: the + * account is added to the logins already there. Push, status checks, approvals and the administrator override stay as + * the operator set them; with no rule, or the whitelist off, nothing is written (whoever writes may merge). Read back: + * the forge drops a whitelist login that cannot write the repository and still answers success, so a whitelist read back + * without the account fails loud. Answers the rule as the forge holds it after. With the administrator's login. */ + async ensureMergerListed(owner: string, repo: string, account: string): Promise<{ changed: boolean; rule: BranchProtection | null }> { + const rule = await this.mainRule(owner, repo); + const listed = rule?.merge_whitelist_usernames ?? []; + if (!rule || rule.enable_merge_whitelist !== true || listed.includes(account)) return { changed: false, rule }; + const name = String(rule.rule_name ?? rule.branch_name); + const path = `${repoPath(owner, repo)}/branch_protections/${encodeURIComponent(name)}`; + const res = await this.request(path, { method: "PATCH", body: JSON.stringify({ merge_whitelist_usernames: [...listed, account] }) }); + if (res.status !== 200) GiteaAdmin.fail(path, res); + const after = (await this.branchProtections(owner, repo)).find((r) => (r.rule_name ?? r.branch_name) === name); + if (!after) throw new Error(`main's rule ${name} of ${owner}/${repo} was written, but the forge does not list it`); + const held = after.merge_whitelist_usernames ?? []; + if (after.enable_merge_whitelist === true && !held.includes(account)) { + throw new Error(`${account} was added to the merge whitelist of main's rule ${name} of ${owner}/${repo}, but the forge ` + + `holds [${held.join(", ")}]: it drops a login that cannot write the repository`); + } + return { changed: true, rule: after }; + } + private async addCollaborator(owner: string, repo: string, user: string, permission: "read" | "write"): Promise { const path = `${repoPath(owner, repo)}/collaborators/${encodeURIComponent(user)}`; const res = await this.request(path, { method: "PUT", body: JSON.stringify({ permission }) }); @@ -1192,9 +1282,9 @@ export class GiteaAdmin { * other answer rejects (novox/hq issue 120). */ async holdsTeamMember(org: string, team: string, username: string, password: string): Promise { - const me = await fetch(`${this.baseUrl}/api/v1/user`, { + const me = await timedFetch(`${this.baseUrl}/api/v1/user`, { headers: { Authorization: "Basic " + Buffer.from(`${username}:${password}`).toString("base64") }, - }); + }, "GET /user", FORGE_REQUEST_TIMEOUT_MS); if (me.status === 401 || me.status === 403) return false; if (me.status !== 200) throw new Error(`Gitea GET /user as ${username}: ${me.status}`); const teams = await this.request(`/orgs/${encodeURIComponent(org)}/teams?limit=50`); diff --git a/modules/gitea/test/commit-trees.test.ts b/modules/gitea/test/commit-trees.test.ts index e685a5c0..f1ae6b7e 100644 --- a/modules/gitea/test/commit-trees.test.ts +++ b/modules/gitea/test/commit-trees.test.ts @@ -183,6 +183,15 @@ test("named paths: each asked path's git object id at the commit, a file's or a }); assert.deepEqual(got.trees, { "": ROOT, "modules/a": h("4"), "modules/b/c": h("8") }); assert.equal((await client.commitTrees("novox", "hq", COMMIT)).paths, undefined, "no paths asked, none answered"); + // Listed directories (ADR 0549 §2, issue 1126): the files directly under each, not deeper; a missing one lists none. + const listed = await client.commitTrees("novox", "hq", COMMIT, undefined, ["modules/a", "modules/b", "", "placement"]); + assert.deepEqual(listed.listed, { + "modules/a": { "main.go": h("6"), "module.json": h("5") }, + "modules/b": {}, + "": { "README.md": h("1"), "module.json": h("2") }, + placement: {}, + }); + assert.equal((await client.commitTrees("novox", "hq", COMMIT)).listed, undefined, "no directory asked, none listed"); } finally { server.close(); } @@ -209,7 +218,12 @@ test("gitea_commit_trees takes paths as {directory: [paths]} and refuses another const { client, server, asked } = await serve([{ sha: COMMIT, url: "u", tree: LISTING, truncated: false, page: 1, total_count: LISTING.length }]); try { const tool = getGiteaTools(client).find((t) => t.name === "gitea_commit_trees")!; - assert.ok("paths" in tool.input, "the tool names the argument"); + assert.ok("paths" in tool.input && "listed" in tool.input, "the tool names the arguments"); + const listedAnswer: any = await tool.run({ owner: "novox", repo: "hq", sha: COMMIT, listed: ["modules/d"] } as any); + assert.deepEqual(listedAnswer.listed, { "modules/d": { "notes.md": h("b") } }); + for (const bad of ["modules/d", ["../x"], ["/abs"], [1]]) { + await assert.rejects(tool.run({ owner: "novox", repo: "hq", sha: COMMIT, listed: bad } as any), /listed/, JSON.stringify(bad)); + } const got: any = await tool.run({ owner: "novox", repo: "hq", sha: COMMIT, paths: { "modules/a": ["main.go"] } } as any); assert.deepEqual(got.paths, { "modules/a": { "main.go": h("6") } }); const before = asked.length; diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index 6f1431aa..295de67b 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -640,6 +640,7 @@ test("no tool's caller text becomes another path or another user: `..` and `?sud await tool("gitea_delete_branch").run({ ...repo, branch: "feat/x" }); await tool("gitea_get_file").run({ ...repo, path: "modules/gitea/module.json" }).catch(() => undefined); assert.deepEqual(state.asked, ["DELETE /repos/novox/mesh-catalog/branches/feat/x", + "GET /repos/novox/mesh-catalog/contents/modules/gitea", // the file's size first (issue 1577) "GET /repos/novox/mesh-catalog/contents/modules/gitea/module.json"]); }); diff --git a/modules/gitea/test/paging.test.ts b/modules/gitea/test/paging.test.ts index a9f40a32..d6eba14d 100644 --- a/modules/gitea/test/paging.test.ts +++ b/modules/gitea/test/paging.test.ts @@ -90,3 +90,48 @@ test("a pass pages until it reaches back past its floor, and no further", async server.close(); } }); + +// **One forge request never holds the others, and a file too big for an answer is not downloaded** (novox/hq issue 1577): a +// request for a 4.9 MB file never ended and every read through the module timed out behind it. +test("a request the forge never answers is cut at the limit and says which request", async () => { + const { GiteaClient } = await import("../client.ts"); + const server = createServer((req) => { if (!req.url?.includes("fast")) return; }); + await new Promise((r) => server.listen(0, r)); + const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t"); + client.requestTimeoutMs = 150; + try { + const started = Date.now(); + await assert.rejects(client.api("/repos/novox/hq/pulls"), /Gitea API GET \/repos\/novox\/hq\/pulls: no answer in 0\.15 s/); + assert.ok(Date.now() - started < 3000); + } finally { + server.closeAllConnections(); + await new Promise((r) => server.close(() => r())); + } +}); + +test("a file over 512 KiB is refused from its directory listing, and its body is never asked for", async () => { + const { GiteaClient } = await import("../client.ts"); + const asked: string[] = []; + const server = createServer((req, res) => { + asked.push(req.url ?? ""); + res.setHeader("Content-Type", "application/json"); + if (req.url?.startsWith("/api/v1/repos/novox/hq/contents/bin?")) { + return res.end(JSON.stringify([{ path: "bin/big", type: "file", size: 5_000_000 }, { path: "bin/small", type: "file", size: 5 }])); + } + if (req.url?.startsWith("/api/v1/repos/novox/hq/contents/bin/small")) { + return res.end(JSON.stringify({ type: "file", encoding: "base64", sha: "s", size: 5, content: Buffer.from("hello").toString("base64") })); + } + res.statusCode = 404; + res.end("{}"); + }); + await new Promise((r) => server.listen(0, r)); + const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t"); + try { + await assert.rejects(client.getFile("novox", "hq", "bin/big", "main"), + /the file is 5000000 bytes, over the 512 KiB a tool answer can carry; read it from a clone/); + assert.ok(!asked.some((u) => u.includes("/contents/bin/big")), asked.join("\n")); + assert.equal((await client.getFile("novox", "hq", "bin/small", "main")).content, "hello"); + } finally { + await new Promise((r) => server.close(() => r())); + } +}); diff --git a/modules/gitea/test/tools-account.test.ts b/modules/gitea/test/tools-account.test.ts index 80519429..c69edb77 100644 --- a/modules/gitea/test/tools-account.test.ts +++ b/modules/gitea/test/tools-account.test.ts @@ -28,6 +28,7 @@ interface Team { id: number; name: string; units: any; members: Set } interface ForgeRepo { owner: string; name: string; full: string; private: boolean; archived?: boolean; ownerVisibility: string; collaborators: Map } const forgeRepos: ForgeRepo[] = []; +const userRules = new Map(); const repoFaults = { refuse: new Set(), permission: new Set() }; function addRepo(full: string, isPrivate: boolean, ownerVisibility = "public"): ForgeRepo { const [owner, name] = full.split("/"); @@ -84,7 +85,7 @@ function fakeForge() { const server = createServer(async (req, res) => { const url = new URL(req.url ?? "/", "http://fake"); const p = url.pathname.replace(/^\/api\/v1/, ""); - const me = who(req); + const me = url.searchParams.has("sudo") && isAdmin(who(req)) ? url.searchParams.get("sudo") : who(req); calls.push(`${req.method} ${p} as ${me}`); if (me === null) return json(res, 401, { message: "unauthorized" }); @@ -168,6 +169,23 @@ function fakeForge() { if (m[2] === "feat/unguarded") return json(res, 200, { name: m[2], protected: false }); return json(res, 404, { message: "branch does not exist" }); } + // A repository of a user: its rules are its administrator's (a site administrator, or a collaborator with admin). + m = p.match(/^\/repos\/(?!novox\/)([^/]+)\/([^/]+)\/branch_protections(?:\/(.*))?$/); + if (m) { + const key = `${m[1]}/${m[2]}`; + const repo = forgeRepos.find((r) => r.full === key); + if (!repo || !["admin", "owner"].includes(permissionOf(me, repo))) return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" }); + if (req.method === "GET" && m[3] === undefined) return json(res, 200, userRules.has(key) ? [userRules.get(key)] : []); + // As gitea does: a whitelist login that cannot write the repository is dropped, and the write still answered. + const keep = (b: any) => (Array.isArray(b.merge_whitelist_usernames) + ? { ...b, merge_whitelist_usernames: b.merge_whitelist_usernames.filter((u: string) => ["write", "admin", "owner"].includes(permissionOf(u, repo))) } : b); + if (req.method === "PATCH" && m[3] === "main") { + if (!userRules.has(key)) return json(res, 404, { message: "branch protection does not exist" }); + Object.assign(userRules.get(key), keep(await read(req))); + return json(res, 200, userRules.get(key)); + } + if (req.method === "POST") { const b = keep(await read(req)); userRules.set(key, b); return json(res, 201, b); } + } // gitea's reqAdmin on the whole group: reading the rules is administration too. m = p.match(/^\/repos\/novox\/([^/]+)\/branch_protections(?:\/(.*))?$/); if (m) { @@ -205,6 +223,7 @@ function fakeForge() { m = p.match(/^\/repos\/([^/]+)\/([^/]+)(?:\/collaborators\/([^/]+)(\/permission)?)?$/); if (m) { const repo = forgeRepos.find((r) => r.owner === decodeURIComponent(m![1]) && r.name === decodeURIComponent(m![2])); + if (repo && req.method === "GET" && !m[3] && url.searchParams.has("sudo") && repoFaults.permission.has(repo.full)) return json(res, 500, { message: "the forge is not well" }); if (!repo || permissionOf(me, repo) === "none") return json(res, 404, { message: "repository does not exist" }); if (req.method === "GET" && !m[3]) { const mine = permissionOf(me, repo); @@ -214,7 +233,8 @@ function fakeForge() { if (req.method === "GET" && m[4]) { if (repoFaults.permission.has(repo.full)) return json(res, 500, { message: "the forge is not well" }); if (!isAdmin(me) && me !== user) return json(res, 403, { message: "only the user or an administrator" }); - return json(res, 200, { permission: permissionOf(user, repo) }); + // As gitea answers: the collaborator's own permission, not what a team gives the account. + return json(res, 200, { permission: repo.collaborators.get(user) ?? "read" }); } if (req.method === "PUT" && !m[4]) { // gitea's reqAdmin: the repository's administration, which on a repository of another user the site administrator alone holds. @@ -608,7 +628,8 @@ test("the merging account is given write on each repository the indexer's module await writes.take({ key: "car-hunter-2", op: "put", value: { v: 1, repository: "jschoubben/car-hunter" } }); assert.equal(forge.calls.length, from, "a repository already written asks nothing more"); const repo = (full: string) => forgeRepos.find((x) => x.full === full)!; - assert.equal(repo("jschoubben/car-hunter").collaborators.get(TOOLS_ACCOUNT), "write"); + assert.equal(repo("jschoubben/car-hunter").collaborators.get(TOOLS_ACCOUNT), "write", "a user's repository is written, not administered"); + assert.equal(userRules.has("jschoubben/car-hunter"), false, "a repository with no rule for main is given none"); assert.equal(repo("novox/hq").collaborators.size, 0, "the organisation's repository is written through its team, no collaborator"); assert.equal((await client.getRepo("jschoubben", "car-hunter")).writes, true); assert.deepEqual(said, [{ type: WRITABLE_EVENT, body: { owner: "jschoubben", repo: "car-hunter", account: TOOLS_ACCOUNT } }]); @@ -636,9 +657,37 @@ test("the merging account is given write on each repository the indexer's module assert.ok(!forge.calls.slice(puts).some((c) => c.startsWith("PUT ")), "no write is asked on a repository no module builds from"); assert.equal(repo("jschoubben/dotfiles").collaborators.get(TOOLS_ACCOUNT), "read"); repoFaults.refuse.clear(); + // The operator's rule is left as set (the operator, 2026-10-26: "Write, rule untouched"): only mesh-tools is added to its + // merge whitelist; a status the mesh does not set and required approvals are said as blocking the mesh's merge, unchanged. + const operators = { rule_name: "main", enable_push: true, enable_merge_whitelist: true, merge_whitelist_usernames: ["jschoubben"], + enable_status_check: true, status_check_contexts: ["mesh/check", "mesh/merge-gate", "ci/external"], required_approvals: 2, + block_admin_merge_override: false, require_signed_commits: true, block_on_outdated_branch: true, block_on_rejected_reviews: true }; + userRules.set("jschoubben/snake", structuredClone(operators)); const after = await grantTool.run({ repositories: ["jschoubben/snake"] }) as any; assert.deepEqual(after.granted, ["jschoubben/snake"]); - assert.deepEqual(((await unwritableTool.run({})) as any).unwritable, []); + assert.equal(repo("jschoubben/snake").collaborators.get(TOOLS_ACCOUNT), "write"); + assert.deepEqual(userRules.get("jschoubben/snake"), { ...operators, merge_whitelist_usernames: ["jschoubben", TOOLS_ACCOUNT] }, "only the whitelist changed"); + assert.deepEqual(after.merge_whitelist, [{ repository: "jschoubben/snake", changed: true, listed: ["jschoubben", TOOLS_ACCOUNT] }]); + const blockedWhy = "writable, but main's rule blocks the mesh's merge: it requires the status ci/external, which the mesh does not set; " + + "it requires 2 approving reviews on the forge, which the gate does not give; it requires signed commits, which the mesh does not make; " + + "it refuses a branch behind main, which the mesh does not bring up to date; " + + "it refuses a merge after a review asked for changes, which the gate does not clear"; + assert.deepEqual(after.merge_blocked, [{ repository: "jschoubben/snake", why: blockedWhy }]); + const again = await unwritableTool.run({}) as any; + assert.deepEqual([again.unwritable, again.merge_blocked], [[], [{ repository: "jschoubben/snake", why: blockedWhy }]]); + const writesBefore = forge.calls.filter((c) => /^PATCH .*branch_protections/.test(c)).length; + assert.deepEqual(((await grantTool.run({ repositories: ["jschoubben/snake"] })) as any).merge_whitelist[0].changed, false); + assert.equal(forge.calls.filter((c) => /^PATCH .*branch_protections/.test(c)).length, writesBefore, "a listed account is not written again"); + // The mesh's own statuses never block: a rule requiring only them, with mesh-tools listed, is not reported. + userRules.set("jschoubben/snake", { rule_name: "main", enable_merge_whitelist: true, merge_whitelist_usernames: [TOOLS_ACCOUNT], + enable_status_check: true, status_check_contexts: ["mesh/check", "mesh/merge-gate", "mesh/repo-check"] }); + assert.deepEqual(((await unwritableTool.run({})) as any).merge_blocked, []); + assert.deepEqual(((await grantTool.run({ repositories: ["jschoubben/snake"] })) as any).merge_blocked, []); + // The forge drops a whitelist login that cannot write and still answers 200: the whitelist is read back and that fails loud. + userRules.set("jschoubben/dotfiles", { rule_name: "main", enable_merge_whitelist: true, merge_whitelist_usernames: ["jschoubben"] }); + await assert.rejects(GiteaAdmin.fromEnv(env).ensureMergerListed("jschoubben", "dotfiles", TOOLS_ACCOUNT), + /mesh-tools was added to the merge whitelist of main's rule main of jschoubben\/dotfiles, but the forge holds \[jschoubben\]: it drops a login that cannot write/); + userRules.delete("jschoubben/dotfiles"); // A module gone from the indexer leaves its repository out of the delivered list. await writes.take({ key: "snake", op: "delete" }); assert.deepEqual(writes.delivered(), ["jschoubben/car-hunter", "novox/hq"]); diff --git a/modules/gitea/tools/index.ts b/modules/gitea/tools/index.ts index 8f21cdb1..5db52501 100644 --- a/modules/gitea/tools/index.ts +++ b/modules/gitea/tools/index.ts @@ -76,6 +76,17 @@ function namedPaths(raw: unknown): Record | undefined { return out; } +/** The `listed` argument of gitea_commit_trees: [directory], each plain (no `.` or `..` segment, not absolute, the root + * the empty name); anything else is refused before the forge is asked. Absent is absent. */ +function listedDirs(raw: unknown): string[] | undefined { + if (raw === undefined || raw === null) return undefined; + if (!Array.isArray(raw) || raw.some((d) => typeof d !== "string")) throw new Error("listed is [directory, ...]"); + for (const d of raw as string[]) { + if (d !== "" && d.split("/").some((seg) => seg === "" || seg === "." || seg === "..")) throw new Error(`listed: ${JSON.stringify(d)} is not a plain relative directory`); + } + return raw as string[]; +} + export function getGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] { return namesChecked(allGiteaTools(gitea, admin)); } @@ -374,7 +385,7 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini // ---- Contents and branches ---- { name: "gitea_get_file", - description: "One file's contents from a repository, decoded, at a branch, tag or commit (default the repository's default branch).", + description: "One file's contents from a repository, decoded, at a branch, tag or commit (default the repository's default branch). Refuses a file over 512 KiB, which a tool answer cannot carry: read it from a clone.", input: { owner: { type: "string", description: "the repository owner" }, repo: { type: "string", description: "the repository name" }, @@ -550,19 +561,20 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini }, { name: "gitea_commit_trees", - description: "A commit's trees from the forge's one recursive listing: the root tree, and the tree of every directory that holds a module.json (the root as an empty name), each by its git hash. What the forge bridge puts in a merge's message (novox/hq ADR 0437 §5). With paths, {directory: [path relative to it]}, the answer also carries paths, {directory: {path: git object id}} of each named file or directory at the commit (novox/hq ADR 0499 §3); a path the commit lacks fails naming the directory and the path. The commit is a full 40-hex sha. A listing the forge cut is refused, never answered in part. Reads only.", + description: "A commit's trees from the forge's one recursive listing: the root tree, and the tree of every directory that holds a module.json (the root as an empty name), each by its git hash. What the forge bridge puts in a merge's message (novox/hq ADR 0437 §5). With paths, {directory: [path relative to it]}, the answer also carries paths, {directory: {path: git object id}} of each named file or directory at the commit (novox/hq ADR 0499 §3); a path the commit lacks fails naming the directory and the path. The commit is a full 40-hex sha. With listed, [directory], the answer also carries listed, {directory: {name: git blob id}} of the files directly under each (the catalogue's placement files, novox/hq ADR 0549 §2). A listing the forge cut is refused, never answered in part. Reads only.", input: { owner: { type: "string", description: "the repository owner" }, repo: { type: "string", description: "the repository name" }, sha: { type: "string", description: "the commit, its full 40-hex sha" }, paths: { type: "object", description: "optional, {directory: [path relative to it]}: the git object id of each named file or directory is answered under paths; the root directory is the empty name" }, + listed: { type: "array", items: { type: "string" }, description: "optional, [directory]: the files directly under each, {directory: {name: git blob id}}, are answered under listed; a directory the commit lacks lists none" }, }, run: async (args) => { const owner = String(args.owner ?? "").trim(), repo = String(args.repo ?? "").trim(); const sha = String(args.sha ?? "").trim(); if (!owner || !repo) throw new Error("name the repository: owner and repo"); if (!/^[0-9a-f]{40}$/i.test(sha)) throw new Error("sha is a commit's full 40-hex sha"); - return gitea.commitTrees(owner, repo, sha, namedPaths(args.paths)); + return gitea.commitTrees(owner, repo, sha, namedPaths(args.paths), listedDirs(args.listed)); }, }, { diff --git a/modules/gitea/writes.ts b/modules/gitea/writes.ts index ae9d41a6..552f7d58 100644 --- a/modules/gitea/writes.ts +++ b/modules/gitea/writes.ts @@ -18,10 +18,18 @@ // `repo.writable` {owner, repo, account}: mesh-delivery-gate, which does not ask the operator's Approve of a pull request // whose merge the forge would refuse, reads that repository again on it. // +// **A repository of a user is written, and main's rule is left as the operator set it** (the operator, 2026-10-26, asked +// "Write, rule untouched"): the account is a write collaborator there, and only added to main's merge whitelist when that +// rule keeps one (client.ts ensureMergerListed). What else in main's rule stops the mesh's merge (a status the mesh does not +// set, approvals the gate does not give) is changed by nobody but said: both tools list such a repository as writable, but +// blocked, with why (merge_blocked). +// // Not taken away: a repository whose last module goes from the indexer keeps its collaborator until a hand removes it; // gitea_unwritable_repos and gitea_grant_write say the delivered list as it is now. import type { ForgeRepository } from "./client.js"; +import type { BranchProtection } from "./protection.js"; +import { CHECK_CONTEXT, CHECK_STATUS, REPO_CHECK_CONTEXT } from "./pulls.js"; /** The indexer's state this module reads: one key per registered module, its value naming the repository. */ export const INDEXER_MODULES = "mesh-delivery-indexer.modules"; @@ -34,8 +42,35 @@ export interface WriteForge { repository(owner: string, repo: string): Promise; repositoryPermission(owner: string, repo: string, user: string): Promise; addWriteCollaborator(owner: string, repo: string, user: string): Promise; + mainRule(owner: string, repo: string): Promise; + ensureMergerListed(owner: string, repo: string, account: string): Promise<{ changed: boolean; rule: BranchProtection | null }>; } +/** The statuses every pull request the mesh delivers carries (mesh-delivery's checks); a rule requiring another blocks. */ +const MESH_CONTEXTS = new Set([CHECK_STATUS, CHECK_CONTEXT, REPO_CHECK_CONTEXT]); + +/** What in main's rule stops the mesh's merge, in words; empty when nothing does. Changed by nobody here: the operator's. */ +export function mergeBlocks(rule: BranchProtection | null, account: string): string[] { + if (!rule) return []; + const blocks: string[] = []; + if (rule.enable_merge_whitelist === true && !(rule.merge_whitelist_usernames ?? []).includes(account)) { + blocks.push(`its merge whitelist [${(rule.merge_whitelist_usernames ?? []).join(", ")}] leaves out ${account} (gitea_grant_write adds it)`); + } + if (rule.enable_status_check === true) { + for (const c of rule.status_check_contexts ?? []) { + if (!MESH_CONTEXTS.has(c)) blocks.push(`it requires the status ${c}, which the mesh does not set`); + } + } + const approvals = Number(rule.required_approvals ?? 0); + if (approvals > 0) blocks.push(`it requires ${approvals} approving review${approvals === 1 ? "" : "s"} on the forge, which the gate does not give`); + if (rule.require_signed_commits === true) blocks.push("it requires signed commits, which the mesh does not make"); + if (rule.block_on_outdated_branch === true) blocks.push("it refuses a branch behind main, which the mesh does not bring up to date"); + if (rule.block_on_rejected_reviews === true) blocks.push("it refuses a merge after a review asked for changes, which the gate does not clear"); + return blocks; +} + +const blockedWhy = (blocks: string[]): string => `writable, but main's rule blocks the mesh's merge: ${blocks.join("; ")}`; + /** One change of the indexer's modules state, as the SDK's state watch hands it. */ export interface ModulesChange { key: string; @@ -51,8 +86,13 @@ export interface Written { delivered: string[]; /** Made writable now, each read back from the forge. */ granted: string[]; - /** Written already (the organisation's team, or a collaborator that writes). */ + /** Written already (a collaborator of a user's repository), or written through the organisation's team. */ already: string[]; + /** Each repository of a user: whether the account was added to main's merge whitelist now, and the whitelist read back + * (null: main's rule keeps no whitelist, or there is no rule, and whoever writes may merge). */ + merge_whitelist: { repository: string; changed: boolean; listed: string[] | null }[]; + /** Writable, but main's rule (the operator's, left as set) stops the mesh's merge, and why. */ + merge_blocked: { repository: string; why: string }[]; /** The forge refused, in its own words; or the repository is the organisation's, whose team should write it. */ refused: { repository: string; why: string }[]; /** Repositories the account cannot write that are left so, and why: they build no registered module. */ @@ -118,7 +158,7 @@ export class DeliveredWrites { "(the watch is not granted or not answered); nothing is granted"); } const delivered = this.delivered(); - const result: Written = { account: this.account, delivered, granted: [], already: [], refused: [], left_read_only: [] }; + const result: Written = { account: this.account, delivered, granted: [], already: [], merge_whitelist: [], merge_blocked: [], refused: [], left_read_only: [] }; const named = only && only.length ? only.map((x) => String(x).trim()) : null; const targets: string[] = []; for (const full of named ?? delivered) { @@ -140,36 +180,55 @@ export class DeliveredWrites { result.refused.push({ repository: full, why: `its permission could not be read — ${said(err)}` }); continue; } - if (WRITES.has(permission)) { - this.writable.add(full); - result.already.push(full); - continue; - } if (owner === this.org) { + if (WRITES.has(permission)) { + this.writable.add(full); + result.already.push(full); + continue; + } // The organisation's repositories are written through its team (tools-account.ts), never a collaborator. result.refused.push({ repository: full, why: `a repository of ${this.org}, which ${this.account} writes through the ` + `team ${this.org}/${this.account}; the forge gives it "${permission}", so the team is not as it should be (it is ` + "set again at the next mint of the tools' token)" }); continue; } - try { - await this.forge.addWriteCollaborator(owner, name, this.account); - } catch (err) { - result.refused.push({ repository: full, why: said(err) }); - continue; + // A repository of a user: the account writes it, and is put on main's merge whitelist when the rule keeps one. + let changed = false; + if (!WRITES.has(permission)) { + try { + await this.forge.addWriteCollaborator(owner, name, this.account); + } catch (err) { + result.refused.push({ repository: full, why: said(err) }); + continue; + } + let after: string; + try { + after = await this.forge.repositoryPermission(owner, name, this.account); + } catch (err) { + result.refused.push({ repository: full, why: `granted, but its permission could not be read back — ${said(err)}` }); + continue; + } + if (!WRITES.has(after)) { + result.refused.push({ repository: full, why: `the grant was answered, but the forge still gives ${this.account} "${after}"` }); + continue; + } + changed = true; } - let after: string; try { - after = await this.forge.repositoryPermission(owner, name, this.account); + const { changed: listedNow, rule } = await this.forge.ensureMergerListed(owner, name, this.account); + result.merge_whitelist.push({ repository: full, changed: listedNow, + listed: rule?.enable_merge_whitelist === true ? (rule.merge_whitelist_usernames ?? []) : null }); + const blocks = mergeBlocks(rule, this.account); + if (blocks.length > 0) result.merge_blocked.push({ repository: full, why: blockedWhy(blocks) }); } catch (err) { - result.refused.push({ repository: full, why: `granted, but its permission could not be read back — ${said(err)}` }); - continue; - } - if (!WRITES.has(after)) { - result.refused.push({ repository: full, why: `the grant was answered, but the forge still gives ${this.account} "${after}"` }); + result.refused.push({ repository: full, why: `${this.account} writes it, but could not be put on main's merge whitelist — ${said(err)}` }); continue; } this.writable.add(full); + if (!changed) { + result.already.push(full); + continue; + } result.granted.push(full); await this.emit(WRITABLE_EVENT, { owner, repo: name, account: this.account }).catch((err) => this.log(`${this.account} writes ${full} now, and that could not be said on the bus — ${said(err)}`)); @@ -192,10 +251,12 @@ export class DeliveredWrites { } /** Every delivered repository the account cannot write, as the forge says now, and why. */ - async unwritable(): Promise<{ account: string; delivered: number; unwritable: { repository: string; permission: string; why: string }[] }> { + async unwritable(): Promise<{ account: string; delivered: number; unwritable: { repository: string; permission: string; why: string }[]; + merge_blocked: { repository: string; why: string }[] }> { if (!this.heard) throw new Error(`which repositories build a registered module is not known yet: ${INDEXER_MODULES} has not been read`); const delivered = this.delivered(); const out: { repository: string; permission: string; why: string }[] = []; + const blocked: { repository: string; why: string }[] = []; for (const full of delivered) { const [owner, name] = split(full); let permission: string; @@ -205,12 +266,21 @@ export class DeliveredWrites { out.push({ repository: full, permission: "unknown", why: `the forge did not answer ${this.account}'s permission on it — ${said(err)}` }); continue; } - if (WRITES.has(permission)) continue; + if (WRITES.has(permission)) { + if (owner === this.org) continue; // the organisation's main rules are the mesh's own (mainrules.ts) + try { + const blocks = mergeBlocks(await this.forge.mainRule(owner, name), this.account); + if (blocks.length > 0) blocked.push({ repository: full, why: blockedWhy(blocks) }); + } catch (err) { + blocked.push({ repository: full, why: `writable, but main's rule could not be read — ${said(err)}` }); + } + continue; + } const modules = [...this.modules].filter(([, r]) => r === full).map(([m]) => m).sort(); out.push({ repository: full, permission, why: `builds ${modules.join(", ")}, and the forge gives ${this.account} ` + `"${permission}" on it: its commit status and its merge are refused until gitea_grant_write grants it` }); } - return { account: this.account, delivered: delivered.length, unwritable: out }; + return { account: this.account, delivered: delivered.length, unwritable: out, merge_blocked: blocked }; } } @@ -224,14 +294,19 @@ export function writeTools(writes: () => DeliveredWrites): Tool[] { description: "Every repository that builds a registered module (the indexer's modules, read over the bus) on which " + "the mesh's merging forge account (mesh-tools, which sets commit statuses and merges) cannot write, with the " + "permission the forge gives it there and why: such a pull request can be checked but never merged, and the " + - "delivery gate does not ask its Approve. Empty when it writes every one. Read with the site administrator's login.", + "delivery gate does not ask its Approve. Empty when it writes every one. Its merge_blocked lists each repository of " + + "a user it writes whose main's rule (the operator's, left as set) still stops the mesh's merge: a merge whitelist " + + "without it, a required status the mesh does not set, required approving reviews. Read with the site administrator's login.", input: {}, run: async () => writes().unwritable(), }, { name: "gitea_grant_write", - description: "Give the mesh's merging forge account (mesh-tools) write, as a collaborator, on every repository that " + - "builds a registered module and that it cannot write now, or on the repositories named (each must build one). " + + description: "Give the mesh's merging forge account (mesh-tools) write, as a collaborator, on every repository of a " + + "user that builds a registered module, or on the repositories named (each must build one). Main's rule there is left " + + "as the operator set it; only when it keeps a merge whitelist is mesh-tools added to it, read back (merge_whitelist). " + + "What else in that rule stops the mesh's merge (a required status the mesh does not set, required approving reviews) " + + "is changed by nobody and said in merge_blocked. " + "The module does this itself at its start and when a module of a new repository is registered; this is the same " + "act by hand, such as when a merge or a commit status was refused 403. A repository of the organisation novox " + "is written through its team and needs nothing. Answers what was granted (each read back from the forge), what " + diff --git a/modules/i3/internal/desktop/reads-outside b/modules/i3/internal/desktop/reads-outside index f6b2d871..7ef55c53 100644 --- a/modules/i3/internal/desktop/reads-outside +++ b/modules/i3/internal/desktop/reads-outside @@ -1 +1,9 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of this package in every module that carries one. +modules/xorg/internal/desktop/*.go +modules/lemurs/internal/desktop/*.go +modules/i3/internal/desktop/*.go +modules/xterm/internal/desktop/*.go +modules/adwaita/internal/desktop/*.go diff --git a/modules/lemurs/internal/desktop/reads-outside b/modules/lemurs/internal/desktop/reads-outside index f6b2d871..7ef55c53 100644 --- a/modules/lemurs/internal/desktop/reads-outside +++ b/modules/lemurs/internal/desktop/reads-outside @@ -1 +1,9 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of this package in every module that carries one. +modules/xorg/internal/desktop/*.go +modules/lemurs/internal/desktop/*.go +modules/i3/internal/desktop/*.go +modules/xterm/internal/desktop/*.go +modules/adwaita/internal/desktop/*.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index f514673c..816793b2 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -84,9 +84,17 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep // The capacity's own change wakes the puller (issue 1887): the runtime's watch of the config value as mesh-config // resolved it; the settings file is read only when that watch cannot start. A test's Capacity is its own. var watch func(string, func(stdio.ProvisionChange) error) error + // And the CPU bounds it (issue 1806's follow-up): the configured capacity is an upper bound, the node's build slice + // gives floor(its quota's cores / the module's cores-per-step) at once, read live; a test judges by its own Capacity. + // And the node's build hours (the operator, 2026-10-26: g14 builds only from 18:00 to 06:00): outside them it takes + // nothing new. + var cores func() float64 + var hours func() (*buildstep.BuildWindow, string) if capacity == nil { + hours = buildstep.NodeBuildHours(m.Settings, m.Node, m.Say) capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say) watch = buildstep.WatchConfigServed + cores = buildstep.NodeCoresPerStep(m.Settings, buildstep.DefaultCoresPerStep[delivery.AskGo], m.Say) } ask := m.askSteps(steps) var load func(string) ([]byte, error) @@ -97,7 +105,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep } } return &buildstep.Puller{Source: source, Language: delivery.AskGo, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, - Capacity: capacity, WatchConfig: watch, Logf: m.Say, Has: m.Has, + Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Logf: m.Say, Has: m.Has, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { if err := m.Check(ctx); err != nil { return nil, err diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go index 91783730..20f6d788 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "fmt" "os" "slices" "strings" @@ -125,12 +126,28 @@ func TestManifestClaimsTheGoBuildSeatAndReadsItsCapacity(t *testing.T) { raw, _ := os.ReadFile(moduleDir + "/module.json") var doc struct { Config map[string]struct { - Default string `json:"default"` + Default json.RawMessage `json:"default"` } `json:"config"` } _ = json.Unmarshal(raw, &doc) - if doc.Config[buildstep.CapacitySetting].Default != buildstep.DefaultCapacities { - t.Errorf("the capacity's default = %q, want %q", doc.Config[buildstep.CapacitySetting].Default, buildstep.DefaultCapacities) + // Issue 1806's follow-up: the cores a step needs is a setting of its own, in the settings file, with the SDK's default. + if cores := string(doc.Config[buildstep.CoresPerStepSetting].Default); cores != fmt.Sprint(buildstep.DefaultCoresPerStep[delivery.AskGo]) { + t.Errorf("the cores-per-step default = %s, want %g", cores, buildstep.DefaultCoresPerStep[delivery.AskGo]) + } + if got := buildstep.NodeCoresPerStep(moduleDir+"/files/settings.json", 0, nil)(); got != buildstep.DefaultCoresPerStep[delivery.AskGo] { + t.Errorf("the settings template holds cores-per-step %g", got) + } + // And the build hours (the operator, 2026-10-26): a setting of their own, empty by default (any hour), in the template. + if hours := string(doc.Config[buildstep.BuildHoursSetting].Default); hours != `""` { + t.Errorf("the build-hours default = %s, want empty", hours) + } + if w, fault := buildstep.NodeBuildHours(moduleDir+"/files/settings.json", "g14", t.Errorf)(); w != nil || fault != "" { + t.Errorf("the settings template holds build hours %v (%s)", w, fault) + } + var capacity string + _ = json.Unmarshal(doc.Config[buildstep.CapacitySetting].Default, &capacity) + if capacity != buildstep.DefaultCapacities { + t.Errorf("the capacity's default = %q, want %q", capacity, buildstep.DefaultCapacities) } settings := resource(t, m, "settings") if settings["path"] != "${dir:state}/settings.json" { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go index 22aebaca..f59242f7 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/beside_digest_test.go @@ -116,7 +116,7 @@ func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) { t.Errorf("the integration digest listed %q", s.command) } } - if DigestVersion != "5" { + if DigestVersion != "6" { t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion) } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/cpu.go b/modules/mesh-build-go/cmd/mesh-build-go/cpu.go index 24457cb4..c1c00903 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/cpu.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/cpu.go @@ -15,6 +15,8 @@ import ( func (m *Module) cpuLimits() buildcpu.Limits { slots := 1 switch { + case m.Held != nil: + slots = m.Held.CapacityNow() // the capacity the puller judges by, bounded by the node's build slice case m.Capacity != nil: slots = m.Capacity() case m.Settings != "": @@ -32,5 +34,9 @@ func (m *Module) threadEnv() []string { // cpuStatus is what the status tool says of it: the limits a step is given, and what the kernel holds this runtime // to (its unit and its slice), read from its cgroup rather than assumed. func (m *Module) cpuStatus() map[string]any { - return map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()} + out := map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()} + if m.Held != nil { + out["capacity"] = m.Held.CapacityDetail() // the capacity judged by and why (issue 1806's follow-up) + } + return out } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index 746a6f85..0e0133e2 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -16,6 +16,7 @@ import ( "io" "io/fs" "os" + "path" "path/filepath" "sort" "strconv" @@ -29,17 +30,25 @@ import ( // commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement). // 4: the siblings' commits enter only the digest of a package whose tests reach a file naming MESH_CHECK_BESIDE (ADR 0536 §1). // 5: a catalogue module's non-Go files enter the digest of its packages (issue 1935). -const DigestVersion = "5" +// 6: a reads-outside marker that lists what its tests read enters the digest with those files' contents (issue 1936). +const DigestVersion = "6" // besideVariable is the name a file bears when it reads where the siblings lie; a package whose tests reach such a file // depends on the siblings' commits. const besideVariable = "MESH_CHECK_BESIDE" // ReadsOutsideMarker is the name of a file in a package's directory that says its tests read files outside it (a -// repository's documents, a sibling directory): such a package is never kept (issue 1935). The digest hashes only what -// the Go toolchain lists, so it cannot name what a test opens elsewhere; a marker file, beside the Go files where a -// reader of the package sees it, is the plain way to say so until the checker keeps the files a test opened (the Go -// test log) with each verdict. +// repository's documents, a sibling directory), which the Go toolchain does not list (issue 1935). It lists, one per +// line, what those tests read, as paths from the repository's root; a path may be a glob (filepath.Match), and a +// directory it names is read whole (issue 1936). Blank lines and lines starting with # are not read. A line starting +// with ? names a path whose absence is what the tests read (a file that must not be there): it may match nothing. +// +// The digest then hashes, besides what the toolchain lists, every line of the marker, every file it names (under its +// path in the tree, by content) and every file of the package's own directory, Go files included (a file its build tags +// exclude is one go list does not hash). A marker that lists nothing, a line that is not a clean relative path, or a +// line without ? that matches nothing makes the package one that is never kept (its digest a fresh random hex): a list +// nobody can trust keeps nothing. The exact fix, the files a test opened kept +// from the Go test log with each verdict, replaces the lists (issue 1936). const ReadsOutsideMarker = "reads-outside" // NeedsMarker is the name of a file in a package's directory that names, one per line, what its integration tests need @@ -128,9 +137,13 @@ type DigestResult struct { // only when they are the siblings it gave the fetch. Beside []string `json:"beside,omitempty"` WithoutTests []string `json:"without_tests,omitempty"` - // ReadsOutside names the packages that carry the marker: each has a digest that no kept verdict can have (a fresh - // random one), so the check runs it every time. + // ReadsOutside names the packages whose marker lists nothing it can trust (ReadsOutsideMarker): each has a digest + // that no kept verdict can have (a fresh random one), so the check runs it every time. ReadsOutside []string `json:"reads_outside,omitempty"` + // neverKept is, per package of ReadsOutside, why its marker cannot be trusted; hashedFiles caches the content lines + // of the files markers name, which several packages share (the catalogue's manifests). + neverKept map[string]string + hashedFiles map[string]string // Needs are the packages whose tests at this level need something of the node, as their NeedsMarker names it. Needs []delivery.PackageNeeds `json:"needs,omitempty"` // JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate " under the @@ -247,7 +260,7 @@ func (m *Module) digestHere(ctx context.Context, workspace, level string) (Diges sort.Slice(res.Needs, func(i, j int) bool { return res.Needs[i].Package < res.Needs[j].Package }) for _, p := range res.ReadsOutside { if m.Say != nil { - m.Say("the %s level of %s always runs: its directory carries %s, so its tests read files outside it that no digest names", level, p, ReadsOutsideMarker) + m.Say("the %s level of %s always runs: its %s %s, so its tests read files outside it that no digest names", level, p, ReadsOutsideMarker, res.neverKept[p]) } } return res, nil @@ -316,18 +329,27 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he res.Needs = append(res.Needs, delivery.PackageNeeds{Package: path, Needs: n}) } } - if _, err := os.Stat(filepath.Join(root.Dir, ReadsOutsideMarker)); err == nil { - // The digest of a package that reads outside itself is one nothing can have kept: the key is a fresh random - // hex, so the ask finds no kept verdict and runs it, and what the checker writes under it is never read again. + declared, why, err := declaredReads(tree, root, res.hashed()) + if err != nil { + return fmt.Errorf("%s: its %s: %w", path, ReadsOutsideMarker, err) + } + if why != "" { + // The digest of a package whose reads outside itself are not listed is one nothing can have kept: the key is a + // fresh random hex, so the ask finds no kept verdict and runs it, and what the checker writes under it is never + // read again. var nonce [32]byte if _, err := rand.Read(nonce[:]); err != nil { return err } res.Packages[path] = hex.EncodeToString(nonce[:]) res.ReadsOutside = append(res.ReadsOutside, path) + if res.neverKept == nil { + res.neverKept = map[string]string{} + } + res.neverKept[path] = why continue } - d, err := packageDigest(root, testMain, pkgs, tree, header, beside) + d, err := packageDigest(root, testMain, pkgs, tree, header, beside, declared) if err != nil { return fmt.Errorf("%s: %w", path, err) } @@ -340,14 +362,15 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he // tests reach in the module's own tree (by their content, under their path in the tree), the version and checksum of // every module outside it, and the package's own test data. The beside lines (the siblings' commits) join the header // only when a file the tests reach names MESH_CHECK_BESIDE. A package of the standard library is the Go version's. -func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header, beside []string) (string, error) { +// The lines a package's ReadsOutsideMarker declares join them (declaredReads). +func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header, beside, declared []string) (string, error) { reached := map[string]*goPackage{root.ImportPath: root} for _, dep := range testMain.Deps { if p := pkgs[dep]; p != nil { reached[dep] = p } } - lines := []string{} + lines := append([]string{}, declared...) readsBeside := false for _, p := range reached { if p.Standard || p.Module == nil { @@ -469,3 +492,104 @@ func contentLine(tree string, p *goPackage, path string) (string, error) { sum := sha256.Sum256(raw) return "file " + label + " " + hex.EncodeToString(sum[:]), nil } + +// hashed is the cache of content lines a digest step shares across its packages. +func (r *DigestResult) hashed() map[string]string { + if r.hashedFiles == nil { + r.hashedFiles = map[string]string{} + } + return r.hashedFiles +} + +// declaredReads are the digest lines a package's ReadsOutsideMarker adds: each of its lines, the content line of every +// file it names, and those of every file of the package's own directory. Without a marker it answers nothing. why is +// set, and the lines nil, when the marker cannot be trusted: it lists nothing, a line is not a clean relative path, or +// a line without ? matches nothing. +func declaredReads(tree string, root *goPackage, cache map[string]string) (lines []string, why string, err error) { + raw, err := os.ReadFile(filepath.Join(root.Dir, ReadsOutsideMarker)) + if errors.Is(err, fs.ErrNotExist) { + return nil, "", nil + } + if err != nil { + return nil, "", err + } + var patterns []string + for _, l := range strings.Split(string(raw), "\n") { + if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") { + patterns = append(patterns, l) + } + } + if len(patterns) == 0 { + return nil, "lists nothing", nil + } + seen := map[string]bool{} + add := func(path string) error { + if seen[path] { + return nil + } + seen[path] = true + l, ok := cache[path] + if !ok { + var err error + if l, err = contentLine(tree, root, path); err != nil { + return err + } + cache[path] = l + } + lines = append(lines, "read "+l) + return nil + } + for _, l := range patterns { + pattern, absent := strings.CutPrefix(l, "?") + if !cleanTreePattern(pattern) { + return nil, fmt.Sprintf("names %q, which is not a clean path from the repository's root", l), nil + } + matches, err := filepath.Glob(filepath.Join(tree, filepath.FromSlash(pattern))) + if err != nil { + return nil, fmt.Sprintf("names %q, which is not a glob", l), nil + } + if len(matches) == 0 && !absent { + return nil, fmt.Sprintf("names %q, which matches nothing (a path whose absence is read starts with ?)", l), nil + } + lines = append(lines, "reads "+l) + for _, match := range matches { + if err := walkFiles(match, add); err != nil { + return nil, "", err + } + } + } + // Every file of the package's own directory: its lists, its marker, and its Go files too, since go list leaves a + // file its build tags exclude (an integration test a unit test globs) out of what the digest hashes. A file the + // toolchain lists is hashed twice, which changes nothing. + err = walkFiles(root.Dir, add) + return lines, "", err +} + +// cleanTreePattern says a marker's line is a path from the repository's root that stays inside it. +func cleanTreePattern(p string) bool { + if p == "" || strings.ContainsAny(p, " \t\\") || strings.HasPrefix(p, "/") || path.Clean(p) != p { + return false + } + for _, e := range strings.Split(p, "/") { + if e == ".." || e == "." { + return false + } + } + return true +} + +// walkFiles calls each with every file at or under path, skipping .git and node_modules, in lexical order. +func walkFiles(path string, each func(string) error) error { + return filepath.WalkDir(path, func(p string, e fs.DirEntry, err error) error { + if err != nil { + return err + } + if e.IsDir() { + if n := e.Name(); p != path && (n == ".git" || n == "node_modules") { + return filepath.SkipDir + } + return nil + } + return each(p) + }) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go index a2bdd155..a6dac3a4 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest_test.go @@ -173,7 +173,7 @@ func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) { testMain := &goPackage{ImportPath: "x/r.test", Deps: []string{ext.ImportPath}} pkgs[root.ImportPath] = root digest := func() string { - d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"}, nil) + d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"}, nil, nil) if err != nil { t.Fatal(err) } @@ -190,28 +190,74 @@ func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) { } } -// Issue 1935: a package whose directory carries the reads-outside marker is never kept: its digest is one no kept verdict -// can have (it differs on every call, though nothing changed), and the answer names it. A package without the marker -// keeps its digest. -func TestAPackageWithTheReadsOutsideMarkerIsNeverKept(t *testing.T) { +// Issue 1935, 1936: a package whose reads-outside marker lists nothing it can trust is never kept: its digest is one no +// kept verdict can have (it differs on every call, though nothing changed), and the answer names it. A marker that lists +// what the tests read is kept: a change to a file it names changes the package's digest, a change elsewhere does not. +func TestAPackageWithTheReadsOutsideMarkerIsKeptOnlyByWhatItLists(t *testing.T) { t.Parallel() m, tree := digestFixture(t) plain := digests(t, m, "unit") if len(plain.ReadsOutside) != 0 { t.Fatalf("a package carries the marker in the fixture: %v", plain.ReadsOutside) } - edit(t, tree, "c/"+ReadsOutsideMarker, "the tests read ../docs\n") - first, second := digests(t, m, "unit"), digests(t, m, "unit") - c := "fixture/digest/c" - if len(first.Packages[c]) != 64 || first.Packages[c] == second.Packages[c] || first.Packages[c] == plain.Packages[c] { - t.Errorf("c carries the marker and its digest is stable: %s, %s, before %s", first.Packages[c], second.Packages[c], plain.Packages[c]) + c, a := "fixture/digest/c", "fixture/digest/a" + if err := os.MkdirAll(filepath.Join(tree, "docs"), 0o755); err != nil { + t.Fatal(err) } - if strings.Join(first.ReadsOutside, ",") != c { - t.Errorf("the answer does not name the package that reads outside: %v", first.ReadsOutside) + edit(t, tree, "docs/one.md", "one\n") + edit(t, tree, "other.md", "other\n") + for name, marker := range map[string]string{ + "an empty marker": "# nothing listed\n\n", + "a sentence": "the tests read ../docs\n", + "a path climbing out": "../docs\n", + "a path matching nothing": "docs/none.md\n", + "an absolute path": "/etc/passwd\n", + "a path that is not clean": "docs/./one.md\n", + "a good line and a bad line": "docs\nnope/*.md\n", + } { + edit(t, tree, "c/"+ReadsOutsideMarker, marker) + first, second := digests(t, m, "unit"), digests(t, m, "unit") + if len(first.Packages[c]) != 64 || first.Packages[c] == second.Packages[c] || first.Packages[c] == plain.Packages[c] { + t.Errorf("%s: c's digest is stable: %s, %s, before %s", name, first.Packages[c], second.Packages[c], plain.Packages[c]) + } + if strings.Join(first.ReadsOutside, ",") != c { + t.Errorf("%s: the answer does not name the package that reads outside: %v", name, first.ReadsOutside) + } + if first.Packages[a] != plain.Packages[a] || first.Packages[a] != second.Packages[a] { + t.Errorf("%s: a has no marker and its digest moved: %s, %s, before %s", name, first.Packages[a], second.Packages[a], plain.Packages[a]) + } } - a := "fixture/digest/a" - if first.Packages[a] != plain.Packages[a] || first.Packages[a] != second.Packages[a] { - t.Errorf("a has no marker and its digest moved: %s, %s, before %s", first.Packages[a], second.Packages[a], plain.Packages[a]) + + // A list: a glob, a directory read whole, and a path whose absence is read. + edit(t, tree, "c/"+ReadsOutsideMarker, "# what c's tests read\ndocs/*.md\ndocs\n?gone\n") + before := digests(t, m, "unit") + if len(before.ReadsOutside) != 0 || len(before.Packages[c]) != 64 || before.Packages[c] == plain.Packages[c] { + t.Fatalf("a listed marker is not kept: %v, %s (no marker %s)", before.ReadsOutside, before.Packages[c], plain.Packages[c]) + } + if again := digests(t, m, "unit"); again.Packages[c] != before.Packages[c] { + t.Errorf("nothing changed and c's digest moved") + } + for _, step := range []struct { + file, text string + moves bool + }{ + {"other.md", "changed\n", false}, // outside the list + {"d/d.go", "package d\n\n// D.\n", false}, // a package c's tests do not reach + {"docs/one.md", "changed\n", true}, // a listed file + {"docs/two.md", "new\n", true}, // a new file the glob matches + {"gone", "here\n", true}, // a path whose absence was read + {"c/note.txt", "changed\n", true}, // the package's own file that is not Go + {"c/x_integration_test.go", "//go:build integration\n\npackage c\n", true}, // a Go file its build tags leave out (go list's IgnoredGoFiles) + } { + edit(t, tree, step.file, step.text) + after := digests(t, m, "unit") + if moved := after.Packages[c] != before.Packages[c]; moved != step.moves { + t.Errorf("a change to %s: c's digest moved %v, want %v", step.file, moved, step.moves) + } + if after.Packages[a] != plain.Packages[a] { + t.Errorf("a change to %s moved a's digest", step.file) + } + before = after } // Issue 890/AC3: a package whose directory carries the needs marker is named with its needs at the integration level, diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index f612dbb5..e9d2f57f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -42,11 +42,13 @@ type FetchResult struct { // head onto it, the paths the merge changes, and the tree of each module directory (a directory holding a // module.json) in the merge. A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the // files, and the workspace is left at the trunk commit. - Base string `json:"base,omitempty"` - Merge string `json:"merge,omitempty"` - Conflicts []string `json:"conflicts,omitempty"` - Changed []string `json:"changed,omitempty"` - Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"` + Base string `json:"base,omitempty"` + Merge string `json:"merge,omitempty"` + Conflicts []string `json:"conflicts,omitempty"` + Changed []string `json:"changed,omitempty"` + // Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge. + Programs *[]buildstep.Program `json:"programs,omitempty"` + Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"` // Beside are the sibling repositories placed as plain directories under the workspace's beside directory, which // build_go_test hands to the tests as MESH_CHECK_BESIDE (ADR 0484 clause 4). Beside []Sibling `json:"beside,omitempty"` diff --git a/modules/mesh-build-go/cmd/mesh-build-go/lint.go b/modules/mesh-build-go/cmd/mesh-build-go/lint.go index 6b3a7d96..32366378 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/lint.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/lint.go @@ -132,6 +132,9 @@ func (m *Module) lint(ctx context.Context, run forgekey.Runner, tree, d string, // lintAll lints each directory, up to m.parallel() at once like the level's tests; the failures join in directory order // and the error is the first directory's, so the answer does not depend on which ended first (issue 1930). func (m *Module) lintAll(ctx context.Context, run forgekey.Runner, tree string, dirs []string, env func(d string) []string) ([]Failure, error) { + // The ask's phases say it (issue 1890): lint of the directories, then their tests. + buildstep.Phase(ctx, fmt.Sprintf("unit: lint of %d directories", len(dirs))) + defer buildstep.Phase(ctx, fmt.Sprintf("unit: tests of %d directories", len(dirs))) type answer struct { failed []Failure err error diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview.go b/modules/mesh-build-go/cmd/mesh-build-go/preview.go index 067dcfff..1636678e 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/preview.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview.go @@ -41,6 +41,9 @@ func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base } res.Base = mr.Base res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed + if mr.Merge != "" { + res.Programs = &mr.Programs + } if res.Merge == "" { return nil } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go b/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go index 55877c84..6df51b81 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go @@ -77,7 +77,7 @@ func TestFetchWithABaseMergesAndAnswersThePreview(t *testing.T) { t.Fatal(err) } if res.Base != baseSHA || res.Merge != mergeSHA || res.Commit != commitSHA || len(res.Conflicts) != 0 || - strings.Join(res.Changed, ",") != "a.go,modules/m/main.go" { + strings.Join(res.Changed, ",") != "a.go,modules/m/main.go" || res.Programs == nil || len(*res.Programs) != 0 { // 1577: a merge says it found no program t.Errorf("result = %s", st.Result) } mt, ok := res.Trees["modules/m"] diff --git a/modules/mesh-build-go/cmd/mesh-build-go/reads-outside b/modules/mesh-build-go/cmd/mesh-build-go/reads-outside index c1b869c0..ea239b40 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/reads-outside +++ b/modules/mesh-build-go/cmd/mesh-build-go/reads-outside @@ -1 +1,7 @@ -These tests read other catalogue modules' files, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the manifests through mesh-sdk's validator (the predefined keys), gitea's and mesh-build-node's. +predefined.json +modules/gitea +modules/mesh-build-node diff --git a/modules/mesh-build-go/files/settings.json b/modules/mesh-build-go/files/settings.json index 3175fe39..56bb28d4 100644 --- a/modules/mesh-build-go/files/settings.json +++ b/modules/mesh-build-go/files/settings.json @@ -1,3 +1,5 @@ { - "capacity": "shanks=3,ace=2,g14=0,novox=0" + "capacity": "shanks=3,ace=2,g14=0,novox=0", + "cores-per-step": 6, + "build-hours": "" } diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index c38009eb..fa959a83 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026191742-47a66e3c0dc8 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026233910-a2759ec92ec9 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index c094e038..ffedfa40 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026191742-47a66e3c0dc8 h1:6OaUWIjcpXOtnu70zeqH0VEaF4oGykrxGpFkaFOzCmY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026191742-47a66e3c0dc8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026233910-a2759ec92ec9 h1:MS7i4NVNQHMAMqRRFMdzbto31BHZY+BuEZY1ql3p4Rs= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026233910-a2759ec92ec9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 2f566cbf..7526d1a0 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -29,6 +29,18 @@ "description": "How many asks of the Go build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.", "pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*", "default": "shanks=3,ace=2,g14=0,novox=0" + }, + "build-hours": { + "type": "string", + "description": "When each node may take this module's build asks, as node=HH:MM-HH:MM pairs in the node's local time, separated by commas, such as g14=18:00-06:00 (an end before its start crosses midnight). Outside its window a node takes no new step and finishes the ones it runs; a node not named, or an empty value, builds at any hour. The operator, 2026-10-26: g14 builds only outside office hours.", + "pattern": "|[a-z0-9_-]+=([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9](,[a-z0-9_-]+=([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9])*", + "default": "" + }, + "cores-per-step": { + "type": "number", + "description": "How many processors one Go step of this module needs (a full check of a large repository takes what it is given: three at about 4 cores each ran past the 30-minute step bound on 2026-10-26). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", + "minimum": 0.1, + "default": 6 } }, "tools": [ @@ -107,7 +119,7 @@ "path": "${dir:state}/settings.json", "mode": "0600", "owner": "mesh-build-go", - "takes-config": ["capacity"], + "takes-config": ["capacity", "cores-per-step", "build-hours"], "merge": "json", "template": "files/settings.json" }, diff --git a/modules/mesh-build-image/README.md b/modules/mesh-build-image/README.md index a6c3b62e..e27eaa30 100644 --- a/modules/mesh-build-image/README.md +++ b/modules/mesh-build-image/README.md @@ -24,8 +24,9 @@ The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can re - `build_image_fetch` fetches a commit over SSH into a workspace (a base commit makes the merge preview, as the other build modules' fetch). Long: answers a step id and ends on its key in `steps`. -- `build_image_build` builds one image artifact: `buildah bud` with the Dockerfile of the module's directory as file - and context, `--target`, and the images `build.on` pins as `--build-arg`. Answers the image's digest. With +- `build_image_build` builds one image artifact: `buildah bud` with the Dockerfile `from` names (in the module's + directory or a subdirectory of it) as file and the module's own directory as context, so a COPY names paths from the + module's root (issue 1703), `--target`, and the images `build.on` pins as `--build-arg`. Answers the image's digest. With `publish: true` (a merged trunk commit only) the module's own process then pushes it to the registry under `/:` (the first 12 hex digits of the digest unless the call gives one), waiting out a registry held still, and the key carries the digest the registry serves and the reference pinned by it. Without diff --git a/modules/mesh-build-image/cmd/mesh-build-image/asks.go b/modules/mesh-build-image/cmd/mesh-build-image/asks.go index 45e03207..ad9899bf 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/asks.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/asks.go @@ -40,9 +40,17 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep // The capacity's own change wakes the puller (issue 1887): the runtime's watch of the config value as mesh-config // resolved it; the settings file is read only when that watch cannot start. A test's Capacity is its own. var watch func(string, func(stdio.ProvisionChange) error) error + // And the CPU bounds it (issue 1806's follow-up): the configured capacity is an upper bound, the node's build slice + // gives floor(its quota's cores / the module's cores-per-step) at once, read live; a test judges by its own Capacity. + // And the node's build hours (the operator, 2026-10-26: g14 builds only from 18:00 to 06:00): outside them it takes + // nothing new. + var cores func() float64 + var hours func() (*buildstep.BuildWindow, string) if capacity == nil { + hours = buildstep.NodeBuildHours(m.Settings, m.Node, m.Say) capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say) watch = buildstep.WatchConfigServed + cores = buildstep.NodeCoresPerStep(m.Settings, buildstep.DefaultCoresPerStep[delivery.AskImage], m.Say) } ask := m.askSteps(steps) var load func(string) ([]byte, error) @@ -50,7 +58,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep load = func(key string) ([]byte, error) { return m.ReadPauses.Get(key) } } return &buildstep.Puller{Source: source, Language: delivery.AskImage, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, - Capacity: capacity, WatchConfig: watch, Logf: m.Say, + Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Logf: m.Say, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { if err := m.Check(ctx); err != nil { return nil, err diff --git a/modules/mesh-build-image/cmd/mesh-build-image/build.go b/modules/mesh-build-image/cmd/mesh-build-image/build.go index c3e1d0e4..323cf3a5 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/build.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/build.go @@ -55,18 +55,18 @@ func (m *Module) buildTool() StepTool { } // unsupportedImage says why this module does not build the artifact, or "" when it does. It is the one place that -// knows which image forms are out: ADR 0487 §1 takes the Dockerfile's directory as the build context and the base +// knows which image forms are out: ADR 0487 §1 builds from the module's own directory (the build context) and the base // from the Dockerfile or an image pinned in `build.on`, so a context that is another repository and a base that is // another module's artifact are not built (the lead's decision for R4c-9, which retires them with build-agent at R4g). func unsupportedImage(manifest catalogue.Manifest, a catalogue.Artifact) string { if a.Context != nil { - return fmt.Sprintf("the image %s of %s builds with the repository %s as its context; mesh-build-image takes the Dockerfile's directory as the build context (ADR 0487 §1) and does not build a whole repository", + return fmt.Sprintf("the image %s of %s builds with the repository %s as its context; mesh-build-image takes the module's own directory as the build context (ADR 0487 §1) and does not build another repository", a.Name, manifest.Module, a.Context.Repository) } if manifest.Build != nil { for _, on := range manifest.Build.On { if on.Module != "" { - return fmt.Sprintf("the image %s of %s stands on the artifact %s of the module %s through the build argument %s; mesh-build-image builds only from the Dockerfile's directory and an image pinned in build.on (ADR 0487 §1)", + return fmt.Sprintf("the image %s of %s stands on the artifact %s of the module %s through the build argument %s; mesh-build-image builds only from the module's own directory and an image pinned in build.on (ADR 0487 §1)", a.Name, manifest.Module, on.Artifact, on.Module, on.Arg) } } @@ -183,8 +183,14 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { return m.push(ctx, workspace, manifest.Module, local, res, tag) } -// dockerfileOf is the Dockerfile an artifact's `from` names and the build context: the directory it lies in. It must -// be a regular file inside the module's own directory (checks/imageplace holds the catalogue to the same). +// dockerfileOf is the Dockerfile an artifact's `from` names and the build context: the module's own directory, the +// one its manifest lies in, whichever subdirectory the Dockerfile sits in. `from` is a path relative to that +// directory, and so is every COPY of the Dockerfile: a recipe in a subdirectory (photos' admin-client/Dockerfile +// copies admin-client/... and client-server/serve.mjs) builds as build-agent built it, and as the fingerprint reads +// the module's whole directory as its build source. Taking the Dockerfile's own directory instead failed every such +// COPY with `copier: stat ... no such file or directory` (issue 1703, 2026-10-26). The context never leaves the module's +// directory, so a module's build reads no other module's files. The Dockerfile must be a regular file inside that +// directory (checks/imageplace holds the catalogue to the same). func dockerfileOf(moduleDir, from string) (dockerfile, contextDir string, err error) { clean := filepath.Clean(filepath.FromSlash(strings.TrimSpace(from))) if from == "" || filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { @@ -195,7 +201,7 @@ func dockerfileOf(moduleDir, from string) (dockerfile, contextDir string, err er if err != nil || !info.Mode().IsRegular() { return "", "", fmt.Errorf("%q is not a file in the module's directory", from) } - return dockerfile, filepath.Dir(dockerfile), nil + return dockerfile, moduleDir, nil } // localName names the image in buildah's store: one name for each artifact of a workspace. diff --git a/modules/mesh-build-image/cmd/mesh-build-image/build_test.go b/modules/mesh-build-image/cmd/mesh-build-image/build_test.go index 3bb92b3a..a8c866ac 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/build_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/build_test.go @@ -94,7 +94,7 @@ func TestBuildRunsBuildahRootlessAndAnswersTheDigest(t *testing.T) { } } if !strings.HasSuffix(bud[0].line, " "+dir) { - t.Errorf("the context is not the Dockerfile's directory:\n%s", bud[0].line) + t.Errorf("the context is not the module's directory:\n%s", bud[0].line) } for _, c := range f.calls { if c.name != "buildah" { @@ -122,6 +122,43 @@ func TestBuildRunsBuildahRootlessAndAnswersTheDigest(t *testing.T) { } } +// Issue 1703, 2026-10-26: a Dockerfile in a subdirectory of the module builds with the module's directory as its +// context, so its COPY of a path from the module's root (photos' admin-client/Dockerfile copies admin-client/... and +// client-server/serve.mjs) finds it; the Dockerfile's own directory as the context failed with `copier: stat +// "/admin-client/package.json": no such file or directory`. +func TestBuildContextIsTheModulesDirectoryForADockerfileInASubdirectory(t *testing.T) { + m, f, states := setup(t, func(s string) string { + return strings.Replace(s, `"from": "Dockerfile"`, `"from": "admin-client/Dockerfile"`, 1) + }) + dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo") + for name, body := range map[string]string{ + "admin-client/Dockerfile": "ARG DEMO_BASE\nFROM ${DEMO_BASE} AS final\nCOPY admin-client/package.json client-server/serve.mjs /\n", + "admin-client/package.json": "{}\n", + "client-server/serve.mjs": "\n", + } { + if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + _, st := runBuild(t, m, states, nil) + if st.State != delivery.StepPassed { + t.Fatalf("step = %+v", st) + } + bud := f.called("bud") + if len(bud) != 1 { + t.Fatalf("buildah bud ran %d times", len(bud)) + } + if !strings.Contains(bud[0].line, "--file "+dir+"/admin-client/Dockerfile ") { + t.Errorf("the Dockerfile is not the subdirectory's:\n%s", bud[0].line) + } + if !strings.HasSuffix(bud[0].line, " "+dir) { + t.Errorf("the context is not the module's directory, so admin-client/ and client-server/ are out of reach:\n%s", bud[0].line) + } +} + // 1703c, issue 1703: a failing Dockerfile step fails the step, its reason saying buildah's error from stderr, never the // last STEP line buildah writes on stdout; nothing is pushed. func TestBuildFailsWithBuildahsError(t *testing.T) { @@ -178,7 +215,7 @@ func TestBuildRefusesAnImageWithARepositoryContextOrAModuleBase(t *testing.T) { }{ "context": {func(s string) string { return strings.Replace(s, `"target": "final"`, `"target": "final", "context": {"seat": "git", "repository": "novox/mesh-controller"}`, 1) - }, []string{"novox/mesh-controller", "Dockerfile's directory", "ADR 0487"}}, + }, []string{"novox/mesh-controller", "module's own directory", "ADR 0487"}}, "module base": {func(s string) string { return strings.Replace(s, `"image": "docker.io/library/busybox@sha256:0000000000000000000000000000000000000000000000000000000000000001"`, `"module": "mesh-sdk", "artifact": "lib"`, 1) }, []string{"artifact lib of the module mesh-sdk", "DEMO_BASE"}}, diff --git a/modules/mesh-build-image/cmd/mesh-build-image/cpu.go b/modules/mesh-build-image/cmd/mesh-build-image/cpu.go index 126b319d..91f1d314 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/cpu.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/cpu.go @@ -15,6 +15,8 @@ import ( func (m *Module) cpuLimits() buildcpu.Limits { slots := 1 switch { + case m.Held != nil: + slots = m.Held.CapacityNow() // the capacity the puller judges by, bounded by the node's build slice case m.Capacity != nil: slots = m.Capacity() case m.Settings != "": @@ -32,5 +34,9 @@ func (m *Module) threadEnv() []string { // cpuStatus is what the status tool says of it: the limits a step is given, and what the kernel holds this runtime // to (its unit and its slice), read from its cgroup rather than assumed. func (m *Module) cpuStatus() map[string]any { - return map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()} + out := map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()} + if m.Held != nil { + out["capacity"] = m.Held.CapacityDetail() // the capacity judged by and why (issue 1806's follow-up) + } + return out } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/fetch.go b/modules/mesh-build-image/cmd/mesh-build-image/fetch.go index e07bbfda..b139be23 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/fetch.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/fetch.go @@ -31,13 +31,15 @@ const ForgeAlias = "mesh-forge" // files in conflict, the paths the merge changes and the tree of each module directory). A head that does not // merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is left at the trunk. type FetchResult struct { - Workspace string `json:"workspace"` - Commit string `json:"commit"` - Base string `json:"base,omitempty"` - Merge string `json:"merge,omitempty"` - Conflicts []string `json:"conflicts,omitempty"` - Changed []string `json:"changed,omitempty"` - Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"` + Workspace string `json:"workspace"` + Commit string `json:"commit"` + Base string `json:"base,omitempty"` + Merge string `json:"merge,omitempty"` + Conflicts []string `json:"conflicts,omitempty"` + Changed []string `json:"changed,omitempty"` + // Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge. + Programs *[]buildstep.Program `json:"programs,omitempty"` + Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"` } func (m *Module) fetchTool() StepTool { @@ -127,6 +129,9 @@ func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base } res.Base = mr.Base res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed + if mr.Merge != "" { + res.Programs = &mr.Programs + } if res.Merge == "" { return nil } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/reads-outside b/modules/mesh-build-image/cmd/mesh-build-image/reads-outside index c1b869c0..b79eb772 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/reads-outside +++ b/modules/mesh-build-image/cmd/mesh-build-image/reads-outside @@ -1 +1,10 @@ -These tests read other catalogue modules' files, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the manifests through mesh-sdk's validator (the predefined keys), with the modules they compose with. +predefined.json +modules/gitea +modules/mesh-build-go +modules/mesh-build-node +modules/buildah +modules/docker diff --git a/modules/mesh-build-image/files/settings.json b/modules/mesh-build-image/files/settings.json index 3175fe39..73ee4fa8 100644 --- a/modules/mesh-build-image/files/settings.json +++ b/modules/mesh-build-image/files/settings.json @@ -1,3 +1,5 @@ { - "capacity": "shanks=3,ace=2,g14=0,novox=0" + "capacity": "shanks=3,ace=2,g14=0,novox=0", + "cores-per-step": 2, + "build-hours": "" } diff --git a/modules/mesh-build-image/go.mod b/modules/mesh-build-image/go.mod index 52658707..aa7efa34 100644 --- a/modules/mesh-build-image/go.mod +++ b/modules/mesh-build-image/go.mod @@ -3,7 +3,7 @@ module meshbuildimage go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026202658-805b4d9a3536 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-image/go.sum b/modules/mesh-build-image/go.sum index ba65afa3..c6d0c7bf 100644 --- a/modules/mesh-build-image/go.sum +++ b/modules/mesh-build-image/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026202658-805b4d9a3536 h1:+SoDdDXF7Tgj38WP4kRpUSps6DJVl6P+w5u/KfdtwnQ= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026202658-805b4d9a3536/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index ae9c5162..c83b9d88 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -24,6 +24,18 @@ "description": "How many asks of the image build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.", "pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*", "default": "shanks=3,ace=2,g14=0,novox=0" + }, + "build-hours": { + "type": "string", + "description": "When each node may take this module's build asks, as node=HH:MM-HH:MM pairs in the node's local time, separated by commas, such as g14=18:00-06:00 (an end before its start crosses midnight). Outside its window a node takes no new step and finishes the ones it runs; a node not named, or an empty value, builds at any hour. The operator, 2026-10-26: g14 builds only outside office hours.", + "pattern": "|[a-z0-9_-]+=([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9](,[a-z0-9_-]+=([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9])*", + "default": "" + }, + "cores-per-step": { + "type": "number", + "description": "How many processors one image step of this module needs (buildah, whose RUN lines are mostly fetches). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", + "minimum": 0.1, + "default": 2 } }, "tools": [ @@ -78,7 +90,7 @@ "path": "${dir:state}/settings.json", "mode": "0600", "owner": "mesh-build-image", - "takes-config": ["capacity"], + "takes-config": ["capacity", "cores-per-step", "build-hours"], "merge": "json", "template": "files/settings.json" }, diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks.go b/modules/mesh-build-node/cmd/mesh-build-node/asks.go index 0e1b845d..f2353133 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks.go @@ -48,9 +48,17 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep // The capacity's own change wakes the puller (issue 1887): the runtime's watch of the config value as mesh-config // resolved it; the settings file is read only when that watch cannot start. A test's Capacity is its own. var watch func(string, func(stdio.ProvisionChange) error) error + // And the CPU bounds it (issue 1806's follow-up): the configured capacity is an upper bound, the node's build slice + // gives floor(its quota's cores / the module's cores-per-step) at once, read live; a test judges by its own Capacity. + // And the node's build hours (the operator, 2026-10-26: g14 builds only from 18:00 to 06:00): outside them it takes + // nothing new. + var cores func() float64 + var hours func() (*buildstep.BuildWindow, string) if capacity == nil { + hours = buildstep.NodeBuildHours(m.Settings, m.Node, m.Say) capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say) watch = buildstep.WatchConfigServed + cores = buildstep.NodeCoresPerStep(m.Settings, buildstep.DefaultCoresPerStep[delivery.AskNode], m.Say) } ask := m.askSteps(steps) var load func(string) ([]byte, error) @@ -58,7 +66,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep load = func(key string) ([]byte, error) { return m.ReadPauses.Get(key) } } return &buildstep.Puller{Source: source, Language: delivery.AskNode, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load, - Capacity: capacity, WatchConfig: watch, Logf: m.Say, + Capacity: capacity, WatchConfig: watch, CoresPerStep: cores, BuildHours: hours, Logf: m.Say, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { if err := m.Prepare(ctx); err != nil { return nil, err diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go b/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go index 1985c863..8ae9a43d 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "fmt" "os" "strings" "testing" @@ -83,12 +84,28 @@ func TestManifestTakesTheBuildCapacity(t *testing.T) { raw, _ := os.ReadFile(moduleDir + "/module.json") var doc struct { Config map[string]struct { - Default string `json:"default"` + Default json.RawMessage `json:"default"` } `json:"config"` } _ = json.Unmarshal(raw, &doc) - if doc.Config[buildstep.CapacitySetting].Default != buildstep.DefaultCapacities { - t.Errorf("the capacity's default = %q", doc.Config[buildstep.CapacitySetting].Default) + // Issue 1806's follow-up: the cores a step needs is a setting of its own, in the settings file, with the SDK's default. + if cores := string(doc.Config[buildstep.CoresPerStepSetting].Default); cores != fmt.Sprint(buildstep.DefaultCoresPerStep[delivery.AskNode]) { + t.Errorf("the cores-per-step default = %s, want %g", cores, buildstep.DefaultCoresPerStep[delivery.AskNode]) + } + if got := buildstep.NodeCoresPerStep(moduleDir+"/files/settings.json", 0, nil)(); got != buildstep.DefaultCoresPerStep[delivery.AskNode] { + t.Errorf("the settings template holds cores-per-step %g", got) + } + // And the build hours (the operator, 2026-10-26): a setting of their own, empty by default (any hour), in the template. + if hours := string(doc.Config[buildstep.BuildHoursSetting].Default); hours != `""` { + t.Errorf("the build-hours default = %s, want empty", hours) + } + if w, fault := buildstep.NodeBuildHours(moduleDir+"/files/settings.json", "g14", t.Errorf)(); w != nil || fault != "" { + t.Errorf("the settings template holds build hours %v (%s)", w, fault) + } + var capacity string + _ = json.Unmarshal(doc.Config[buildstep.CapacitySetting].Default, &capacity) + if capacity != buildstep.DefaultCapacities { + t.Errorf("the capacity's default = %q", capacity) } if resource(t, m, "settings")["path"] != "${dir:state}/settings.json" { t.Error("no settings file") diff --git a/modules/mesh-build-node/cmd/mesh-build-node/cpu.go b/modules/mesh-build-node/cmd/mesh-build-node/cpu.go index b40ceda4..5936b615 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/cpu.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/cpu.go @@ -15,6 +15,8 @@ import ( func (m *Module) cpuLimits() buildcpu.Limits { slots := 1 switch { + case m.Held != nil: + slots = m.Held.CapacityNow() // the capacity the puller judges by, bounded by the node's build slice case m.Capacity != nil: slots = m.Capacity() case m.Settings != "": @@ -32,5 +34,9 @@ func (m *Module) threadEnv() []string { // cpuStatus is what the status tool says of it: the limits a step is given, and what the kernel holds this runtime // to (its unit and its slice), read from its cgroup rather than assumed. func (m *Module) cpuStatus() map[string]any { - return map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()} + out := map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()} + if m.Held != nil { + out["capacity"] = m.Held.CapacityDetail() // the capacity judged by and why (issue 1806's follow-up) + } + return out } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go index 5f712e16..31307a46 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go @@ -51,11 +51,13 @@ type FetchResult struct { // commit, the merge commit of the head onto it, the paths the merge changes and the tree of each module directory. // A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is // left at the trunk commit with nothing installed. - Base string `json:"base,omitempty"` - Merge string `json:"merge,omitempty"` - Conflicts []string `json:"conflicts,omitempty"` - Changed []string `json:"changed,omitempty"` - Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"` + Base string `json:"base,omitempty"` + Merge string `json:"merge,omitempty"` + Conflicts []string `json:"conflicts,omitempty"` + Changed []string `json:"changed,omitempty"` + // Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge. + Programs *[]buildstep.Program `json:"programs,omitempty"` + Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"` // Group is the group's package the fetch packed and served to the TypeScript members (the call's `package`): // "@ ". Group string `json:"group,omitempty"` diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go index 01f2c376..f993a2b7 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go @@ -175,7 +175,7 @@ func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *test files["modules/m/package-lock.json"] = `{}` m, f := fetchMachine(t, files, map[string]string{"@novox/other/index.js": "o", "lodash/index.js": "l"}) m.Toolchain.Registry = func() (Registry, error) { - return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "pw"}, nil + return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "s3cret-pw!"}, nil } var npmrc string inner := f.effect @@ -195,7 +195,8 @@ func TestAnotherNovoxPackageIsInstalledFromTheRegistryThroughAUserconfig(t *test if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/") || !strings.Contains(npmrc, ":_auth=") { t.Errorf("npm ci was given the config %q", npmrc) } - if strings.Contains(strings.Join(npmCalls(f)[0].args, " "), "pw") { + // The password holds characters a workspace's random name never does, so a name like "mvpwby…" is no false match. + if strings.Contains(strings.Join(npmCalls(f)[0].args, " "), "s3cret-pw!") { t.Errorf("the credential is on npm's command line") } r := out.(FetchResult) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/preview.go b/modules/mesh-build-node/cmd/mesh-build-node/preview.go index 6d641b9b..48011ed7 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/preview.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/preview.go @@ -27,6 +27,9 @@ func (m *Module) merge(ctx context.Context, run forgekey.Runner, res *FetchResul } res.Base = mr.Base res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed + if mr.Merge != "" { + res.Programs = &mr.Programs + } if res.Merge == "" { return nil } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/reads-outside b/modules/mesh-build-node/cmd/mesh-build-node/reads-outside index c1b869c0..26937843 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/reads-outside +++ b/modules/mesh-build-node/cmd/mesh-build-node/reads-outside @@ -1 +1,5 @@ -These tests read other catalogue modules' files, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# its manifest through mesh-sdk's validator, which reads the root's predefined keys. +predefined.json diff --git a/modules/mesh-build-node/files/settings.json b/modules/mesh-build-node/files/settings.json index 3175fe39..73ee4fa8 100644 --- a/modules/mesh-build-node/files/settings.json +++ b/modules/mesh-build-node/files/settings.json @@ -1,3 +1,5 @@ { - "capacity": "shanks=3,ace=2,g14=0,novox=0" + "capacity": "shanks=3,ace=2,g14=0,novox=0", + "cores-per-step": 2, + "build-hours": "" } diff --git a/modules/mesh-build-node/go.mod b/modules/mesh-build-node/go.mod index 64efec43..3575f370 100644 --- a/modules/mesh-build-node/go.mod +++ b/modules/mesh-build-node/go.mod @@ -3,7 +3,7 @@ module meshbuildnode go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026191742-47a66e3c0dc8 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026233910-a2759ec92ec9 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-node/go.sum b/modules/mesh-build-node/go.sum index 1765062f..32f3a421 100644 --- a/modules/mesh-build-node/go.sum +++ b/modules/mesh-build-node/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026191742-47a66e3c0dc8 h1:6OaUWIjcpXOtnu70zeqH0VEaF4oGykrxGpFkaFOzCmY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026191742-47a66e3c0dc8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026233910-a2759ec92ec9 h1:MS7i4NVNQHMAMqRRFMdzbto31BHZY+BuEZY1ql3p4Rs= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026233910-a2759ec92ec9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index 701419b2..baf8b291 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -25,6 +25,18 @@ "description": "How many asks of the Node build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.", "pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*", "default": "shanks=3,ace=2,g14=0,novox=0" + }, + "build-hours": { + "type": "string", + "description": "When each node may take this module's build asks, as node=HH:MM-HH:MM pairs in the node's local time, separated by commas, such as g14=18:00-06:00 (an end before its start crosses midnight). Outside its window a node takes no new step and finishes the ones it runs; a node not named, or an empty value, builds at any hour. The operator, 2026-10-26: g14 builds only outside office hours.", + "pattern": "|[a-z0-9_-]+=([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9](,[a-z0-9_-]+=([01][0-9]|2[0-3]):[0-5][0-9]-([01][0-9]|2[0-3]):[0-5][0-9])*", + "default": "" + }, + "cores-per-step": { + "type": "number", + "description": "How many processors one Node step of this module needs (tsc, esbuild and a test runner; a median step takes 15 s). The capacity above is an upper bound: on each node the module runs at most floor(the cores of its build slice's CPU quota / this) steps at once, the quota read live from its cgroup (issue 1806), so a lowered build CPU share never starves its steps. Above 0.", + "minimum": 0.1, + "default": 2 } }, "tools": [ @@ -100,7 +112,7 @@ "path": "${dir:state}/settings.json", "mode": "0600", "owner": "mesh-build-node", - "takes-config": ["capacity"], + "takes-config": ["capacity", "cores-per-step", "build-hours"], "merge": "json", "template": "files/settings.json" }, diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index 1334b097..63428e01 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -115,7 +115,7 @@ func resolving(c *Config) { } return nil } - for _, read := range []string{AssignmentsRead, DeclaredConfigRead, PredefinedRead} { + for _, read := range []string{AssignmentsRead, PlacementRead, DeclaredConfigRead, PredefinedRead} { go func(read string) { for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { time.Sleep(wait) diff --git a/modules/mesh-config/cmd/mesh-config/pulled_in_test.go b/modules/mesh-config/cmd/mesh-config/pulled_in_test.go index e4120e2c..dbf9ebfb 100644 --- a/modules/mesh-config/cmd/mesh-config/pulled_in_test.go +++ b/modules/mesh-config/cmd/mesh-config/pulled_in_test.go @@ -21,4 +21,16 @@ func TestAModulePulledInHasItsValuesResolvedOnItsNode(t *testing.T) { if r, _ := c.Reconcile(); len(r.Moved) != 4 || !r.Moved[0].Gone { t.Errorf("a module no longer pulled in moved %+v", r.Moved) } + + // 1126 (ADR 0549 §3, §5): a module the catalogue's placement names on a node resolves there as an assigned one does, + // while the two sources stand; it goes when neither names it. + m.mirror(t, PlacementRead, "placement.ace", `{"v":1,"node":"ace","modules":[{"module":"dunst"}]}`) + if r, err := c.Reconcile(); err != nil || len(r.Moved) != 4 { + t.Fatalf("a placed module's values moved %+v (%v)", r.Moved, err) + } + expect(t, m, "ace", "10", SourceDefault) + _ = m.Delete(PlacementRead, "placement.ace") + if r, _ := c.Reconcile(); len(r.Moved) != 4 || !r.Moved[0].Gone { + t.Errorf("a module no longer placed moved %+v", r.Moved) + } } diff --git a/modules/mesh-config/cmd/mesh-config/resolve.go b/modules/mesh-config/cmd/mesh-config/resolve.go index bc21ec61..ccf35335 100644 --- a/modules/mesh-config/cmd/mesh-config/resolve.go +++ b/modules/mesh-config/cmd/mesh-config/resolve.go @@ -20,7 +20,10 @@ const ( PredefinedState = "predefined" ProposalsState = "proposals" - AssignmentsRead = "mesh-controller.assignments" + AssignmentsRead = "mesh-controller.assignments" + // PlacementRead is the indexer's placement, read from the catalogue at each merge (novox/hq ADR 0549 §2, §3): each + // node's key `placement.` with the modules placed there. + PlacementRead = "mesh-delivery-indexer.placement" DeclaredConfigRead = "mesh-controller.declared-config" PredefinedRead = "mesh-controller.predefined" diff --git a/modules/mesh-config/cmd/mesh-config/sources.go b/modules/mesh-config/cmd/mesh-config/sources.go index df6292dc..34f8dddf 100644 --- a/modules/mesh-config/cmd/mesh-config/sources.go +++ b/modules/mesh-config/cmd/mesh-config/sources.go @@ -27,6 +27,38 @@ func (c *Config) sources() (Sources, error) { }); err != nil { return s, fmt.Errorf("the assignments cannot be read: %w", err) } + // And every module the catalogue's placement names on a node (ADR 0549 §3): until ADR 0547's third milestone both + // sources stand (ADR 0549 §5), so a pair either names is resolved, and none loses its values while the two differ. + // Placement that cannot be read yet (the indexer's state not on the bus) leaves the assignments alone, said. + standing := map[string]bool{} + for _, a := range s.Assignments { + standing[a.Node+"."+a.Module] = true + } + if err := c.each(PlacementRead, func(key string, raw []byte) error { + node, ok := strings.CutPrefix(key, "placement.") + if !ok { + return nil + } + var p struct { + Node string `json:"node"` + Modules []struct { + Module string `json:"module"` + } `json:"modules"` + } + if err := json.Unmarshal(raw, &p); err != nil || p.Node != node { + problems = append(problems, fmt.Sprintf("the placement %s is not one", key)) + return nil + } + for _, m := range p.Modules { + if m.Module != "" && !standing[node+"."+m.Module] { + standing[node+"."+m.Module] = true + s.Assignments = append(s.Assignments, Assignment{Node: node, Module: m.Module}) + } + } + return nil + }); err != nil { + problems = append(problems, fmt.Sprintf("the catalogue's placement cannot be read, so the assignments alone are resolved: %v", err)) + } if err := c.each(DeclaredConfigRead, func(key string, raw []byte) error { m, err := ParseModuleDeclarations(raw) if err != nil { diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json index f93497e1..d6b5bdd0 100644 --- a/modules/mesh-config/module.json +++ b/modules/mesh-config/module.json @@ -29,6 +29,7 @@ ], "reads": [ "mesh-controller.assignments", + "mesh-delivery-indexer.placement", "mesh-controller.declared-config", "mesh-controller.predefined" ], diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go index 3c385094..1e08b31c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go @@ -148,8 +148,32 @@ func touchesSaid(d *Delivery) string { return strings.Join(d.Check.Modules, ", ") } +// checkClause is the check's state as the ask's summary says it after the pull request's or the group's name: sep and +// "passed its check", or ": its check has no verdict yet: it is running (since 21:05 UTC)" and the like. +func checkClause(c checkSaid, sep string) string { + if c.passed() { + return sep + c.Words + } + return ": " + c.Words +} + +// approveCaveat is what an Approve does beyond the merge while the check has not passed: it merges before the check ends, +// or although it failed (the check gates no merge, check-gates-merge false). Empty once it passed. +func approveCaveat(c checkSaid) string { + switch c.Key { + case checkSaidPass: + return "" + case checkSaidFail: + return " although its check failed" + } + return " now, before its check ends" +} + // composeMergeAsk is the ask for a delivery, or for a group's members together. -func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery, heads map[string]string, now time.Time) asks.Ask { +// check is the state of the check at the heads it covers (checkAt): the ask says "passed its check" only when it did, and +// otherwise that Approve merges before the check ends, or although it failed (issue 1733). +func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery, heads map[string]string, check checkSaid, + now time.Time) asks.Ask { var a asks.Ask a.ID, a.Who, a.Expires = id, asks.Operator, now.Add(AskLasts) a.OnExpiry = "nothing is merged; the ask is made again" @@ -170,8 +194,8 @@ func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery, } if group == "" { a.Headline = phone("Merge into main: "+pullSaid(d)+" "+d.Title, asks.HeadlineLength) - summary := fmt.Sprintf("%s %s passed its checks. Approve merges it into main at %s; Decline leaves it unmerged.", - pullSaid(d), d.Title, shortOf(d.Commit, 12)) + summary := fmt.Sprintf("%s %s%s. Approve merges it into main at %s%s; Decline leaves it unmerged.", + pullSaid(d), d.Title, checkClause(check, " "), shortOf(d.Commit, 12), approveCaveat(check)) if unreviewed != "" { summary = strings.ToUpper(unreviewed[:1]) + unreviewed[1:] + ". " + summary } @@ -187,8 +211,12 @@ func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery, names = append(names, pullSaid(m)) } a.Headline = phone("Merge into main: group "+group+" ("+strings.Join(names, ", ")+")", asks.HeadlineLength) - summary := fmt.Sprintf("The group %s, %d pull requests, passed its checks. Approve merges all of them into main in one pass, at the heads named; Decline leaves them unmerged.", - group, len(members)) + caveat := approveCaveat(check) + if caveat != "" { + caveat = "," + caveat + } + summary := fmt.Sprintf("The group %s, %d pull requests%s. Approve merges all of them into main in one pass, at the heads named%s; Decline leaves them unmerged.", + group, len(members), checkClause(check, ", "), caveat) if unreviewed != "" { summary = strings.ToUpper(unreviewed[:1]) + unreviewed[1:] + ". " + summary } @@ -299,6 +327,7 @@ func (h *Holder) headsHold(d *Delivery, c *ReadyRun) bool { // §3), and says so once on each pull request it covered (ADR 0421 §6). An ask for a head a newer one replaced ends too. // A Decline stands, and is not ended. Under the lock; the cancels run with the look's asks. func (h *Holder) endStaleAsks() { + h.endStaleSettles() for _, d := range h.deliveries { r := d.Ready if r == nil || r.Merged != nil || d.MergedAs != "" || r.Declined != nil || (r.AskID == "" && r.Approved == nil) || len(r.Heads) == 0 { @@ -309,6 +338,10 @@ func (h *Holder) endStaleAsks() { h.cancels = append(h.cancels, h.dropAsk(r.AskID, "its head "+shortOf(d.Commit, 12)+" is no longer the pull request's head, or the pull request closed")) case !h.headsHold(d, r): h.cancels = append(h.cancels, h.dropAsk(r.AskID, "a head it covers moved on: the heads it covers are no longer the group's heads")) + case r.Approved == nil && r.AskID != "" && r.AskCheck != "" && !h.checkGates() && h.checkMoved(d, r.AskCheck): + // The check gates no merge, so the ask was made before its verdict: once the check passes or fails (or a pass it + // named is gone), the ask is asked again in words that say so (issue 1733). Running and not started say the same. + h.cancels = append(h.cancels, h.withdrawAsk(r.AskID, "the check at its heads is now: "+h.checkAt(h.askMembers(d)).Words)) case r.Approved == nil && r.AskID != "" && !r.AskedAt.IsZero() && !h.Now().Before(r.AskedAt.Add(AskLasts)): // The ask lapsed at the router with nobody's answer, and nothing in the gate's inputs says so: it is taken away and // written again, which the teller asks afresh (ADR 0441 §1). @@ -317,6 +350,16 @@ func (h *Holder) endStaleAsks() { } } +// checkMoved is whether the check at the heads of d's ask reached a verdict its pending ask does not say: it passed or failed +// since, or the pass it said is gone. Under the lock. +func (h *Holder) checkMoved(d *Delivery, said string) bool { + now := h.checkAt(h.askMembers(d)).Key + if now == said { + return false + } + return now == checkSaidPass || now == checkSaidFail || said == checkSaidPass || said == checkSaidFail +} + // approval is what a delivery that may merge waits on for the operator: empty once its head is approved, and the ask // to make when none is made yet. Under the lock. func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) { @@ -367,19 +410,20 @@ func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) { asking = max(asking, m.Ready.Asking+1) } } - ask := composeMergeAsk(sdk.AskingID(id, asking), d, group, members, heads, now) + check := h.checkAt(members) + ask := composeMergeAsk(sdk.AskingID(id, asking), d, group, members, heads, check, now) taken := []*Delivery{} var reads []memberRead for _, m := range members { h.runOf(m, now) m.Ready.AskID, m.Ready.Ask, m.Ready.Heads, m.Ready.AskedAt, m.Ready.Asks = id, &ask, heads, now, max(m.Ready.Asks, r.Asks) - m.Ready.AskedKey, m.Ready.Asking = id, asking + m.Ready.AskedKey, m.Ready.Asking, m.Ready.AskCheck = id, asking, check.Key h.keep(m) taken = append(taken, m) reads = append(reads, readOf(m, m.Commit)) } - h.Logf("[mesh-delivery] %s passed its checks: the operator is asked to Approve its merge into main (%s, ask %d)", - d.ID, id, r.Asks) + h.Logf("[mesh-delivery] %s (%s): the operator is asked to Approve its merge into main (%s, ask %d)", + d.ID, check.Words, id, r.Asks) gid := "" if group != "" { gid = d.Group @@ -432,7 +476,7 @@ func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) { h.mu.Lock() defer h.mu.Unlock() undo() - h.Logf("[mesh-delivery] %s passed its checks and the operator is not asked: %s", d.ID, says) + h.Logf("[mesh-delivery] %s (%s): the operator is not asked: %s", d.ID, check.Words, says) h.lookAt(h.Now(), "an ask not made: the forge account cannot write a repository it covers") return } @@ -579,7 +623,7 @@ func (h *Holder) hearAsks() { return } h.mu.Lock() - keys := h.pendingAsks() + keys := append(h.pendingAsks(), h.settleKeys()...) h.mu.Unlock() for _, key := range keys { answer, err := h.Operator.Where(key) @@ -591,7 +635,12 @@ func (h *Holder) hearAsks() { continue } h.mu.Lock() - said := h.answered(key, *answer) + var said []func() + if h.isSettleKey(key) { + h.settled(key, *answer) // a person's settle ask (settle.go, novox/hq ADR 0556 clause 5) + } else { + said = h.answered(key, *answer) + } h.mu.Unlock() for _, s := range said { s() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 138c7589..9d690822 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -89,6 +89,9 @@ type Holder struct { Turns TurnStore turnsHeld map[string]sdk.Turn turnsLoaded bool + // reach is the delivery view's reach of each repository, by its key (reached.go, novox/hq ADR 0556): what `deliveries` + // and `show` derive whether a merge is on the machines from. Nothing here records it. + reach map[string]sdk.Reach // tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407). tried owedTries // forgeWanted is whether the forge's open pull requests are to be read at the next React: a check may have @@ -139,6 +142,7 @@ func (h *Holder) init() { h.unchecked = map[string]unchecked{} h.checker = map[string]checkerVerdict{} h.runningSince, h.turnsHeld = map[string]time.Time{}, map[string]sdk.Turn{} + h.reach = map[string]sdk.Reach{} h.started = h.Now() } @@ -757,7 +761,7 @@ func (h *Holder) pullMerged(p PullEvent, late bool) { } ts = append(ts, settle(d, h.factsAt(now))...) if d.State == Held && d.HeldWhy == "" { - d.HeldWhy = "merged without a passing check: only a person decides what follows" + d.HeldWhy = HeldUnchecked } h.moved(d, ts) h.closeGroupOf(d) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index be04c7f1..52d68da4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -2,8 +2,9 @@ // runtime launches. It owns the delivery — one commit in one repository, from its pull request's head to // every machine — and the delivery group above it: their states in one compiled table, kept in its own state // on the bus, every transition said as an event, noted on the commit and shown on the pull request through -// the forge's holder. It asks the Approve of a merge and merges once; it asks the controller nothing and derives nothing -// after the merge (ADR 0539 §17). stdout +// the forge's holder. It asks the Approve of a merge and merges once; it asks the controller nothing and records nothing +// after the merge (ADR 0539 §17): whether a merge is on the machines it derives when asked from the delivery view's reach +// (ADR 0556 clause 4), and a merge held without a passing check is settled on the operator's answer (clause 5). stdout // is the MCP channel; what this module says, it says on stderr. package main @@ -116,6 +117,21 @@ func run(h *Holder, l *listening) { time.Sleep(wait) } }() + // The delivery view's reach (novox/hq ADR 0556 clause 4): how far each repository reached, which `deliveries` and `show` + // derive whether a merge is on the machines from. Each change is kept; nothing is moved by it. + go func() { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.State(reachState).Watch("", func(c stdio.StateChange) error { + h.ReachChanged(c.Key, c.Value, c.Op == "delete") + return nil + }) + if err == nil { + return + } + logf("[mesh-delivery] not hearing the delivery view's reach yet (%v); asking again in %s", err, wait) + time.Sleep(wait) + } + }() // The checker's verdicts (issue 1302, ADR 0447 §2): the Approve on mesh/check is asked only on its verdict at exactly the // heads an ask covers, and a group's verdict is that one; each change asks for a look. go func() { @@ -290,7 +306,8 @@ func tools(h *Holder, l *listening) []stdio.Tool { list := []stdio.Tool{ {Name: seat + "deliveries", Description: "Every delivery not final, those that ended in the last day, and any stop `stalled` still lists, one line each: its id " + - "(owner/repository@commit), its state, its group, what it waits for and since when. Narrowed by state, " + + "(owner/repository@commit), its state, its group, what it waits for and since when, and for a merged one whether it is on " + + "the machines (machines, derived from the delivery view's reach, novox/hq ADR 0556). Narrowed by state, " + "repository or group.", Input: map[string]any{"type": "object", "properties": map[string]any{"state": str("one state"), "repository": str("owner/repository"), "group": str("a group's id (its branch name)"), @@ -301,7 +318,8 @@ func tools(h *Holder, l *listening) []stdio.Tool { }}, {Name: seat + "show", Description: "One delivery or group whole: every transition with when and why, and its group. " + - "What the merge moved is source.main; whether it reached the machines is the view's.", + "For a merged delivery, whether its merge is on the machines (machines), derived from the delivery view's reach of its " + + "repository (novox/hq ADR 0556), and else what holds it.", Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("a delivery's id, or a group's")}, "required": []string{"id"}}, Run: func(a map[string]any) (any, error) { return h.Show(strArg(a, "id")) }}, @@ -362,6 +380,30 @@ func tools(h *Holder, l *listening) []stdio.Tool { } return h.Close(strArg(a, "id"), strArg(a, "why")) }}, + {Name: seat + "settle", + Description: "Ask the operator to settle deliveries held because their merge reached the trunk without a passing check: " + + "one ask on the operator's channel for all of them, as an Approve is asked. The verb moves nothing; on the operator's " + + "answer each goes to merged, and whether it is on the machines is derived as for any merge (novox/hq ADR 0556). " + + "Refused, nothing asked, when any delivery named is not held or is held for another reason, naming why.", + Input: map[string]any{"type": "object", "additionalProperties": false, "required": []string{"ids", "why"}, + "properties": map[string]any{ + "ids": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "minItems": 1, + "description": "the held deliveries' ids (owner/repository@commit)"}, + "why": str("why they may be settled, shown to the operator")}}, + Run: func(a map[string]any) (any, error) { + var ids []string + switch v := a["ids"].(type) { + case []any: + for _, x := range v { + if s, ok := x.(string); ok { + ids = append(ids, s) + } + } + case []string: + ids = v + } + return h.Settle(ids, strArg(a, "why"), aPerson) + }}, {Name: "delivery_turns", Description: "Every merge turn (novox/hq ADR 0548 §3): per repository, the approved merges that wait to merge into it in the order of " + "their Approve, the check of another delivery each waits for and since when, whether the gate wrote the turn to its bucket " + diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 4ca96530..4e16ea29 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -70,8 +70,9 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { if len(m.Seats) != 0 { t.Fatal("the manifest declares a seat; mesh-delivery is the mesh's own") } - // The eight verbs the seat keeps (ADR 0539 §17): times, what-if, release, stop and retire-history went with the plans. - if want := []string{"deliveries", "show", "groups", "checks", "table", "stalled", "recheck", "close"}; !reflect.DeepEqual(m.Claims[0].Serves, want) { + // The eight verbs the seat keeps (ADR 0539 §17): times, what-if, release, stop and retire-history went with the plans; and + // settle, a person's ask to settle merges held without a passing check (ADR 0556 clause 5). + if want := []string{"deliveries", "show", "groups", "checks", "table", "stalled", "recheck", "close", "settle"}; !reflect.DeepEqual(m.Claims[0].Serves, want) { t.Fatalf("serves %v, wanted %v", m.Claims[0].Serves, want) } // R6-C1: no controller verb is invoked and no plan event is consumed. @@ -98,8 +99,9 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { t.Fatalf("state %v", buckets) } // What it reads: the teller's answers and the checker's verdicts (issue 1302). The carry from today's module's buckets, the - // seat's holder and the machines' haves went with the seat's hand-over (ADR 0539 §17). - if !reflect.DeepEqual(m.Reads, []string{"mesh-delivery-teller.answers", "mesh-delivery-checker.verdicts"}) { + // seat's holder and the machines' haves went with the seat's hand-over (ADR 0539 §17). The delivery view's reach, which + // `deliveries` and `show` derive whether a merge is on the machines from (ADR 0556 clause 4); no forge tool is asked for it. + if !reflect.DeepEqual(m.Reads, []string{"mesh-delivery-teller.answers", "mesh-delivery-checker.verdicts", "mesh-delivery-view.reach"}) { t.Fatalf("reads %v", m.Reads) } // It calls no controller verb and reads the forge with three tools; the forge's writes are the bridge's (ADR 0437 §8). It diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go index b38e721f..a3055a32 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go @@ -287,7 +287,7 @@ func (h *Holder) ForgeMerged(f sdk.Forge) { r, now := d.Ready, h.Now() d.Owed = append(d.Owed, Effect{Kind: EffectEmit, Event: "merged", Line: string(mustJSON(mergedEvent(d, f.Head))), Since: now}) r.Merged, r.Answer, r.RefusedSince, r.merging, r.MergeCommit = &now, "merged", nil, false, f.Merge - h.Logf("[mesh-delivery] %s merged on the operator's Approve at %s, its check passed against its trunk's head", d.ID, shortOf(f.Head, 12)) + h.Logf("[mesh-delivery] %s merged on the operator's Approve at %s (%s)", d.ID, shortOf(f.Head, 12), h.checkAt(h.askMembers(d)).Words) h.keep(d) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go index 9f2a4de8..e46c968f 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go @@ -105,6 +105,10 @@ type Delivery struct { // trunk's head, before the operator is asked (novox/hq ADR 0421 §3, issue 1022). Cleared when it is asked. FreshWanted bool `json:"fresh_wanted,omitempty"` + // Settle is the settle ask a person had written for it while it is held for a merge without a passing check (settle.go, + // novox/hq ADR 0556 clause 5): its key in the gate's asks, why, who asked and when, and the operator's answer once read. + Settle *SettleAsk `json:"settle,omitempty"` + Transitions []Transition `json:"transitions,omitempty"` // Owed are the forge's and the bus's effects of its transitions not yet done: said only after the // delivery is kept, retried until done. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go index 10151f1f..76386e98 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go @@ -302,3 +302,72 @@ func checkerRefusal(id string) error { "its trunk or its group, and nothing is asked of it here (novox/hq ADR 0447). Read where its check stands with the "+ "checker's checker_pull tool", id) } + +// The states of the check at an ask's heads, as the ask says them (issue 1733). +const ( + checkSaidPass = "pass" + checkSaidRunning = "running" + checkSaidNone = "none" + checkSaidFail = "fail" +) + +// checkSaid is the state of the check at the heads an ask covers, and its words for the ask: what the checker's verdict at +// exactly those heads says, never what the gate would like it to say (issue 1733). Key is one of the checkSaid states. +type checkSaid struct { + Key string + Words string +} + +// passed is whether the check at the heads passed. +func (c checkSaid) passed() bool { return c.Key == checkSaidPass } + +// checkAt reads the check at these members' heads: the checker's verdict for exactly those heads (its verdicts bucket, +// keyed by their hash), and, while none is held, the forge's mesh/check at a member's head (a failure or a pending one +// there). "passed its check" is said only for a passing verdict at these heads. Under the lock. +func (h *Holder) checkAt(members []*Delivery) checkSaid { + v, hash, ok := h.checkerAt(members) + if ok { + switch v.State { + case checkerPass: + return checkSaid{Key: checkSaidPass, Words: "passed its check"} + case checkerRunning: + words := "its check has no verdict yet: it is running" + if since, held := h.runningSince[hash]; held && !since.IsZero() { + words += " (since " + since.UTC().Format("15:04") + " UTC)" + } + return checkSaid{Key: checkSaidRunning, Words: words} + default: + why := firstOf(oneLine(v.Failed.Test), oneLine(v.Failed.Words), v.State) + if v.Failed.Test != "" && v.Failed.Words != "" { + why = oneLine(v.Failed.Test) + ": " + oneLine(v.Failed.Words) + } + return checkSaid{Key: checkSaidFail, Words: "its check failed: " + clip(why, 120)} + } + } + for _, m := range members { + switch m.CheckState { + case "failure", "error": + why := "mesh/check is " + m.CheckState + " at " + shortOf(m.Commit, 12) + if m.Check != nil && m.Check.Summary != "" { + why = oneLine(m.Check.Summary) + } + return checkSaid{Key: checkSaidFail, Words: "its check failed: " + clip(why, 120)} + } + } + succeeded := 0 + for _, m := range members { + switch m.CheckState { + case "pending": + return checkSaid{Key: checkSaidRunning, Words: "its check has no verdict yet: it is running"} + case "success": + succeeded++ + } + } + switch { + case succeeded > 0 && len(members) == 1: + return checkSaid{Key: checkSaidPass, Words: "passed its check"} // its head's mesh/check is the checker's verdict at it + case succeeded == len(members) && succeeded > 0: + return checkSaid{Key: checkSaidNone, Words: "its members' own checks passed; the check of their heads together has not answered"} + } + return checkSaid{Key: checkSaidNone, Words: "its check has no verdict yet"} +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go new file mode 100644 index 00000000..6456be41 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go @@ -0,0 +1,102 @@ +package main + +import ( + "fmt" + "strings" + + sdk "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Whether a merge is on the machines (novox/hq ADR 0556 clause 4). The gate records only the merge (ADR 0539 §17, ADR 0429 +// §1): it reads the delivery view's reach of each repository, which names the repository's reached head (the newest merge every +// node runs a build containing), and derives when asked, for a merged delivery, whether its merge is at or before that head. +// The order of two merges into one trunk is the gate's own: the moment each was merged, the trunk being a line of merges. A +// reached head the gate holds no merge of is said, never asked of the forge. + +// reachState is the delivery view's state the gate reads. +const reachState = sdk.ViewModule + ".reach" + +// ReachChanged takes one change of the view's reach. +func (h *Holder) ReachChanged(key string, value []byte, deleted bool) { + h.mu.Lock() + defer h.mu.Unlock() + if deleted { + delete(h.reach, key) + return + } + r, err := sdk.DecodeReach(value) + if err != nil { + h.Logf("[mesh-delivery] the view's reach %s is not readable, so its merges read as not known: %v", key, err) + delete(h.reach, key) + return + } + h.reach[key] = r +} + +// The words a merged delivery's machines are said in. +const ( + onMachines = "on the machines" + notYetOn = "not yet on the machines: " + notKnownWhere = "not known: " +) + +// machines says whether a merged delivery's merge is on the machines, derived from the view's reach; empty for a delivery +// that is not merged. Under the lock. +func (h *Holder) machines(d *Delivery) string { + if d.State != Merged || d.MergedAs == "" { + return "" + } + key, err := sdk.RepositoryKey(d.Repository) + if err != nil { + return notKnownWhere + err.Error() + } + r, ok := h.reach[key] + switch { + case !ok: + return notKnownWhere + "the delivery view says no reach of " + d.Repository + } + // A merge that changed a module a node wants and runs nothing countable of is not yet there, whatever the reached head + // (the module holds no head back, ADR 0556 clause 2). + for _, mr := range r.Modules { + if mr.Nodes == 0 || mr.Floor != "" { + continue + } + for _, c := range mr.Changes { + if sameCommit(d.MergedAs, c.Commit) { + return notYetOn + mr.Waiting + } + } + } + switch { + case r.Reached == "": + return notYetOn + holdsSaid(r) + case sameCommit(d.MergedAs, r.Reached): + return onMachines + } + head := h.mergeOf(d.Repository, r.Reached) + switch { + case head == nil: + return notKnownWhere + fmt.Sprintf("the reached head %s of %s is no merge this gate holds", shortOf(r.Reached, 12), d.Repository) + case !d.MergedAt.After(head.MergedAt): + return onMachines + } + return notYetOn + holdsSaid(r) +} + +// holdsSaid is what holds a repository's next head, in words. +func holdsSaid(r sdk.Reach) string { + if len(r.Holds) == 0 { + return "the delivery view names nothing that holds it yet" + } + return strings.Join(r.Holds, "; ") +} + +// mergeOf is the delivery of a repository that merged as the commit; nil when none does. Under the lock. +func (h *Holder) mergeOf(repository, commit string) *Delivery { + for _, d := range h.deliveries { + if d.MergedAs != "" && strings.EqualFold(d.Repository, repository) && sameCommit(d.MergedAs, commit) { + return d + } + } + return nil +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached_test.go new file mode 100644 index 00000000..657a42eb --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "strings" + "testing" + "time" + + sdk "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// 1280 (novox/hq ADR 0556 clause 4): `deliveries` and `show` say for a merged delivery whether it is on the machines, derived +// when asked from the view's reach and the gate's own order of merges: at or before the reached head it is, after it not yet +// and why; a repository the view has no reach of, or a reached head the gate holds no merge of, is not known. Nothing records it. +func TestAMergedDeliverySaysWhetherItIsOnTheMachines(t *testing.T) { + w := newWorld(t) + at := w.now.Add(-3 * time.Hour) + merge := func(commit string, n int) *Delivery { + d := &Delivery{ID: "novox/web@" + commit[:12], Repository: "novox/web", Commit: commit[:12], Number: n, State: Merged, + Since: at, MergedAs: strings.Repeat(commit[:1], 40), MergedAt: at.Add(time.Duration(n) * time.Minute)} + w.h.mu.Lock() + w.h.keep(d) + w.h.mu.Unlock() + return d + } + first, second, third := merge("111111111111", 1), merge("222222222222", 2), merge("333333333333", 3) + reach := func(r sdk.Reach) { + key, _ := sdk.RepositoryKey(r.Repository) + raw, err := sdk.EncodeReach(r) + if err != nil { + t.Fatal(err) + } + w.h.ReachChanged(key, raw, false) + } + machines := func(d *Delivery) string { w.h.mu.Lock(); defer w.h.mu.Unlock(); return w.h.machines(d) } + if got := machines(first); !strings.HasPrefix(got, notKnownWhere) { + t.Errorf("with no reach: %q", got) + } + reach(sdk.Reach{Repository: "novox/web", Reached: second.MergedAs, ReachedSrc: 7, Holds: []string{"web on g14 runs the build of 2222, not yet that of 3333"}}) + for d, want := range map[*Delivery]string{first: onMachines, second: onMachines, third: notYetOn + "web on g14"} { + if got := machines(d); !strings.HasPrefix(got, want) { + t.Errorf("%s: %q, want %q", d.ID, got, want) + } + } + reach(sdk.Reach{Repository: "novox/web", Reached: strings.Repeat("9", 40), ReachedSrc: 8}) + if got := machines(first); !strings.Contains(got, "no merge this gate holds") { + t.Errorf("a reached head the gate holds no merge of: %q", got) + } + reach(sdk.Reach{Repository: "novox/web", Holds: []string{"web on ace runs nothing yet"}}) + if got := machines(first); got != notYetOn+"web on ace runs nothing yet" { + t.Errorf("no reached head: %q", got) + } + // A module a node wants and runs nothing countable of holds no head back, and a merge that changed it is not yet there. + reach(sdk.Reach{Repository: "novox/web", Reached: third.MergedAs, ReachedSrc: 9, Modules: map[string]sdk.ModuleReach{ + "web": {FP: "src2:x", Nodes: 1, Waiting: "web on g14 runs nothing yet", Changes: []sdk.Change{{Commit: second.MergedAs, FP: "src2:x"}}}}}) + for d, want := range map[*Delivery]string{first: onMachines, second: notYetOn + "web on g14 runs nothing yet", third: onMachines} { + if got := machines(d); got != want { + t.Errorf("with a module running nowhere countable, %s: %q, want %q", d.ID, got, want) + } + } + // The verbs say it; a delivery not merged says nothing of the machines. + reach(sdk.Reach{Repository: "novox/web", Reached: third.MergedAs, ReachedSrc: 9}) + lines := w.h.Deliveries("", "novox/web", "", true) + if len(lines) != 3 || lines[0].Machines != onMachines { + t.Errorf("deliveries = %+v", lines) + } + shown, err := w.h.Show(third.ID) + if err != nil || shown.(map[string]any)["machines"] != onMachines { + t.Errorf("show = %v, %v", shown, err) + } + if got := machines(&Delivery{State: Held, MergedAs: "x", Repository: "novox/web"}); got != "" { + t.Errorf("a held delivery says %q of the machines", got) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reads-outside b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reads-outside index f6b2d871..8deabea3 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reads-outside +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# every manifest: the gate is the seat's only claimant. +modules/*/module.json diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go index 4b96bc11..aee84748 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go @@ -115,6 +115,10 @@ type ReadyRun struct { Heads map[string]string `json:"heads,omitempty"` AskedAt time.Time `json:"asked_at"` Asks int `json:"asks,omitempty"` + // AskCheck is the state of the check at the heads the pending ask names, as its words said it (checkAt: pass, running, + // none or fail; issue 1733): while the check gates no merge, an ask whose check reached a verdict since is asked again + // in new words, so it never keeps a claim the check no longer holds. + AskCheck string `json:"ask_check,omitempty"` // Approved and Declined are when the operator chose, Warrant what the router's warrant says of who and how; Refused // why an answer was not taken (it did not hold to the ask). Approved *time.Time `json:"approved,omitempty"` @@ -186,7 +190,8 @@ func (h *Holder) readyAsks(now time.Time) []func() { if d.Ready == nil && d.State == Ready { h.runOf(d, now) h.Poke() // its check, if none is asked, is worth asking now (askCheck) - h.Logf("[mesh-delivery] %s: its checks passed; the operator's Approve of its merge into main is asked once it may merge", d.ID) + h.Logf("[mesh-delivery] %s is ready (%s); the operator's Approve of its merge into main is asked once it may merge", + d.ID, h.checkAt([]*Delivery{d}).Words) } if a := h.sayAfterLine(d); a != nil { asks = append(asks, a) @@ -227,11 +232,12 @@ func (h *Holder) readyAsks(now time.Time) []func() { if d.Base != MergeBranch { if r.OtherBase != d.Base { r.OtherBase = d.Base - h.Logf("[mesh-delivery] %s passed its checks and is not asked: it merges into %s, not %s", d.ID, d.Base, MergeBranch) + check := h.checkAt([]*Delivery{d}).Words // ready need not mean passed: the check may gate no merge (issue 1733) + h.Logf("[mesh-delivery] %s (%s) is not asked: it merges into %s, not %s", d.ID, check, d.Base, MergeBranch) h.keep(d) - asks = append(asks, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** This pull request's checks passed at "+ - "%s, and it merges into %s. Only a merge into %s is this owner's; a merge into another branch is its "+ - "author's (novox/hq ADR 0398 §4).", d.Commit, d.Base, MergeBranch))) + asks = append(asks, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** This pull request is at %s (%s), "+ + "and it merges into %s. Only a merge into %s is this owner's; a merge into another branch is its "+ + "author's (novox/hq ADR 0398 §4).", d.Commit, check, d.Base, MergeBranch))) } continue } @@ -524,13 +530,14 @@ func (h *Holder) readyLines(now time.Time) []StalledLine { case d.Base == "" && d.Number != 0 && now.Sub(r.Seen) >= headHeardBound: out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(r.Seen).Round(time.Second).String(), Bound: headHeardBound.String(), H2: "none: a later announcement of the pull request's head (the merge tool announces it again, the forge's announcer restarts, or a new push) or an edit of its description; a recheck does not fill it — the operator's", - Says: fmt.Sprintf("%s#%d passed its checks at %s and waits for %s", d.Repository, d.Number, shortOf(r.Head, 12), waitsForHead)}) + Says: fmt.Sprintf("%s#%d at %s (%s) waits for %s", d.Repository, d.Number, shortOf(r.Head, 12), + h.checkAt([]*Delivery{d}).Words, waitsForHead)}) case r.CannotWrite != "" && r.Approved == nil && r.CannotWriteSince != nil: out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(*r.CannotWriteSince).Round(time.Second).String(), Bound: "0s", H2: "none: the forge account given write on the repository (gitea_grant_write; the gitea module grants it itself once a module of it is registered), which asks the operator then", - Says: fmt.Sprintf("%s#%d passed its checks at %s and the operator is not asked for its Approve: %s", - d.Repository, d.Number, shortOf(r.Head, 12), r.CannotWrite)}) + Says: fmt.Sprintf("%s#%d at %s (%s): the operator is not asked for its Approve: %s", + d.Repository, d.Number, shortOf(r.Head, 12), h.checkAt([]*Delivery{d}).Words, r.CannotWrite)}) case r.Failed != "" && d.State == Rejected: // Its cure is its author's new head: listed here at once (`show`), told on the pull request at once, and the // operator's only once no new head came within the author's bound (novox/hq ADR 0505 §5). diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go index e15cc1e1..6e139b14 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go @@ -1,11 +1,15 @@ package main import ( + "encoding/json" "errors" "sort" "strings" "testing" "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + sdk "git.novox.be/novox/mesh-sdk/go/delivery" ) // **A pull request whose checks passed and whose merge the operator approved is merged by this owner** (novox/hq issues @@ -65,6 +69,73 @@ func TestAnApprovedPullRequestIsMergedThroughTheMergeTool(t *testing.T) { t.Fatalf("merged again: %v", w.forge.merges) } + // 1733: the Approve's ask says the check's state at the heads it names, read from the checker's verdict there: "passed + // its check" only when it passed. While the check gates no merge, the ask is made before any verdict, says that Approve + // merges before the check ends, and is asked again in new words once the check passes or fails; a running check is no + // new word. + for _, row := range []struct { + name, state, failed string + gated bool + asks int + want, not []string + }{ + {name: "gated, passed", gated: true, state: "pass", asks: 1, + want: []string{"app#7 Fix it passed its check. Approve merges it into main at " + readyHead[:12] + ";"}, + not: []string{"before its check ends", "no verdict yet"}}, + {name: "ungated, no verdict", asks: 1, + want: []string{"its check has no verdict yet", "Approve merges it into main at " + readyHead[:12] + " now, before its check ends"}, + not: []string{"passed"}}, + {name: "ungated, running", state: "running", asks: 1, + want: []string{"its check has no verdict yet", "before its check ends"}, not: []string{"passed"}}, + {name: "ungated, then passed", state: "pass", asks: 2, + want: []string{"passed its check. Approve merges it into main at " + readyHead[:12] + ";"}, not: []string{"before its check ends"}}, + {name: "ungated, then failed", state: "fail", failed: "TestThing: wanted 1, got 2", asks: 2, + want: []string{"its check failed: TestThing: wanted 1, got 2", "although its check failed"}, not: []string{"passed"}}, + } { + t.Run(row.name, func(t *testing.T) { + v := newWorld(t) + if !row.gated { + ungated(v) + } + v.op.auto = false + p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") + p.Title = "Fix it" + v.h.PullUpdated(p) + put := func() { + hash, _ := sdk.HeadsHash([]sdk.Head{headOf("novox/app", 7, readyHead)}) + value, _ := json.Marshal(map[string]any{"v": 1, "group": "g", "heads": hash, "state": row.state, + "failed": map[string]string{"words": row.failed}}) + v.h.CheckerVerdictChanged("g."+hash, value) + } + if row.gated { + v.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + } + v.settleAll() + if !row.gated && row.state != "" { + put() + v.settleAll() + } + open := v.op.open() + if len(open) != 1 || v.op.count() != row.asks { + t.Fatalf("open asks %v, asked %d times; wanted one open, asked %d times", open, v.op.count(), row.asks) + } + e := v.op.ask(open[0]).Explanation + for _, w := range row.want { + if !strings.Contains(e, w) { + t.Fatalf("the ask does not say %q:\n%s", w, e) + } + } + for _, w := range row.not { + if strings.Contains(e, w) { + t.Fatalf("the ask says %q:\n%s", w, e) + } + } + if err := asks.Glance(e); err != nil { + t.Fatal(err) + } + }) + } + // check-gates-merge false (the operator, 2026-10-25 22:56): a reviewed pull request is asked with no verdict, another // merge moving its trunk puts it out of nothing, a failed verdict is shown on it and holds nothing, and the gate's // status and show say the mode. @@ -255,6 +326,11 @@ func TestAGroupMergesEveryMemberInOnePassOnlyOnceComposedAndApproved(t *testing. if u.op.count() != 1 { t.Fatalf("ungated, the group is asked %d times; wanted once", u.op.count()) } + // 1733: the group's ask claims no pass it has not had. + if e := u.op.ask(u.op.open()[0]).Explanation; strings.Contains(e, "passed") || + !strings.Contains(e, "2 pull requests: its check has no verdict yet. Approve merges all of them into main in one pass, at the heads named, now, before its check ends;") { + t.Fatalf("ungated, the group's ask does not say its check has no verdict yet:\n%s", e) + } u.op.answer(u.op.open()[0], OptionApprove) u.settleAll() if len(u.forge.merges) != 2 { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go new file mode 100644 index 00000000..a95ef111 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go @@ -0,0 +1,252 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + sdk "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A person settles a delivery held because its merge reached the trunk without a passing check (novox/hq ADR 0556 clause 5). +// The verb `settle` moves nothing: it writes one ask, for every delivery named, into the gate's asks, which the teller puts to +// the operator as it puts an Approve, its options binding mesh-delivery.settle (sdk.AskVerb). On the answer settle each +// delivery takes the table's asked row held → merged with the operator's words; on leave they stay held. Whether a settled +// merge is on the machines is then derived as for any merge (reached.go). A hold of another kind, or a delivery not held, is +// refused, naming why. + +// The settle ask's two answers, both at the level approve. +const ( + OptionSettle = "settle" + OptionLeave = "leave" +) + +// SettleAsk is the settle ask a delivery is covered by. +type SettleAsk struct { + Key string `json:"key"` + Asking int `json:"asking,omitempty"` + Why string `json:"why"` + By string `json:"by,omitempty"` + At time.Time `json:"at"` + Answered *time.Time `json:"answered,omitempty"` + Answer string `json:"answer,omitempty"` + Words string `json:"words,omitempty"` +} + +// Settle is the `settle` verb: one ask of the operator for every delivery named, each held for a merge without a passing +// check. Nothing moves until the operator answers. +func (h *Holder) Settle(ids []string, why, by string) (map[string]any, error) { + why = strings.TrimSpace(why) + if why == "" { + return nil, fmt.Errorf("settle says why: it is shown to the operator") + } + if len(ids) == 0 { + return nil, fmt.Errorf("settle names at least one held delivery (ids)") + } + if h.Operator == nil { + return nil, fmt.Errorf("this owner has no way to ask the operator") + } + h.mu.Lock() + var covered []*Delivery + var refused []string + seen := map[string]bool{} + for _, id := range ids { + id = strings.TrimSpace(id) + if seen[id] { + continue + } + seen[id] = true + d := h.deliveries[id] + switch { + case d == nil: + refused = append(refused, id+": no such delivery") + case d.State != Held: + refused = append(refused, fmt.Sprintf("%s: it is %s, not held", id, d.State)) + case !heldUnchecked(d): + refused = append(refused, fmt.Sprintf("%s: it is held for another reason, which settle does not take (%s)", id, d.HeldWhy)) + default: + covered = append(covered, d) + } + } + if len(refused) > 0 { + h.mu.Unlock() + sort.Strings(refused) + return nil, fmt.Errorf("settle refused, nothing asked: %s", strings.Join(refused, "; ")) + } + sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) + var names []string + var heads []sdk.Head + for _, d := range covered { + names = append(names, d.ID) + heads = append(heads, sdk.Head{Repository: d.Repository, Number: d.Number, Head: d.Commit}) + } + group := sdk.SettleGroup(names) + key, err := sdk.MergeAskKey(group, heads) + if err != nil { + h.mu.Unlock() + return nil, fmt.Errorf("the settle ask has no key: %w", err) + } + now := h.Now() + // The same deliveries asked again (the ask lapsed, or was left unanswered) are a new asking of the same key, under its own + // router id: the router drops a second ask under an id it kept. + asking := 1 // the first asking (sdk.Ask.Asking: 0 or 1 is the first, n the n-th) + for _, d := range covered { + if d.Settle != nil && d.Settle.Key == key && max(d.Settle.Asking, 1) >= asking { + asking = max(d.Settle.Asking, 1) + 1 + } + } + ask := sdk.Ask{Group: group, Heads: heads, Asking: asking, Ask: composeSettleAsk(sdk.AskingID(key, asking), covered, why, now)} + for _, d := range covered { + d.Settle = &SettleAsk{Key: key, Asking: asking, Why: why, By: by, At: now} + h.keep(d) + } + h.mu.Unlock() + if err := h.Operator.Ask(key, ask); err != nil { + return nil, fmt.Errorf("the settle ask %s could not be written (call settle again): %w", key, err) + } + h.Logf("[mesh-delivery] asked the operator to settle %s: %s", strings.Join(names, ", "), why) + return map[string]any{"asked": key, "deliveries": names, + "said": fmt.Sprintf("The operator is asked once to settle %d deliveries; nothing moves until the answer.", len(covered))}, nil +} + +// settleAct is what a settle ask's answer binds: its verb (mesh-delivery.settle), this ask, the answer and the heads. +func settleAct(group, ask, option string, heads map[string]string) asks.Act { + return asks.Act{"verb": sdk.AskVerb(group), "ask": ask, "option": option, "heads": headsString(heads)} +} + +// composeSettleAsk is the operator's ask to settle these held deliveries. +func composeSettleAsk(id string, ds []*Delivery, why string, now time.Time) asks.Ask { + var a asks.Ask + a.ID, a.Who, a.Expires = id, asks.Operator, now.Add(AskLasts) + a.OnExpiry = "they stay held; settle asks again" + a.About = "delivery.settle" + var pulls, lines []string + heads := map[string]string{} + for _, d := range ds { + pulls = append(pulls, pullSaid(d)) + heads[d.ID] = d.Commit + lines = append(lines, phone(fmt.Sprintf("%s %q merged at %s without a passing check", pullSaid(d), d.Title, + shortOf(d.MergedAs, 12)), 400)) + } + a.Headline = phone(fmt.Sprintf("Settle %d merges made without a passing check", len(ds)), asks.HeadlineLength) + a.Explanation = strings.Join([]string{ + phone(fmt.Sprintf("%s reached main without a passing check and are held. Settle ends the hold: each reads merged, and on the machines once every machine runs it. Leave keeps them held.", + strings.Join(pulls, ", ")), asks.SummaryLength), + phone("Why: "+why, asks.PointLength), + }, "\n") + a.Details = strings.Join(lines, "\n") + group := sdk.SettleGroup(func() []string { + var ids []string + for _, d := range ds { + ids = append(ids, d.ID) + } + return ids + }()) + for _, o := range []struct{ id, label, does string }{ + {OptionSettle, "Settle", "each delivery named ends its hold and reads merged"}, + {OptionLeave, "Leave", "they stay held"}, + } { + binds, _ := asks.ActDigest(settleAct(group, id, o.id, heads)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +// isSettleKey is whether an ask key is a settle ask's; under the lock. +func (h *Holder) isSettleKey(key string) bool { + _, group, _, err := sdk.ParseAskKey(key) + return err == nil && strings.HasPrefix(group, sdk.SettleGroupPrefix) +} + +// settleKeys are the settle asks waiting for an answer; under the lock. +func (h *Holder) settleKeys() []string { + seen := map[string]bool{} + var out []string + for _, d := range h.deliveries { + if s := d.Settle; s != nil && s.Answered == nil && d.State == Held && !seen[s.Key] { + seen[s.Key] = true + out = append(out, s.Key) + } + } + sort.Strings(out) + return out +} + +// settled applies the teller's answer to a settle ask; under the lock. The teller held the warrant (the act it bound is +// mesh-delivery.settle); an answer to an earlier asking, or one choosing what the ask did not offer, moves nothing. +func (h *Holder) settled(key string, ans sdk.Answer) { + var covered []*Delivery + for _, d := range h.deliveries { + if d.Settle != nil && d.Settle.Key == key && d.Settle.Answered == nil { + covered = append(covered, d) + } + } + if len(covered) == 0 { + h.Logf("[mesh-delivery] the answer to the settle ask %s covers no delivery that waits for it: nothing settled", key) + return + } + sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) + if asked, at, ok := strings.Cut(ans.Warrant, "@"); ok && at != "" && asked != sdk.AskingID(key, covered[0].Settle.Asking) { + h.Logf("[mesh-delivery] the answer to the settle ask %s was given to another asking (%s): not taken", key, asked) + return + } + if ans.Option != OptionSettle && ans.Option != OptionLeave { + h.refuse(fmt.Errorf("the answer to the settle ask %s chose %s, which it did not offer: nothing settled", key, ans.Option)) + return + } + now := h.Now() + for _, d := range covered { + d.Settle.Answered, d.Settle.Answer, d.Settle.Words = &now, ans.Option, ans.Words + if ans.Option == OptionLeave { + h.Logf("[mesh-delivery] %s: the operator left it held (%s)", d.ID, ans.Words) + h.keep(d) + continue + } + t, err := Apply(d, EvSettled, true, Facts{Now: now, By: "the operator", Why: "settled: " + ans.Words}) + if err != nil { + h.refuse(err) + h.keep(d) + continue + } + d.HeldWhy = "" + h.moved(d, []Transition{t}) + } + // Answered: the ask is taken away, as a merge ask is once acted on. + h.cancels = append(h.cancels, h.cancelSettle(key)) +} + +// cancelSettle takes a settle ask away from the gate's asks. +func (h *Holder) cancelSettle(key string) func() { + return func() { + if h.Operator == nil { + return + } + if err := h.Operator.Cancel(key); err != nil { + h.Logf("[mesh-delivery] the settle ask %s could not be taken away (taken away again at the next look): %v", key, err) + } + } +} + +// endStaleSettles takes away every settle ask nobody answered within its bound; its deliveries stay held and settle may ask +// again. Under the lock; the cancels run with the look's asks. +func (h *Holder) endStaleSettles() { + done := map[string]bool{} + for _, d := range h.deliveries { + s := d.Settle + if s == nil || s.Answered != nil || done[s.Key] || h.Now().Before(s.At.Add(AskLasts)) { + continue + } + done[s.Key] = true + h.Logf("[mesh-delivery] the settle ask %s lapsed with no answer: taken away, %s stays held", s.Key, d.ID) + h.cancels = append(h.cancels, h.cancelSettle(s.Key)) + } + for _, d := range h.deliveries { + if s := d.Settle; s != nil && done[s.Key] { + now := h.Now() + s.Answered, s.Answer = &now, "lapsed" + h.keep(d) + } + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go new file mode 100644 index 00000000..bbe88fae --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + sdk "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// 1280 (novox/hq ADR 0556 clause 5): settle writes one ask for several deliveries held for a merge without a passing check and +// moves none of them; it refuses, asking nothing, a delivery held for another reason and one not held, naming why; the +// operator's Leave keeps them held, and Settle, asked again under a new asking, moves each to merged with the operator's words. +func TestSettleAsksTheOperatorOnceAndMovesOnlyOnTheAnswer(t *testing.T) { + w := newWorld(t) + w.op.auto = false + merged := w.now.Add(-time.Hour) + put := func(id string, state State, why string) *Delivery { + repo, commit, _ := strings.Cut(id, "@") + d := &Delivery{ID: id, Repository: repo, Commit: commit, Number: len(id), Title: "a fix", State: state, Since: merged, + MergedAs: "m" + commit, MergedAt: merged, HeldWhy: why} + w.h.mu.Lock() + w.h.keep(d) + w.h.mu.Unlock() + return d + } + a := put("novox/web@aaaaaaaaaaaa", Held, HeldUnchecked) + b := put("novox/sdk@bbbbbbbbbbbb", Held, HeldUnchecked) + put("novox/web@cccccccccccc", Held, "a conflict on the trunk") + put("novox/web@dddddddddddd", Merged, "") + for ids, words := range map[string]string{ + "novox/web@aaaaaaaaaaaa novox/web@cccccccccccc": "held for another reason", + "novox/web@aaaaaaaaaaaa novox/web@dddddddddddd": "not held", + "novox/web@aaaaaaaaaaaa novox/web@eeeeeeeeeeee": "no such delivery", + } { + if _, err := w.h.Settle(strings.Fields(ids), "on every machine since 10-25", aPerson); err == nil || !strings.Contains(err.Error(), words) { + t.Errorf("settle %s: %v, want refused naming %q", ids, err, words) + } + } + if _, err := w.h.Settle([]string{a.ID}, " ", aPerson); err == nil { + t.Error("a settle with no why was asked") + } + if w.op.count() != 0 { + t.Fatalf("a refused settle asked the operator: %d asks", w.op.count()) + } + out, err := w.h.Settle([]string{b.ID, a.ID}, "on every machine since 10-25", aPerson) + if err != nil { + t.Fatal(err) + } + key, _ := out["asked"].(string) + if open := w.op.open(); len(open) != 1 || open[0] != key || w.op.count() != 1 { + t.Fatalf("asks open %v (asked %d), want the one key %s", open, w.op.count(), key) + } + ask := w.op.ask(key) + heads := map[string]string{a.ID: a.Commit, b.ID: b.Commit} + group := sdk.SettleGroup([]string{a.ID, b.ID}) + for _, o := range ask.Options { + if want, _ := asks.ActDigest(settleAct(group, ask.ID, o.ID, heads)); o.Binds != want { + t.Errorf("the option %s binds %s, not mesh-delivery.settle of these heads", o.ID, o.Binds) + } + } + w.h.hearAsks() + if a.State != Held || b.State != Held { + t.Fatalf("moved before an answer: %s, %s", a.State, b.State) + } + w.op.answer(key, OptionLeave) + w.h.hearAsks() + if a.State != Held || a.Settle == nil || a.Settle.Answer != OptionLeave { + t.Fatalf("after Leave: %s %+v", a.State, a.Settle) + } + // Asked again: a new asking of the same key, under its own id. + if _, err := w.h.Settle([]string{a.ID, b.ID}, "asked again", aPerson); err != nil { + t.Fatal(err) + } + if again := w.op.ask(key); again.ID == ask.ID || a.Settle.Asking != 2 { + t.Fatalf("asked again under %s (asking %d), want a new id", again.ID, a.Settle.Asking) + } + // An answer under a settle key no delivery waits for moves nothing. + other, _ := sdk.MergeAskKey(sdk.SettleGroup([]string{"novox/web@ffffffffffff"}), []sdk.Head{{Repository: "novox/web", Number: 1, Head: "ffffffffffff"}}) + w.h.mu.Lock() + w.h.settled(other, sdk.Answer{Option: OptionSettle, Warrant: "x@now", Words: "chose Settle"}) + w.h.mu.Unlock() + if a.State != Held || b.State != Held || !strings.Contains(w.logbuf.String(), "covers no delivery") { + t.Fatalf("an answer for other ids moved %s, %s", a.State, b.State) + } + w.op.answer(key, OptionSettle) + w.h.hearAsks() + w.h.mu.Lock() + cancels := len(w.h.cancels) + w.h.mu.Unlock() + if cancels == 0 { + t.Error("the answered settle ask is not taken away") + } + for _, d := range []*Delivery{a, b} { + last := d.Transitions[len(d.Transitions)-1] + if d.State != Merged || last.Event != EvSettled || last.By != "the operator" || !strings.Contains(last.Why, "chose Settle") { + t.Errorf("%s after Settle: %s, %+v", d.ID, d.State, last) + } + } + // A settle ask nobody answers within its bound is taken away; its delivery stays held. + c := put("novox/web@999999999999", Held, HeldUnchecked) + if _, err := w.h.Settle([]string{c.ID}, "once more", aPerson); err != nil { + t.Fatal(err) + } + w.later(AskLasts + time.Minute) + w.h.mu.Lock() + w.h.endStaleSettles() + w.h.mu.Unlock() + if c.State != Held || c.Settle.Answer != "lapsed" { + t.Errorf("a lapsed settle ask: %s %+v", c.State, c.Settle) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go index f95563a0..bfc56be9 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go @@ -77,8 +77,18 @@ const ( EvMerged Event = "merged" // it reached the trunk EvMergedUnchecked Event = "merged-unchecked" // it reached the trunk without a passing check EvTrunkMoved Event = "trunk moved" // another merge moved its trunk, and its verdict names the trunk commit before (novox/hq ADR 0323) + EvSettled Event = "settled" // the operator settled a merge held without a passing check (novox/hq ADR 0556 clause 5) ) +// HeldUnchecked is the why of a delivery held because its merge reached the trunk without a passing check: the one hold a +// person may settle (novox/hq ADR 0556 clause 5). +const HeldUnchecked = "merged without a passing check: only a person decides what follows" + +// heldUnchecked is whether a held delivery is held for a merge without a passing check, and reached the trunk. +func heldUnchecked(d *Delivery) bool { + return d.State == Held && d.MergedAs != "" && strings.HasPrefix(d.HeldWhy, "merged without a passing check") +} + // Facts are what a guard reads beyond the delivery itself: the moment, and who asks, for an act. type Facts struct { Now time.Time @@ -177,6 +187,20 @@ var Table = []Row{ {From: []State{Proposed, Rejected}, Event: EvMergedUnchecked, To: Held, Guard: "merged without a passing check — a verdict known with it is taken first: only a person decides what follows", Holds: func(d *Delivery, f Facts) error { return unless(d.MergedAs != "", "not merged") }}, + // **A person settles a merge held without a passing check** (novox/hq ADR 0556 clause 5): only on the operator's answer to + // the settle ask, with its words; a hold of another kind is not taken here. Whether it is on the machines is then derived. + {From: []State{Held}, Event: EvSettled, To: Merged, Act: true, + Guard: "held because its merge reached the trunk without a passing check, and the operator answered settle to the ask " + + "`settle` wrote, with the words of that answer", + Holds: func(d *Delivery, f Facts) error { + switch { + case !heldUnchecked(d): + return errors.New("it is not held for a merge without a passing check: " + d.HeldWhy) + case f.By == "" || f.Why == "": + return errors.New("a settle names the operator's answer and its words") + } + return nil + }}, {From: []State{Rejected, Ready}, Event: EvRecheck, To: Proposed, Act: true, Guard: "a person asked, with why, or its group changed", Holds: func(d *Delivery, f Facts) error { return unless(f.Why != "", "a recheck says why") }}, diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go index ab2bccbf..bf92f306 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go @@ -34,6 +34,9 @@ func aDeliveryFor(r Row) (*Delivery, Facts) { case EvTrunkMoved: d.Check = &Verdict{Gate: "pass", Head: "c0ffee000000", Trunk: "d28ca95bbbbb"} d.TrunkNow = "690966dccccccccc" + case EvSettled: + d.MergedAs, d.HeldWhy = "aaaa1111", HeldUnchecked + f.Why, f.By = "settled: the operator chose Settle", "the operator" } return d, f } @@ -103,7 +106,7 @@ func TestEveryRowOfTheTableIsTakenWhenItsGuardHoldsAndRefusedWhenNot(t *testing. func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) { events := []Event{EvAnnounced, EvAccepted, EvRefused, EvRecheck, EvReask, EvLost, EvNewHead, EvClosed, EvMerged, - EvMergedUnchecked, EvTrunkMoved} + EvMergedUnchecked, EvTrunkMoved, EvSettled} for _, from := range append([]State{None}, AllStates...) { for _, ev := range events { for _, act := range []bool{false, true} { @@ -128,14 +131,17 @@ func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) { // The rules the table is for, read off the table. func TestTheTableKeepsItsRules(t *testing.T) { for _, r := range Table { + // The one row out of a final state is a person's settle of a merge held without a passing check, taken on the + // operator's answer (novox/hq ADR 0556 clause 5): it is already on the trunk, and moves from held to merged. + settle := r.Event == EvSettled && r.Act && len(r.From) == 1 && r.From[0] == Held && r.To == Merged for _, from := range r.From { - // Nothing leaves a final state. - if from.Final() { + // Nothing else leaves a final state. + if from.Final() && !settle { t.Errorf("%s is final and the table leaves it by %s", from, r.Event) } } // Only a ready delivery is merged, and nothing but its merge, or a merge without a passing check, puts one on the trunk. - if r.To == Merged && !(r.Event == EvMerged && len(r.From) == 1 && r.From[0] == Ready) { + if r.To == Merged && !settle && !(r.Event == EvMerged && len(r.From) == 1 && r.From[0] == Ready) { t.Errorf("%v —%s→ merged: only a ready delivery's merge merges", r.From, r.Event) } if r.To == Held && r.Event != EvMergedUnchecked { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go index 166832cc..07689592 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go @@ -22,6 +22,9 @@ type Line struct { Waits string `json:"waits,omitempty"` Owed int `json:"owed,omitempty"` Stalled bool `json:"stalled,omitempty"` + // Machines says, for a merged delivery, whether its merge is on the machines, derived from the delivery view's reach + // (reached.go, novox/hq ADR 0556 clause 4); recorded nowhere. + Machines string `json:"machines,omitempty"` } // waitsFor is what a delivery waits for, in words. @@ -61,7 +64,7 @@ func (h *Holder) Deliveries(state, repository, group string, all bool) []Line { continue } l := Line{ID: d.ID, State: d.State, Since: d.Since.UTC().Format(time.RFC3339), For: now.Sub(d.Since).Round(time.Second).String(), - Group: d.Group, Number: d.Number, Waits: h.waitsFor(d), Owed: len(d.Owed)} + Group: d.Group, Number: d.Number, Waits: h.waitsFor(d), Owed: len(d.Owed), Machines: h.machines(d)} l.Stalled = pastBound(d, now) out = append(out, l) } @@ -80,6 +83,9 @@ func (h *Holder) Show(id string) (any, error) { defer h.mu.Unlock() if d := h.deliveries[id]; d != nil { out := map[string]any{"delivery": d, "waits": h.waitsFor(d), "check-gates-merge": h.checkGates()} + if m := h.machines(d); m != "" { + out["machines"] = m + } if g := h.groups[d.Group]; g != nil { state, why, _ := h.groupState(g, h.membersOf(g)) out["group"] = map[string]any{"group": g, "state": state, "why": why} @@ -326,5 +332,9 @@ func (h *Holder) Status(listening string) map[string]any { // HeldLine says a held delivery in one line: which, since when and why. A merge without a passing check is a person's to // read; the gate moves nothing after the merge. func HeldLine(d *Delivery) string { - return fmt.Sprintf("%s held since %s: %s", d.ID, d.Since.UTC().Format(time.RFC3339), d.HeldWhy) + line := fmt.Sprintf("%s held since %s: %s", d.ID, d.Since.UTC().Format(time.RFC3339), d.HeldWhy) + if heldUnchecked(d) { + line += " (mesh-delivery.settle asks the operator to settle it, ADR 0556)" + } + return line } diff --git a/modules/mesh-delivery-gate/go.mod b/modules/mesh-delivery-gate/go.mod index 3b7ee541..aee248e7 100644 --- a/modules/mesh-delivery-gate/go.mod +++ b/modules/mesh-delivery-gate/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-gate go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026111253-9c61918ac36f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-gate/go.sum b/modules/mesh-delivery-gate/go.sum index 346bdcce..b3dab4f0 100644 --- a/modules/mesh-delivery-gate/go.sum +++ b/modules/mesh-delivery-gate/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026111253-9c61918ac36f h1:9+TZpNq8Ifj1+qmbidE6l/7qkbua4yLdzRMrezNvf1A= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026111253-9c61918ac36f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61 h1:/c6ncICWjq55nBf8TlDM0ROe/PuH+qZIdnuVbtg8UuM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 6dd019be..06e16ec7 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -14,7 +14,8 @@ "table", "stalled", "recheck", - "close" + "close", + "settle" ] } ], @@ -70,7 +71,8 @@ ], "reads": [ "mesh-delivery-teller.answers", - "mesh-delivery-checker.verdicts" + "mesh-delivery-checker.verdicts", + "mesh-delivery-view.reach" ], "tools": [ "delivery_hand_now", diff --git a/modules/mesh-issues/cmd/mesh-issues/delivered_test.go b/modules/mesh-issues/cmd/mesh-issues/delivered_test.go index cea93929..67a810be 100644 --- a/modules/mesh-issues/cmd/mesh-issues/delivered_test.go +++ b/modules/mesh-issues/cmd/mesh-issues/delivered_test.go @@ -130,6 +130,33 @@ func TestANewerRevisionCoversTheOlderMergesOfItsRepositoryAsTheForgeSaysAncestry if v, _ := h2.View(n); v.Delivery != "fix merged, not delivered: mesh-catalog#33" { t.Fatalf("another repository's head covered it: %q", v.Delivery) } + // 1280 (ADR 0556 clauses 3 and 6): the view's per-repository form, a repository's reached head with no group, delivers the + // merges at or before that head and not a later one. + h3 := newHarness(t) + reached, later := fixIssue(t, h3, "mesh-catalog#34"), fixIssue(t, h3, "mesh-catalog#35") + mergedFix(t, h3, 34, "feat/e", "6666666666660000") + mergedFix(t, h3, 35, "feat/f", "7777777777770000") + h3.Contains = func(_, _, commit, in string) (bool, error) { + return commit == "6666666666660000" && in == "8888888888880000", nil + } + say := func(rev int, notYet ...string) { + raw, _ := json.Marshal(map[string]any{"revision": rev, "group": "", "repository": "novox/mesh-catalog", + "heads": map[string]string{"novox/mesh-catalog": "8888888888880000"}, "not_yet": notYet}) + if err := h3.OnDelivered(stdio.Envelope{Key: DeliveredEvent, Body: raw}); err != nil { + t.Fatal(err) + } + } + // A merge the event says is not yet on the machines (its module runs on no node yet) waits for the next event. + say(15, "6666666666660000") + if v, _ := h3.View(reached); v.Delivery != "fix merged, not delivered: mesh-catalog#34" { + t.Errorf("a merge said not yet: %q", v.Delivery) + } + say(15) + for n, want := range map[uint64]string{reached: "delivered to every machine it should reach", later: "fix merged, not delivered: mesh-catalog#35"} { + if v, _ := h3.View(n); v.Delivery != want { + t.Errorf("per repository: issue %d reads %q, wanted %q", n, v.Delivery, want) + } + } } // A fix kept with only its merged flag (heard through `checks`, or kept before release 6) is matched to a delivered revision diff --git a/modules/mesh-issues/cmd/mesh-issues/delivery.go b/modules/mesh-issues/cmd/mesh-issues/delivery.go index b09e1fc7..75812f9f 100644 --- a/modules/mesh-issues/cmd/mesh-issues/delivery.go +++ b/modules/mesh-issues/cmd/mesh-issues/delivery.go @@ -429,7 +429,8 @@ const DeliveredEvent = "mesh-delivery-view.delivered" // group moved, or whose merge is the head the revision holds for its repository, reached the machines at that revision. The same // revision twice changes nothing (a delivered fix stays delivered). A fix is matched by the group it merged in, by its own merge // commit, or because its merge is an ancestor of the head the revision holds for its repository (coveredBy): a newer revision -// covers the older ones, which the view may never have said. +// covers the older ones, which the view may never have said. The view also says each repository's reached head in this form, +// with no group and that repository's head alone (novox/hq ADR 0556 clause 3): a merge at or before it is on the machines. func (t *Tracker) OnDelivered(e stdio.Envelope) error { var body struct { Revision uint64 `json:"revision"` @@ -437,6 +438,9 @@ func (t *Tracker) OnDelivered(e stdio.Envelope) error { Heads map[string]string `json:"heads"` // At is when the view judged the revision delivered, when it says so; else this tracker's moment of hearing it. At time.Time `json:"at"` + // NotYet are, in the per-repository form, merges at or before the head that are not yet on the machines (a module a node + // wants and runs nothing countable of, ADR 0556 clause 2): a fix merged as one of them waits for a later event. + NotYet []string `json:"not_yet"` } if err := json.Unmarshal(e.Body, &body); err != nil || (body.Group == "" && len(body.Heads) == 0) { return nil @@ -456,7 +460,8 @@ func (t *Tracker) OnDelivered(e stdio.Envelope) error { if f != nil && f.Merged && !contains(is.Delivered, pr) && f.MergedAs == "" { t.fillMerge(n, f, pr) } - if f == nil || !f.Merged || contains(is.Delivered, pr) || !(deliveredBy(f, body.Group, body.Heads) || t.coveredBy(f, body.Heads)) { + if f == nil || !f.Merged || contains(is.Delivered, pr) || notYet(f, body.NotYet) || + !(deliveredBy(f, body.Group, body.Heads) || t.coveredBy(f, body.Heads)) { continue } pr := pr @@ -584,6 +589,16 @@ func deliveredBy(f *Fix, group string, heads map[string]string) bool { return false } +// notYet is whether a fix's merge is among the merges an event says are not yet on the machines. +func notYet(f *Fix, merges []string) bool { + for _, m := range merges { + if f.MergedAs != "" && sameCommitIn(f.MergedAs, m) { + return true + } + } + return false +} + // sameCommitIn is whether two commits are one, however either is abbreviated (at least seven characters). func sameCommitIn(a, b string) bool { a, b = strings.ToLower(a), strings.ToLower(b) diff --git a/modules/mesh-issues/cmd/mesh-issues/list_progress_test.go b/modules/mesh-issues/cmd/mesh-issues/list_progress_test.go index ef252169..844989f1 100644 --- a/modules/mesh-issues/cmd/mesh-issues/list_progress_test.go +++ b/modules/mesh-issues/cmd/mesh-issues/list_progress_test.go @@ -2,6 +2,7 @@ package main import ( "reflect" + "strings" "testing" "time" ) @@ -26,6 +27,22 @@ func TestListSaysWhenEachIssueMovedAndKeepsTheLinkedOnes(t *testing.T) { if _, err := h.Link(LinkAsk{Number: quiet, Kind: LinkDesign, Target: "03-DESIGN/01-to-be/47-delivery-from-commit-to-delivered.md"}); err != nil { t.Fatal(err) } + // Issue 1781: a milestone or a commit link places the issue and moves nothing, nor keeps its worker current. + if _, err := h.milestoneSet("lead@shanks", MilestoneSetAsk{Design: 47, ID: "R6", Outcome: "x"}); err != nil { + t.Fatal(err) + } + h.fixing(t, noted) + worked, _ := h.Get(noted) + clock = clock.Add(time.Hour) + for _, l := range []LinkAsk{{Number: noted, Kind: LinkMilestone, Target: "47/R6"}, + {Number: noted, Kind: LinkCommit, Target: "mesh-catalog@" + strings.Repeat("a", 40)}} { + if _, err := h.link("seed@shanks", l); err != nil { + t.Fatal(err) + } + } + if after, _ := h.Get(noted); !after.Worker.Active.Equal(worked.Worker.Active) { + t.Fatalf("a milestone link kept the worker current: %v, was %v", after.Worker.Active, worked.Worker.Active) + } start := time.Date(2026, 10, 26, 12, 0, 0, 0, time.UTC) for _, c := range []struct { name string @@ -35,7 +52,7 @@ func TestListSaysWhenEachIssueMovedAndKeepsTheLinkedOnes(t *testing.T) { refuse bool }{ {"numbers say moved", ListAsk{Numbers: []uint64{quiet, noted, linked}}, []uint64{quiet, noted, linked}, - []time.Time{start.Add(2 * time.Hour), start.Add(time.Hour), start.Add(2 * time.Hour)}, false}, + []time.Time{start.Add(2 * time.Hour), start.Add(2 * time.Hour), start.Add(2 * time.Hour)}, false}, {"linked to a decision", ListAsk{Linked: "decision:0493"}, []uint64{linked}, nil, false}, {"linked to a design", ListAsk{Linked: " design:03-DESIGN/01-to-be/47-delivery-from-commit-to-delivered.md"}, []uint64{quiet}, nil, false}, {"linked and numbers together", ListAsk{Linked: "decision:0493", Numbers: []uint64{quiet, noted}}, nil, nil, false}, diff --git a/modules/mesh-issues/cmd/mesh-issues/reads-outside b/modules/mesh-issues/cmd/mesh-issues/reads-outside index f6b2d871..628ce47b 100644 --- a/modules/mesh-issues/cmd/mesh-issues/reads-outside +++ b/modules/mesh-issues/cmd/mesh-issues/reads-outside @@ -1 +1,7 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# every module folder: its manifest, and whether it holds code (go.mod, package.json). +modules/*/module.json +modules/*/go.mod +modules/*/package.json diff --git a/modules/mesh-issues/cmd/mesh-issues/tracker.go b/modules/mesh-issues/cmd/mesh-issues/tracker.go index 2f06b304..03780639 100644 --- a/modules/mesh-issues/cmd/mesh-issues/tracker.go +++ b/modules/mesh-issues/cmd/mesh-issues/tracker.go @@ -675,7 +675,9 @@ func (t *Tracker) link(by string, a LinkAsk) (*Linked, error) { return fmt.Errorf("issue %d already has the %s link to %s", is.Number, kind, target) } is.Links = append(is.Links[:i:i], is.Links[i+1:]...) - touch(is, by, at) + if !placesOnly(kind) { + touch(is, by, at) + } return nil } } @@ -692,7 +694,9 @@ func (t *Tracker) link(by string, a LinkAsk) (*Linked, error) { return fmt.Errorf("issue %d has %d links, the most one issue keeps", is.Number, MostLinks) } is.Links = append(is.Links, Link{Kind: kind, Target: target, By: by, At: at}) - touch(is, by, at) + if !placesOnly(kind) { + touch(is, by, at) + } return nil }) if err != nil { @@ -768,7 +772,7 @@ type Row struct { } // lastMoved is when an issue last changed, as its own record says: the latest of its opening, its closing, its -// notes, its links, its worker's start and last act, its import and what the delivery last said of each fix. +// notes, its links but those that only place it, its worker's start and last act, its import and what the delivery last said of each fix. func lastMoved(is *Issue) time.Time { at := is.Opened later := func(t time.Time) { @@ -783,7 +787,9 @@ func lastMoved(is *Issue) time.Time { later(n.At) } for _, l := range is.Links { - later(l.At) + if !placesOnly(l.Kind) { + later(l.At) + } } if w := is.Worker; w != nil { later(w.Since) @@ -798,6 +804,10 @@ func lastMoved(is *Issue) time.Time { return at } +// placesOnly says a link of this kind only places the issue (its milestone) or names a commit that fixed it: no work +// on it, so it neither moves the issue nor keeps its worker current (issue 1781: the seed linked 188 issues at once). +func placesOnly(kind string) bool { return kind == LinkMilestone || kind == LinkCommit } + // checkLinked is a list's linked filter, kind:target, said the one way a link of that kind is kept. func checkLinked(s string) (kind, target string, err error) { k, t, ok := strings.Cut(strings.TrimSpace(s), ":") diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go b/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go index ec7b655c..f8882f1d 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/progress.go @@ -168,6 +168,8 @@ func (k *KB) Progress(a ProgressAsk) (*Progress, error) { if err != nil { return nil, err } + // A cut takes the warnings from the last: the calls that failed are said before the answers that did not parse. + callsFirst(p.Warnings) fitProgress(p, boundOf(a.Bound)) return p, nil } @@ -835,8 +837,21 @@ func movedSay(t *time.Time) string { return " · moved " + when(t) } +// callsFirst puts the warnings of calls that failed before those of answers that did not parse, in place. +func callsFirst(w []string) { + sort.SliceStable(w, func(i, j int) bool { + return !strings.Contains(w[i], " answered something else") && strings.Contains(w[j], " answered something else") + }) +} + // fitProgress keeps an answer within its bound: the items go first, then the tree's last lines, saying what was cut. +// A minimal answer never carries the items: its tree says the same, one line per milestone, so leaving them out is how +// the minimal form is made (the items of design 47 alone passed 8192 bytes) and no cut. A detailed answer's items, and +// everything else that goes, are said in cut. func fitProgress(p *Progress, bound int) { + if p.Detail == DetailMinimal && p.Tree != "" { + p.Items = nil + } if size(p) <= bound { return } diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go index 0b6cec47..7c26e289 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/progress_test.go @@ -306,12 +306,23 @@ func TestProgressOfADesignMinimalAndDetailed(t *testing.T) { if len(raw) > DefaultBound || p.Detail != DetailMinimal || !strings.Contains(p.Source, "the issue tracker's milestone records of design 47") { t.Fatalf("%d bytes, %s, %s", len(raw), p.Detail, p.Source) } + if p.Cut != "" || len(p.Items) != 0 { + t.Errorf("a minimal answer leaves the items to its tree and is not a cut: cut %q, %d items", p.Cut, len(p.Items)) + } + warns := []string{"mesh-delivery.show a answered something else: x", "issue-tracker.get 1: refused", "the tracker did not name 2", "issue-tracker.get 3 answered something else: y"} + callsFirst(warns) + if strings.Join(warns, "|") != "issue-tracker.get 1: refused|the tracker did not name 2|mesh-delivery.show a answered something else: x|issue-tracker.get 3 answered something else: y" { + t.Errorf("call failures do not come first: %q", warns) + } if seats.asked["seat:issue-tracker.list"] != 1 || seats.asked["seat:mesh-delivery.deliveries"] != 0 || seats.asked["seat:issue-tracker.get"] != 0 { t.Fatalf("minimal asked %v", seats.asked) } if w := strings.Join(p.Warnings, "\n"); !strings.Contains(w, "the tracker neither answered nor named missing") || !strings.Contains(w, "973, 525") { t.Errorf("a row the tracker did not answer is not said: %v", p.Warnings) } + if big, _ := k.Progress(ProgressAsk{Of: "design 47", Bound: 8192}); big.Cut != "" || big.Tree != p.Tree { + t.Errorf("a minimal answer at bound 8192 is cut: %q", big.Cut) + } lines := strings.Split(p.Tree, "\n") if !strings.HasPrefix(lines[0], "✗ design 47 · Delivery: from a commit to delivered — ") { t.Fatalf("first line %q", lines[0]) diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/reads-outside b/modules/mesh-knowledge/cmd/mesh-knowledge/reads-outside index f6b2d871..8ac753d2 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/reads-outside +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the root's retired lines. +retired-lines diff --git a/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/reads-outside b/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/reads-outside index f6b2d871..828cd198 100644 --- a/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/reads-outside +++ b/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# mesh-mail's mailevent.go, of which this package holds a copy. +modules/mesh-mail/cmd/mesh-mail/mailevent.go diff --git a/modules/mesh-mail-reader/cmd/mesh-mail-reader/reads-outside b/modules/mesh-mail-reader/cmd/mesh-mail-reader/reads-outside index f6b2d871..828cd198 100644 --- a/modules/mesh-mail-reader/cmd/mesh-mail-reader/reads-outside +++ b/modules/mesh-mail-reader/cmd/mesh-mail-reader/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# mesh-mail's mailevent.go, of which this package holds a copy. +modules/mesh-mail/cmd/mesh-mail/mailevent.go diff --git a/modules/mesh-mail/cmd/mesh-mail/reads-outside b/modules/mesh-mail/cmd/mesh-mail/reads-outside index f6b2d871..9088f24e 100644 --- a/modules/mesh-mail/cmd/mesh-mail/reads-outside +++ b/modules/mesh-mail/cmd/mesh-mail/reads-outside @@ -1 +1,5 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# messenger's content rule, of which content_rule.go is a copy. +modules/messenger/cmd/messenger/content.go diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/reads-outside b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/reads-outside index c1b869c0..5fba8c24 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/reads-outside +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/reads-outside @@ -1 +1,7 @@ -These tests read other catalogue modules' files, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# every manifest and the templates the moved modules name (in files/, or templates/). +modules/*/module.json +modules/*/files +modules/*/templates diff --git a/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go b/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go index e266dc3d..aca4fc1d 100644 --- a/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go @@ -5,6 +5,8 @@ package main // key (or a first one) raises `secret-changed` for the machine's core secrets, the same key says nothing. import ( + "git.novox.be/novox/mesh-sdk/go/seal" + "encoding/json" "os" "path/filepath" @@ -100,14 +102,17 @@ authorization { users = [ { user: "novox.mesh-vault", password: "pw", permissions: { publish: { deny: [">"] } - subscribe: { allow: ["mesh.seat.mesh-vault.call.node-key.*"] } + subscribe: { allow: ["mesh.seat.mesh-vault.call.node-key.*", "mesh.seat.mesh-vault.call.own.*"] } allow_responses: true } } { user: "node.ace", password: "pw", permissions: { - publish: { allow: ["mesh.seat.mesh-vault.call.node-key.node_ace"] } + publish: { allow: ["mesh.seat.mesh-vault.call.node-key.node_ace", "mesh.seat.mesh-vault.call.own.node_ace"] } subscribe: { allow: ["_INBOX.node.ace.>"] } } } { user: "node.g14", password: "pw", permissions: { - publish: { allow: ["mesh.seat.mesh-vault.call.node-key.node_g14"] } + publish: { allow: ["mesh.seat.mesh-vault.call.node-key.node_g14", "mesh.seat.mesh-vault.call.own.node_g14"] } subscribe: { allow: ["_INBOX.node.g14.>"] } } } + { user: "controller", password: "pw", permissions: { + publish: { allow: ["mesh.seat.mesh-vault.call.own.controller"] } + subscribe: { allow: ["_INBOX.controller.>"] } } } ] } ` @@ -135,39 +140,48 @@ func TestOnARealBusANodeEngineStatesOnlyItsOwnKey(t *testing.T) { t.Cleanup(s.Shutdown) v := newVault(t, nil) - tool := toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".node-key") + v.Askers = Declared{Read: assignments{"ace.web": `{"node":"ace","module":"web"}`}.read, Store: v.Store} + tools := map[string]toolDef{"node-key": toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".node-key"), + "own": toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".own")} vaultConn := connectAs(t, s.ClientURL(), "novox.mesh-vault") - if _, err := vaultConn.Subscribe("mesh.seat.mesh-vault.call.node-key.*", func(m *nats.Msg) { + serve := func(m *nats.Msg) { parts := strings.Split(m.Subject, ".") var args map[string]any _ = json.Unmarshal(m.Data, &args) answer := busAnswer{} - if got, err := tool.runAs(untoken(parts[len(parts)-1]), args); err != nil { + if got, err := tools[parts[len(parts)-2]].runAs(untoken(parts[len(parts)-1]), args); err != nil { answer.Error = err.Error() } else { answer.Result, _ = json.Marshal(got) } raw, _ := json.Marshal(answer) _ = m.Respond(raw) - }); err != nil { - t.Fatal(err) + } + for _, verb := range []string{"node-key", "own"} { + if _, err := vaultConn.Subscribe("mesh.seat.mesh-vault.call."+verb+".*", serve); err != nil { + t.Fatal(err) + } } _ = vaultConn.Flush() - say := func(user, subject, key string) (string, error) { - raw, _ := json.Marshal(map[string]string{"key": key}) + call := func(user, subject string, args any) (json.RawMessage, error) { + raw, _ := json.Marshal(args) msg, err := connectAs(t, s.ClientURL(), user).Request(subject, raw, 3*time.Second) if err != nil { - return "", err + return nil, err } var a busAnswer _ = json.Unmarshal(msg.Data, &a) if a.Error != "" { - return a.Error, errString(a.Error) + return nil, errString(a.Error) } - return "", nil + return a.Result, nil } - aceKey, _ := aSealingKey(t) + say := func(user, subject, key string) (string, error) { + _, err := call(user, subject, map[string]string{"key": key}) + return "", err + } + aceKey, acePriv := aSealingKey(t) g14Key, _ := aSealingKey(t) if _, err := say("node.ace", "mesh.seat.mesh-vault.call.node-key.node_ace", aceKey); err != nil { t.Fatal(err) @@ -188,4 +202,32 @@ func TestOnARealBusANodeEngineStatesOnlyItsOwnKey(t *testing.T) { if got, held, _ := v.Store.NodeKey("g14"); !held || got != g14Key { t.Errorf("g14's key is %q, %v", got, held) } + + // Issue 1126 (ADR 0547 §2, ADR 0550 §5): `own` on the same bus. node.ace asks its own and gets its module's secret + // sealed to the key it stated, opening with its key alone; g14 cannot ask in ace's name, and asking in its own gets + // none of ace's; the controller is refused, as no node-engine. + if _, err := moveIn(t, v, "node.ace", MoveOf{Node: "ace", Module: "web", Name: "password", Origin: "made", Generation: "g-1", + SealingKey: aceKey}, "ace-web-pw"); err != nil { + t.Fatal(err) + } + raw, err := call("node.ace", "mesh.seat.mesh-vault.call.own.node_ace", map[string]any{}) + var own OwnAnswer + if err != nil || json.Unmarshal(raw, &own) != nil || own.Node != "ace" || len(own.Own) != 1 { + t.Fatalf("ace's own over the bus: %s, %v", raw, err) + } + if value, err := seal.Open(own.Own[0].Sealed, acePriv); err != nil || string(value) != "ace-web-pw" { + t.Errorf("ace's secret does not open with ace's key: %q, %v", value, err) + } + if strings.Contains(string(raw), "ace-web-pw") { + t.Error("ace's secret crossed the bus in the clear") + } + if _, err := call("node.g14", "mesh.seat.mesh-vault.call.own.node_ace", map[string]any{}); err == nil { + t.Error("node.g14 asked ace's own in ace's name") + } + if raw, err := call("node.g14", "mesh.seat.mesh-vault.call.own.node_g14", map[string]any{}); err != nil || strings.Contains(string(raw), "web") { + t.Errorf("g14 asking its own was answered %s, %v", raw, err) + } + if _, err := call("controller", "mesh.seat.mesh-vault.call.own.controller", map[string]any{}); err == nil || !strings.Contains(err.Error(), "not a node-engine") { + t.Errorf("the controller asking own: %v", err) + } } diff --git a/modules/mesh-vault/cmd/mesh-vault/ownnode.go b/modules/mesh-vault/cmd/mesh-vault/ownnode.go new file mode 100644 index 00000000..b873dc44 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/ownnode.go @@ -0,0 +1,288 @@ +package main + +// A machine's node-engine composes its own modules (novox/hq ADR 0547 §2, issue 1126), and asks the vault for their own +// secrets sealed to its own key (ADR 0349 §11, ADR 0550 §5): the verb `own`, answered to the node-engine of a machine +// alone, about that machine alone (the machine is the caller's, as the bus proved it, never an argument). Each value is +// sealed to the sealing key the machine's engine stated (`node-key`), so what crosses the bus opens on that machine only, +// and the composition carries it sealed (no composition carries a secret in the clear). A module handed to the +// provisioner is left out: its client fills its secrets from `get` (ADR 0349 §17). + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// NodeOwnSecret is one own secret of a module on the asking machine, sealed to the machine's key. +type NodeOwnSecret struct { + Module string `json:"module"` + Name string `json:"name"` + Origin string `json:"origin"` + Generation int `json:"generation"` + Sealed seal.Box `json:"sealed"` +} + +// OwnAnswer is what `own` answers: the machine, the sealing key every value is sealed to (raw X25519, base64, as its +// engine stated it), each own secret of its modules, and each pair credential it asked for that the vault holds. +type OwnAnswer struct { + Node string `json:"node"` + Key string `json:"key"` + Own []NodeOwnSecret `json:"own"` + Credentials []NodeCredential `json:"credentials,omitempty"` + Missing []CredentialAsked `json:"missing,omitempty"` +} + +// CredentialAsked names one pair credential by its ends, as a node composing its own asks for it (mesh-sdk +// compose.CredentialKey): Role is `consumer` when the asking machine consumes it, `provider` when it grants it. +type CredentialAsked struct { + Role string `json:"role"` + Provision string `json:"provision"` + Consumer string `json:"consumer"` + ConsumerModule string `json:"consumer_module"` + Provider string `json:"provider"` + Local string `json:"local,omitempty"` +} + +// NodeCredential is one pair credential answered, sealed to the asking machine's key, with the name the vault keeps it +// under (`pair/...` per pair, `provision////` issued for one holder, or the provision's +// one shared value `provision//`). +type NodeCredential struct { + CredentialAsked + Name string `json:"name"` + Generation int `json:"generation"` + Sealed seal.Box `json:"sealed"` +} + +// credentialNames are the names a pair credential may be kept under, the most particular first. +func credentialNames(c CredentialAsked) []string { + pair := "pair/" + c.Provider + "/" + c.Provision + "/" + c.Consumer + "/" + c.ConsumerModule + if c.Local != "" { + pair += "/" + c.Local + } + return []string{pair, "provision/" + c.Provider + "/" + c.Provision + "/" + c.Consumer + "/" + c.ConsumerModule, + "provision/" + c.Provider + "/" + c.Provision} +} + +// credentialRefusal says why node may not have c, or "": a consumer's credential to the machine its consuming module +// stands on, a provider's to the providing machine, and nothing to a machine that is neither. +func (v *Vault) credentialRefusal(node string, c CredentialAsked) string { + for _, part := range []string{c.Provision, c.Consumer, c.ConsumerModule, c.Provider} { + if !namePart.MatchString(part) { + return fmt.Sprintf("%q is not a name the mesh writes", part) + } + } + if c.Local != "" && !namePart.MatchString(c.Local) { + return fmt.Sprintf("%q is not a name the mesh writes", c.Local) + } + switch c.Role { + case "consumer": + if c.Consumer != node { + return fmt.Sprintf("%s asks as a consumer on %s, and is not that machine", node, c.Consumer) + } + // The consuming module stands on the machine: read, never assumed (an assignment that cannot be read refuses). + read, why := v.assignmentReader(node) + if why != "" { + return why + } + raw, err := read(node + "." + c.ConsumerModule) + if err != nil { + return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", node, c.ConsumerModule, err) + } + if raw == nil { + return fmt.Sprintf("%s is not on %s: neither assigned there nor required by a module there", c.ConsumerModule, node) + } + return "" + case "provider": + if c.Provider != node { + return fmt.Sprintf("%s asks as the provider %s, and is not that machine", node, c.Provider) + } + return "" + } + return fmt.Sprintf("the role %q is not consumer or provider", c.Role) +} + +// assignmentReader is the vault's reader of the assignments, or why it has none: the rule `own` keeps reads them, +// and without them it refuses rather than hands (a rule that cannot be read is no rule). +func (v *Vault) assignmentReader(node string) (AssignmentReader, string) { + d, ok := v.Askers.(Declared) + if !ok || d.Read == nil { + return nil, "the vault cannot read the assignments, so it cannot tell what " + node + "'s modules are; nothing is handed" + } + return standing(d.Read), "" +} + +// sharedRefusal says why node may not have the provision's one shared value `provision//` for c, +// or "": the vault's record of who holds it names the consumer's module among its consumers, or the machine as its +// provider. Without that record nothing is handed: the shared value is not one any machine may have by naming a +// provider (ADR 0550 §5). +func (v *Vault) sharedRefusal(node string, c CredentialAsked) (string, error) { + name := "provision/" + c.Provider + "/" + c.Provision + sh, ok, err := v.Store.ReadShared(name) + if err != nil { + return "", err + } + if !ok { + return fmt.Sprintf("the vault keeps no record of who holds %s", name), nil + } + switch c.Role { + case "consumer": + want := consumerOf(c.Consumer, c.ConsumerModule, c.Local) + for _, held := range sh.Consumers { + if held == want { + return "", nil + } + } + return fmt.Sprintf("%s is held by %s, not by %s", name, strings.Join(sh.Consumers, ", "), want), nil + case "provider": + if sh.Provider == node { + return "", nil + } + return fmt.Sprintf("%s is provided from %s, not %s", name, sh.Provider, node), nil + } + return fmt.Sprintf("the role %q is not consumer or provider", c.Role), nil +} + +// composeOrigin is the vault's origin of a value as the composition reads it: made by the mesh, or given (accepted at a +// terminal, typed at a desk, issued by a provider's software). +func composeOrigin(origin string) string { + if origin == "" || origin == OriginMade { + return OriginMade + } + return OriginAccepted +} + +// OwnOfNode answers the node-engine of a machine the own secrets of its modules, each sealed to the machine's stated key. +func (v *Vault) OwnOfNode(caller string, asked ...CredentialAsked) (OwnAnswer, error) { + node, ok := nodeEngineOf(caller) + if !ok { + if caller == "" || caller == stdio.Unproven { + caller = "an unproven caller" + } + return OwnAnswer{}, fmt.Errorf("own answers a machine's node-engine alone, about its own machine; %s is not a node-engine", caller) + } + key, had, err := v.Store.NodeKey(node) + if err != nil { + return OwnAnswer{}, err + } + if !had { + return OwnAnswer{}, fmt.Errorf("the vault holds no secret named own/%s yet: %s's node-engine has stated no sealing key "+ + "(node-key), so there is no key of its own to seal to", node, node) + } + to, err := publicPEM(key) + if err != nil { + return OwnAnswer{}, fmt.Errorf("%s's sealing key as recorded is not usable: %w", node, err) + } + all, err := v.Store.List() + if err != nil { + return OwnAnswer{}, err + } + out := OwnAnswer{Node: node, Key: key} + for _, r := range all { + parts := strings.Split(r.Name, "/") + if len(parts) != 4 || parts[0] != "own" || parts[1] != node { + continue + } + handed, err := v.handed(node, parts[2]) + if err != nil { + return OwnAnswer{}, err + } + if handed { + continue + } + value, err := seal.Open(r.AtRest, v.Own.PrivateKey) + if err != nil { + return OwnAnswer{}, fmt.Errorf("%s does not open with the vault's key: %w", r.Name, err) + } + sealed, err := seal.Seal(value, to) + if err != nil { + return OwnAnswer{}, err + } + if err := v.recordHanding(Handing{To: caller, Name: r.Name, Generation: r.Generation, Outcome: "handed"}); err != nil { + return OwnAnswer{}, fmt.Errorf("%s is not handed out: who was handed it could not be recorded: %w", r.Name, err) + } + out.Own = append(out.Own, NodeOwnSecret{Module: parts[2], Name: parts[3], Origin: composeOrigin(r.Origin), + Generation: r.Generation, Sealed: sealed}) + } + sort.Slice(out.Own, func(i, j int) bool { + if out.Own[i].Module != out.Own[j].Module { + return out.Own[i].Module < out.Own[j].Module + } + return out.Own[i].Name < out.Own[j].Name + }) + for _, c := range asked { + if why := v.credentialRefusal(node, c); why != "" { + return OwnAnswer{}, fmt.Errorf("%s may not ask for the credential of %s from %s to %s on %s: %s", caller, + c.Provision, c.Provider, c.ConsumerModule, c.Consumer, why) + } + found := false + names := credentialNames(c) + for i, name := range names { + r, held, err := v.Store.Read(name) + if err != nil { + return OwnAnswer{}, err + } + if !held { + continue + } + if i == len(names)-1 { + // The provision's one shared value: handed only to a machine the record of its holders names. + why, err := v.sharedRefusal(node, c) + if err != nil { + return OwnAnswer{}, err + } + if why != "" { + v.log("%s is not handed %s: %s", node, name, why) + continue + } + } + value, err := seal.Open(r.AtRest, v.Own.PrivateKey) + if err != nil { + return OwnAnswer{}, fmt.Errorf("%s does not open with the vault's key: %w", name, err) + } + sealed, err := seal.Seal(value, to) + if err != nil { + return OwnAnswer{}, err + } + if err := v.recordHanding(Handing{To: caller, Name: name, Generation: r.Generation, Outcome: "handed"}); err != nil { + return OwnAnswer{}, fmt.Errorf("%s is not handed out: who was handed it could not be recorded: %w", name, err) + } + out.Credentials = append(out.Credentials, NodeCredential{CredentialAsked: c, Name: name, Generation: r.Generation, Sealed: sealed}) + found = true + break + } + if !found { + out.Missing = append(out.Missing, c) + } + } + v.log("handed %s's node-engine the %d own secrets of its modules and %d credentials (%d not held), sealed to its key", + node, len(out.Own), len(out.Credentials), len(out.Missing)) + return out, nil +} + +// handed says the plan of module on node hands it to the provisioner, so its secrets are filled from `get`, not composed. +// A plan that cannot be read is an error, never "not handed": `own` refuses rather than hands a secret the provisioner +// holds. +func (v *Vault) handed(node, module string) (bool, error) { + read, why := v.assignmentReader(node) + if why != "" { + return false, errors.New(why) + } + raw, err := read(node + "." + module) + if err != nil { + return false, fmt.Errorf(notYetPrefix+"the assignment %s.%s could not be read, so whether it is handed to the "+ + "provisioner is not known: %w", node, module, err) + } + if raw == nil { + return false, nil + } + var a plannedSecrets + if err := json.Unmarshal(raw, &a); err != nil { + return false, fmt.Errorf("the assignment %s.%s is not the controller's: %w", node, module, err) + } + return a.Provisioning != nil, nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go b/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go new file mode 100644 index 00000000..131e4455 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// Issue 1126 (ADR 0547 §2, ADR 0550 §5): `own` answers a machine's node-engine the own secrets of its modules, sealed to +// the key that machine's engine stated, about that machine alone; another caller, another machine's secrets and a +// machine with no stated key get nothing. +func TestOwnAnswersAMachineItsModulesSecretsSealedToItsKey(t *testing.T) { + plans := assignments{"anchor.web": `{"node":"anchor","module":"web"}`, "laptop.web": `{"node":"laptop","module":"web"}`} + v := newVault(t, nil) + v.Askers = Declared{Read: plans.read, Store: v.Store} + anchorPub, anchorPriv := aSealingKey(t) + laptopPub, _ := aSealingKey(t) + for _, m := range []struct { + caller string + of MoveOf + value string + }{ + {"node.anchor", MoveOf{Node: "anchor", Module: "web", Name: "password", Origin: "made", Generation: "g-1", SealingKey: anchorPub}, "anchor-web-pw"}, + {"node.anchor", MoveOf{Node: "anchor", Module: "web", Name: "api-key", Origin: "accepted", Generation: "g-1", SealingKey: anchorPub}, "anchor-web-key"}, + {"node.laptop", MoveOf{Node: "laptop", Module: "web", Name: "password", Origin: "made", Generation: "g-1", SealingKey: laptopPub}, "laptop-web-pw"}, + } { + if _, err := moveIn(t, v, m.caller, m.of, m.value); err != nil { + t.Fatal(err) + } + } + got, err := v.OwnOfNode("node.anchor") + if err != nil { + t.Fatal(err) + } + if got.Node != "anchor" || got.Key != anchorPub || len(got.Own) != 2 { + t.Fatalf("anchor was answered %+v", got) + } + want := map[string][2]string{"api-key": {"accepted", "anchor-web-key"}, "password": {"made", "anchor-web-pw"}} + for _, s := range got.Own { + w, ok := want[s.Name] + if !ok || s.Module != "web" || s.Origin != w[0] || s.Generation < 1 { + t.Errorf("anchor's %s: %+v", s.Name, s) + continue + } + value, err := seal.Open(s.Sealed, anchorPriv) + if err != nil || string(value) != w[1] { + t.Errorf("anchor's %s does not open to its value with anchor's key: %q, %v", s.Name, value, err) + } + } + for caller, says := range map[string]string{ + "controller": "is not a node-engine", + "anchor.web": "is not a node-engine", + "": "an unproven caller", + "node.unknown": "no secret named own/unknown yet", + } { + if _, err := v.OwnOfNode(caller); err == nil || !strings.Contains(err.Error(), says) { + t.Errorf("%q: %v, want a refusal saying %q", caller, err, says) + } + } + if laptop, err := v.OwnOfNode("node.laptop"); err != nil || len(laptop.Own) != 1 || laptop.Own[0].Name != "password" { + t.Errorf("laptop was answered %+v, %v; want its one secret alone", laptop, err) + } + + // A pair credential: laptop's web consumes anchor's postgres. Each end is answered the one value sealed to its own key; + // a credential the vault holds none of is named missing; a machine at neither end is refused. + if _, err := moveIn(t, v, "node.laptop", MoveOf{Provider: "anchor", Provision: "postgres", Consumer: "laptop", + ConsumerModule: "web", Origin: "made", Generation: "g-1", SealingKey: laptopPub}, "pg-pair"); err != nil { + t.Fatal(err) + } + pair := CredentialAsked{Provision: "postgres", Consumer: "laptop", ConsumerModule: "web", Provider: "anchor"} + asConsumer, asProvider := pair, pair + asConsumer.Role, asProvider.Role = "consumer", "provider" + none := CredentialAsked{Role: "consumer", Provision: "redis", Consumer: "laptop", ConsumerModule: "web", Provider: "anchor"} + laptop, err := v.OwnOfNode("node.laptop", asConsumer, none) + if err != nil || len(laptop.Credentials) != 1 || len(laptop.Missing) != 1 || laptop.Missing[0] != none { + t.Fatalf("laptop as consumer: %+v, %v", laptop, err) + } + if !strings.HasPrefix(laptop.Credentials[0].Name, "pair/anchor/postgres/laptop/web") { + t.Errorf("laptop's credential is kept as %s", laptop.Credentials[0].Name) + } + anchor, err := v.OwnOfNode("node.anchor", asProvider) + if err != nil || len(anchor.Credentials) != 1 { + t.Fatalf("anchor as provider: %+v, %v", anchor, err) + } + if value, err := seal.Open(anchor.Credentials[0].Sealed, anchorPriv); err != nil || string(value) != "pg-pair" { + t.Errorf("anchor's copy does not open to the value with its key: %q, %v", value, err) + } + for who, c := range map[string]CredentialAsked{"node.anchor": asConsumer, "node.laptop": asProvider} { + if _, err := v.OwnOfNode(who, c); err == nil || !strings.Contains(err.Error(), "is not that machine") { + t.Errorf("%s asking %+v: %v", who, c, err) + } + } + + // Review of L2 (C1, C2): `own` fails closed. A provision's one shared value is handed only to a consumer the record + // of its holders names, never to a machine that names its provider; assignments that cannot be read, or a vault + // that reads none, refuse rather than hand. + if _, err := v.keep("provision/anchor/redis", "shared-redis", "test", "", OriginMade); err != nil { + t.Fatal(err) + } + if err := v.Store.WriteShared(Shared{Name: "provision/anchor/redis", Provider: "anchor", Provision: "redis", + Consumers: []string{"anchor/web"}}); err != nil { + t.Fatal(err) + } + if got, err := v.OwnOfNode("node.laptop", none); err != nil || len(got.Credentials) != 0 || len(got.Missing) != 1 { + t.Errorf("laptop, whose web the shared value's record does not name, got %+v, %v", got, err) + } + anchorRedis := CredentialAsked{Role: "consumer", Provision: "redis", Consumer: "anchor", ConsumerModule: "web", Provider: "anchor"} + if got, err := v.OwnOfNode("node.anchor", anchorRedis); err != nil || len(got.Credentials) != 1 { + t.Errorf("anchor's web, a holder the record names, got %+v, %v", got, err) + } + v.Askers = Declared{Read: func(string) ([]byte, error) { return nil, errors.New("the bucket did not answer") }, Store: v.Store} + if _, err := v.OwnOfNode("node.anchor"); err == nil || !strings.Contains(err.Error(), "could not be read") { + t.Errorf("own with the assignments unread: %v", err) + } + v.Askers = nil + if _, err := v.OwnOfNode("node.anchor", anchorRedis); err == nil || !strings.Contains(err.Error(), "cannot read the assignments") { + t.Errorf("own with no reader of the assignments: %v", err) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/tools.go b/modules/mesh-vault/cmd/mesh-vault/tools.go index 1bd6fe9f..2ef64c8d 100644 --- a/modules/mesh-vault/cmd/mesh-vault/tools.go +++ b/modules/mesh-vault/cmd/mesh-vault/tools.go @@ -251,6 +251,34 @@ func definitions(env *Env) []toolDef { // clause 9, adopt.go): a public half, never the private one. return map[string]any{"recorded": true, "vault_key": v.Own.PublicKey}, nil }}, + {Name: Seat + ".own", + Description: "The own secrets of this machine's modules, each sealed to the machine's sealing key (novox/hq ADR 0547 §2, " + + "ADR 0550 §5, issue 1126): from the node-engine of the machine alone, about its own machine (the bus proves which), " + + "for the compositions it makes itself. Answers the machine, the key each value is sealed to, and per module and " + + "name the origin, the generation and the sealed value; a module handed to the provisioner is left out. With credentials, " + + "also each pair credential the machine consumes or grants, sealed the same way. Never a value in the clear.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "credentials": map[string]any{"type": "array", "description": "the pair credentials the machine's composition " + + "reads, each {role: consumer|provider, provision, consumer, consumer_module, provider, local}: answered " + + "sealed to the machine's key when the machine is that consumer's or that provider, and named missing when " + + "the vault holds none", "items": map[string]any{"type": "object"}}}}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + var asked []CredentialAsked + if raw, ok := a["credentials"]; ok && raw != nil { + b, err := json.Marshal(raw) + if err == nil { + err = json.Unmarshal(b, &asked) + } + if err != nil { + return nil, fmt.Errorf("credentials is a list of {role, provision, consumer, consumer_module, provider, local}: %w", err) + } + } + return v.OwnOfNode(caller, asked...) + }}, {Name: Seat + ".move-asks", Description: "Copy rows of the controller's record of asks at a desk into the vault's, as they are and once " + "(design 57's V3): from the controller alone, at most 500 at a time. No value is in any of them.", diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index cdff2835..304b95df 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -177,7 +177,8 @@ "rotate", "move-in", "node-key", - "move-asks" + "move-asks", + "own" ] } ] diff --git a/modules/minio/cmd/object-store/reads-outside b/modules/minio/cmd/object-store/reads-outside index c1b869c0..756af267 100644 --- a/modules/minio/cmd/object-store/reads-outside +++ b/modules/minio/cmd/object-store/reads-outside @@ -1 +1,5 @@ -These tests read other catalogue modules' files, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# brother's store.json, against the object store's. +modules/brother/files/store.json diff --git a/modules/networkmanager/cmd/uplink-tools/reads-outside b/modules/networkmanager/cmd/uplink-tools/reads-outside index f6b2d871..ceb6d7da 100644 --- a/modules/networkmanager/cmd/uplink-tools/reads-outside +++ b/modules/networkmanager/cmd/uplink-tools/reads-outside @@ -1 +1,6 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of uplink-tools in both modules that hold the uplink. +modules/networkmanager/cmd/uplink-tools/*.go +modules/systemd-networkd/cmd/uplink-tools/*.go diff --git a/modules/nextcloud-client/cmd/nextcloud-client-tools/reads-outside b/modules/nextcloud-client/cmd/nextcloud-client-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/nextcloud-client/cmd/nextcloud-client-tools/reads-outside +++ b/modules/nextcloud-client/cmd/nextcloud-client-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/nm-applet/cmd/nm-applet-tools/reads-outside b/modules/nm-applet/cmd/nm-applet-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/nm-applet/cmd/nm-applet-tools/reads-outside +++ b/modules/nm-applet/cmd/nm-applet-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/openrazer/cmd/openrazer-tools/reads-outside b/modules/openrazer/cmd/openrazer-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/openrazer/cmd/openrazer-tools/reads-outside +++ b/modules/openrazer/cmd/openrazer-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/pacman/cmd/pacman-tools/reads-outside b/modules/pacman/cmd/pacman-tools/reads-outside index 2c69a776..856f5874 100644 --- a/modules/pacman/cmd/pacman-tools/reads-outside +++ b/modules/pacman/cmd/pacman-tools/reads-outside @@ -1 +1,5 @@ -These tests read the repository's root predefined.json, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the root's predefined keys. +predefined.json diff --git a/modules/polychromatic/cmd/polychromatic-tools/reads-outside b/modules/polychromatic/cmd/polychromatic-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/polychromatic/cmd/polychromatic-tools/reads-outside +++ b/modules/polychromatic/cmd/polychromatic-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/postgres/cmd/postgres-provider/reads-outside b/modules/postgres/cmd/postgres-provider/reads-outside index c1b869c0..1aaa7f89 100644 --- a/modules/postgres/cmd/postgres-provider/reads-outside +++ b/modules/postgres/cmd/postgres-provider/reads-outside @@ -1 +1,5 @@ -These tests read other catalogue modules' files, outside this module (issue 1935): never kept. +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# every manifest. +modules/*/module.json diff --git a/modules/slack/cmd/slack-tools/reads-outside b/modules/slack/cmd/slack-tools/reads-outside index f6b2d871..72d043ba 100644 --- a/modules/slack/cmd/slack-tools/reads-outside +++ b/modules/slack/cmd/slack-tools/reads-outside @@ -1 +1,25 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of desktop.go and its tests in every module that carries one. +modules/blueman/cmd/blueman-tools/desktop.go +modules/blueman/cmd/blueman-tools/desktop_test.go +modules/blueman/cmd/blueman-tools/copies_test.go +modules/forticlient/cmd/forticlient-tools/desktop.go +modules/forticlient/cmd/forticlient-tools/desktop_test.go +modules/forticlient/cmd/forticlient-tools/copies_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop.go +modules/nextcloud-client/cmd/nextcloud-client-tools/desktop_test.go +modules/nextcloud-client/cmd/nextcloud-client-tools/copies_test.go +modules/nm-applet/cmd/nm-applet-tools/desktop.go +modules/nm-applet/cmd/nm-applet-tools/desktop_test.go +modules/nm-applet/cmd/nm-applet-tools/copies_test.go +modules/openrazer/cmd/openrazer-tools/desktop.go +modules/openrazer/cmd/openrazer-tools/desktop_test.go +modules/openrazer/cmd/openrazer-tools/copies_test.go +modules/polychromatic/cmd/polychromatic-tools/desktop.go +modules/polychromatic/cmd/polychromatic-tools/desktop_test.go +modules/polychromatic/cmd/polychromatic-tools/copies_test.go +modules/slack/cmd/slack-tools/desktop.go +modules/slack/cmd/slack-tools/desktop_test.go +modules/slack/cmd/slack-tools/copies_test.go diff --git a/modules/systemd-networkd/cmd/uplink-tools/reads-outside b/modules/systemd-networkd/cmd/uplink-tools/reads-outside index f6b2d871..ceb6d7da 100644 --- a/modules/systemd-networkd/cmd/uplink-tools/reads-outside +++ b/modules/systemd-networkd/cmd/uplink-tools/reads-outside @@ -1 +1,6 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of uplink-tools in both modules that hold the uplink. +modules/networkmanager/cmd/uplink-tools/*.go +modules/systemd-networkd/cmd/uplink-tools/*.go diff --git a/modules/xorg/internal/desktop/reads-outside b/modules/xorg/internal/desktop/reads-outside index f6b2d871..7ef55c53 100644 --- a/modules/xorg/internal/desktop/reads-outside +++ b/modules/xorg/internal/desktop/reads-outside @@ -1 +1,9 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of this package in every module that carries one. +modules/xorg/internal/desktop/*.go +modules/lemurs/internal/desktop/*.go +modules/i3/internal/desktop/*.go +modules/xterm/internal/desktop/*.go +modules/adwaita/internal/desktop/*.go diff --git a/modules/xterm/internal/desktop/reads-outside b/modules/xterm/internal/desktop/reads-outside index f6b2d871..7ef55c53 100644 --- a/modules/xterm/internal/desktop/reads-outside +++ b/modules/xterm/internal/desktop/reads-outside @@ -1 +1,9 @@ -These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935). +# What these tests read outside their package, as paths from the repository's root (a glob, or a directory read whole; +# ? before a path whose absence is read). mesh-build-go hashes them into the package's digest, so its verdict is kept +# until one of them changes (issue 1936). An empty list, or a line matching nothing, keeps nothing. +# the copies of this package in every module that carries one. +modules/xorg/internal/desktop/*.go +modules/lemurs/internal/desktop/*.go +modules/i3/internal/desktop/*.go +modules/xterm/internal/desktop/*.go +modules/adwaita/internal/desktop/*.go