Compare commits
114
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8bc54a3e27 | ||
|
|
15378cfba7 | ||
|
|
539a31103b | ||
|
|
dfcdf234c5 | ||
|
|
5bc9578634 | ||
|
|
3ca48ff230 | ||
|
|
6c0024e43b | ||
|
|
a2e5a20f52 | ||
|
|
845479d7cd | ||
|
|
2ebcb18097 | ||
|
|
00e54d46d7 | ||
|
|
933370df7c | ||
|
|
437c524b81 | ||
|
|
dc96d7c2dd | ||
|
|
f9247342d1 | ||
|
|
b1d875410a | ||
|
|
89e15c7bcd | ||
|
|
a21da33d48 | ||
|
|
b53b0e7e2c | ||
|
|
48dd41db91 | ||
|
|
16331cdfdc | ||
|
|
ab3722d3b3 | ||
|
|
66c404fd39 | ||
|
|
3ad9d42be9 | ||
|
|
083515da1b | ||
|
|
ac5d257a1c | ||
|
|
896bd88380 | ||
|
|
24f50c7c86 | ||
|
|
e0bf87c2e3 | ||
|
|
ef1e98f867 | ||
|
|
e3db481491 | ||
|
|
21b4925892 | ||
|
|
098c6d3ec7 | ||
|
|
0fdfecfa29 | ||
|
|
97f0f4813c | ||
|
|
8d3f81c2c0 | ||
|
|
8f10d3192c | ||
|
|
1e3439429f | ||
|
|
e9e8c7c901 | ||
|
|
db35f2b0cb | ||
|
|
f90d3bfcc0 | ||
|
|
37c47f8f95 | ||
|
|
a45e467ddd | ||
|
|
d3f9a1855d | ||
|
|
c97cea21cf | ||
|
|
2f05e70734 | ||
|
|
5ab9d6c296 | ||
|
|
e61c5ee5bf | ||
|
|
b5f812270e | ||
|
|
a18eeeef5a | ||
|
|
5b6dfc70f4 | ||
|
|
3cce15fa81 | ||
|
|
86ba54b06e | ||
|
|
af91a7e660 | ||
|
|
d45767bb2f | ||
|
|
027ea030fa | ||
|
|
6eb9eb53de | ||
|
|
0270027bf9 | ||
|
|
835bf22b7d | ||
|
|
e85b7da63b | ||
|
|
c112df0c1c | ||
|
|
8e791896f1 | ||
|
|
b07a101caf | ||
|
|
72effa1fb1 | ||
|
|
e53ac973e9 | ||
|
|
f1f18bf2bd | ||
|
|
c41c968086 | ||
|
|
939475cb63 | ||
|
|
5e24deb0f1 | ||
|
|
76a0a726f0 | ||
|
|
df821c8a4a | ||
|
|
f40269caec | ||
|
|
2149d8f3e7 | ||
|
|
e65c8c8961 | ||
|
|
ea8e0def6e | ||
|
|
f8ac498d64 | ||
|
|
276422a85b | ||
|
|
28ef6e9e27 | ||
|
|
f339d0aab7 | ||
|
|
43fccb843c | ||
|
|
f550f1c6b2 | ||
|
|
71f47c3291 | ||
|
|
dc703d1511 | ||
|
|
d70135bf08 | ||
|
|
ff0f43ea2f | ||
|
|
2b2517e4a5 | ||
|
|
b7a4a1a6b0 | ||
|
|
c1a9ad2abc | ||
|
|
c93f77ac19 | ||
|
|
75f3f52589 | ||
|
|
900a2da3e3 | ||
|
|
b7f3664839 | ||
|
|
9511074f40 | ||
|
|
9651a8b2da | ||
|
|
32c66a98c1 | ||
|
|
62437f8964 | ||
|
|
ac4d156cb7 | ||
|
|
95e69276c3 | ||
|
|
56b285d3e8 | ||
|
|
5cc401683c | ||
|
|
79ce72a7b7 | ||
|
|
f40fa6e8e0 | ||
|
|
b015576a13 | ||
|
|
e4fb4cc793 | ||
|
|
1de9a903c8 | ||
|
|
0bb6584437 | ||
|
|
c0bc0d4826 | ||
|
|
efac1b4856 | ||
|
|
bc77afa6bb | ||
|
|
e4373768af | ||
|
|
2cb2e27cdf | ||
|
|
0f4e631cdd | ||
|
|
802c72e4ed | ||
|
|
eb717cc514 |
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/locks
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,105 @@
|
||||
// Command locks refuses a TypeScript module that names packages beyond the SDK in its package.json and commits no
|
||||
// package-lock.json (novox/hq ADR 0469 §5, issue 1868): the Node build module installs a module's own packages only
|
||||
// from its committed lock, so such a module is refused by the builder and could not be delivered.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding. The packages that count are those the build module counts (mesh-build-node fetch.go
|
||||
// ownDependencies): the `dependencies` beyond the SDK, typescript and the @types packages.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Finding is one module refused: it names Packages beyond the SDK and has no lock file.
|
||||
type Finding struct {
|
||||
Module string
|
||||
Packages []string
|
||||
}
|
||||
|
||||
// Check reads every module under root/modules that builds a TypeScript artifact.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Finding
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Language string `json:"language"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
typescript := false
|
||||
for _, a := range m.Build.Artifacts {
|
||||
typescript = typescript || a.Language == "typescript"
|
||||
}
|
||||
if !typescript {
|
||||
continue
|
||||
}
|
||||
dir := filepath.Dir(path)
|
||||
pkgRaw, err := os.ReadFile(filepath.Join(dir, "package.json"))
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pkg struct {
|
||||
Dependencies map[string]json.RawMessage `json:"dependencies"`
|
||||
}
|
||||
if err := json.Unmarshal(pkgRaw, &pkg); err != nil {
|
||||
return nil, fmt.Errorf("%s: package.json: %w", m.Module, err)
|
||||
}
|
||||
var names []string
|
||||
for n := range pkg.Dependencies {
|
||||
if n != "@novox/mesh-sdk" && n != "typescript" && !strings.HasPrefix(n, "@types/") {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "package-lock.json")); err != nil {
|
||||
sort.Strings(names)
|
||||
out = append(out, Finding{Module: m.Module, Packages: names})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Printf("%s names packages beyond the SDK (%s) and has no package-lock.json: commit one made by the build module's toolchain (novox/hq ADR 0469 §5)\n", f.Module, strings.Join(f.Packages, ", "))
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("locks: every TypeScript module that names packages beyond the SDK commits its lock")
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// module writes modules/<name> with a TypeScript artifact, the package.json given ("" for none) and, when lock, a lock.
|
||||
func module(t *testing.T, root, name, language, pkg string, lock bool) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", name)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write := func(file, body string) {
|
||||
if err := os.WriteFile(filepath.Join(dir, file), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write("module.json", `{"module":"`+name+`","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"`+language+`","from":"index.ts"}]}}`)
|
||||
if pkg != "" {
|
||||
write("package.json", pkg)
|
||||
}
|
||||
if lock {
|
||||
write("package-lock.json", "{}")
|
||||
}
|
||||
}
|
||||
|
||||
const sdkOnly = `{"dependencies":{"@novox/mesh-sdk":"^0.1.1"},"devDependencies":{"typescript":"^5","@types/node":"^22"}}`
|
||||
const withPg = `{"dependencies":{"@novox/mesh-sdk":"^0.1.1","pg":"^8","@types/pg":"^8"}}`
|
||||
|
||||
func TestAModuleNamingOnlyTheSDKNeedsNoLock(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "plain", "typescript", sdkOnly, false)
|
||||
module(t, root, "none", "typescript", "", false)
|
||||
got, err := Check(root)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("%v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleNamingMorePackagesWithoutALockIsRefusedNamingThem(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "db", "typescript", withPg, false)
|
||||
module(t, root, "db-locked", "typescript", withPg, true)
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []Finding{{Module: "db", Packages: []string{"pg"}}}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleThatIsNotTypeScriptIsNotJudged(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "go", "go", withPg, false)
|
||||
if got, err := Check(root); err != nil || len(got) != 0 {
|
||||
t.Fatalf("%v %v", got, err)
|
||||
}
|
||||
}
|
||||
+181
-13
@@ -11,13 +11,23 @@
|
||||
// another module, after ${dir:<id>} is resolved within each module and both homes are read as one root.
|
||||
// Archives are not containers for this rule. This holds for ~/.ssh what checks/sshdir held (ADR 0415 §6).
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
// 5. a directory that is, lies inside or holds a directory a system package owns under /var/lib, or any other
|
||||
// resource inside one (issue 1841: pacman's state directory, placed with "place": ".", was /var/lib/pacman, the
|
||||
// package manager's own database directory). A placed directory is resolved as the mesh resolves it with the
|
||||
// default data root: "." is /var/lib/<module>, "mesh" /var/lib/mesh/<module>, neither /var/lib/<module>/<id>.
|
||||
// The system directories are checks/owned/system, captured from a machine (pacman -Qo, /usr/lib/tmpfiles.d);
|
||||
// checks/owned/system-known names the exceptions still to fix, by module, resource and issue. That list only
|
||||
// shrinks: an entry that no longer collides is refused, and with -base one the trunk's copy does not hold.
|
||||
//
|
||||
// go run . [-base <trunk's system-known>] <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
@@ -27,6 +37,12 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The lists of rule 5, relative to the catalogue's root.
|
||||
const (
|
||||
SystemList = "checks/owned/system"
|
||||
SystemKnown = "checks/owned/system-known"
|
||||
)
|
||||
|
||||
// Home is the one root both machine homes are read as.
|
||||
const Home = "${home}"
|
||||
|
||||
@@ -43,9 +59,9 @@ type Finding struct {
|
||||
}
|
||||
|
||||
type resource struct {
|
||||
ID, Type, Path, Unknown string
|
||||
Knows []string
|
||||
HasKnows bool
|
||||
ID, Type, Path, Unknown, Place string
|
||||
Knows []string
|
||||
HasKnows bool
|
||||
}
|
||||
|
||||
type parsed struct {
|
||||
@@ -65,6 +81,7 @@ func load(file string) (*parsed, error) {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Path string `json:"path"`
|
||||
Place string `json:"place"`
|
||||
Unknown *string `json:"unknown"`
|
||||
Knows []string `json:"knows"`
|
||||
} `json:"resources"`
|
||||
@@ -81,7 +98,7 @@ func load(file string) (*parsed, error) {
|
||||
out.Module = filepath.Base(filepath.Dir(file))
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
x := resource{ID: r.ID, Type: r.Type, Path: r.Path, Knows: r.Knows, HasKnows: r.Knows != nil}
|
||||
x := resource{ID: r.ID, Type: r.Type, Path: r.Path, Place: r.Place, Knows: r.Knows, HasKnows: r.Knows != nil}
|
||||
if r.Unknown != nil {
|
||||
x.Unknown = *r.Unknown
|
||||
if x.Unknown == "" {
|
||||
@@ -143,14 +160,23 @@ type declared struct {
|
||||
}
|
||||
|
||||
// Check reads every manifest under root/modules and answers what it refuses.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
func Check(root string, base []string) ([]Finding, error) {
|
||||
files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(files)
|
||||
system, err := readSystem(filepath.Join(root, SystemList))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
known, err := readLines(filepath.Join(root, SystemKnown))
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, err
|
||||
}
|
||||
var out []Finding
|
||||
var paths []declared
|
||||
collided := map[string]bool{}
|
||||
type container struct{ module, path string }
|
||||
var containers []container
|
||||
for _, file := range files {
|
||||
@@ -170,6 +196,7 @@ func Check(root string) ([]Finding, error) {
|
||||
dirs[r.ID] = "/placed/" + m.Module + "/" + r.ID
|
||||
}
|
||||
}
|
||||
out = append(out, inSystem(m, system, known, collided)...)
|
||||
for _, r := range m.Resources {
|
||||
p := norm(r.Path, dirs)
|
||||
if p != "" {
|
||||
@@ -212,6 +239,28 @@ func Check(root string) ([]Finding, error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, k := range known {
|
||||
if !collided[k] {
|
||||
module, id, _ := strings.Cut(k, " ")
|
||||
id, _, _ = strings.Cut(id, " ")
|
||||
out = append(out, Finding{5, module, id, fmt.Sprintf("is listed in %s, and no longer lies in a system "+
|
||||
"package's directory: take the line out, so the list only shrinks", SystemKnown)})
|
||||
}
|
||||
}
|
||||
if base != nil {
|
||||
held := map[string]bool{}
|
||||
for _, b := range base {
|
||||
held[b] = true
|
||||
}
|
||||
for _, k := range known {
|
||||
if !held[k] {
|
||||
module, id, _ := strings.Cut(k, " ")
|
||||
id, _, _ = strings.Cut(id, " ")
|
||||
out = append(out, Finding{5, module, id, fmt.Sprintf("is a new line in %s, which the trunk's copy "+
|
||||
"does not hold: a module's directory is placed outside a system package's, never listed", SystemKnown)})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, d := range paths {
|
||||
for _, c := range containers {
|
||||
if c.module != d.module && (d.path == c.path || strings.HasPrefix(d.path, c.path+"/")) {
|
||||
@@ -222,21 +271,140 @@ func Check(root string) ([]Finding, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
// systemDir is one directory a system package owns, and what says so.
|
||||
type systemDir struct{ Path, Source string }
|
||||
|
||||
// readLines is a list's lines, without comments and blank lines.
|
||||
func readLines(file string) ([]string, error) {
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
found, err := Check(root)
|
||||
var lines []string
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if line = strings.TrimSpace(line); line != "" && !strings.HasPrefix(line, "#") {
|
||||
lines = append(lines, line)
|
||||
}
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// readSystem is the list of system directories: "<absolute path> <package>: <how it was read>" per line. Absent or
|
||||
// empty, the check fails: a list that says nothing would let every module through.
|
||||
func readSystem(file string) ([]systemDir, error) {
|
||||
lines, err := readLines(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rule 5 reads the system packages' directories from %s: %w", file, err)
|
||||
}
|
||||
if len(lines) == 0 {
|
||||
return nil, fmt.Errorf("%s names no system package's directory, so rule 5 would pass everything", file)
|
||||
}
|
||||
var out []systemDir
|
||||
for _, line := range lines {
|
||||
p, source, _ := strings.Cut(line, " ")
|
||||
if !strings.HasPrefix(p, "/var/lib/") || strings.TrimSpace(source) == "" {
|
||||
return nil, fmt.Errorf("%s: %q is not \"/var/lib/<directory> <package>: <how it was read>\"", file, line)
|
||||
}
|
||||
out = append(out, systemDir{path.Clean(p), strings.TrimSpace(source)})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// placedAt is where the mesh places a pathless directory with the default data root (mesh-sdk catalogue dirsFor).
|
||||
func placedAt(module string, r resource) string {
|
||||
switch r.Place {
|
||||
case ".":
|
||||
return "/var/lib/" + module
|
||||
case "mesh":
|
||||
return "/var/lib/mesh/" + module
|
||||
}
|
||||
return "/var/lib/" + module + "/" + r.ID
|
||||
}
|
||||
|
||||
// inSystem is rule 5 for one module: each resource whose resolved path collides with a system package's directory,
|
||||
// unless system-known lists it (then it is marked collided, so a stale line is found).
|
||||
func inSystem(m *parsed, system []systemDir, known []string, collided map[string]bool) []Finding {
|
||||
real := map[string]string{}
|
||||
for _, r := range m.Resources {
|
||||
if r.Type == "directory" && r.Path == "" && r.ID != "" {
|
||||
real[r.ID] = placedAt(m.Module, r)
|
||||
}
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r.ID != "" && r.Path != "" {
|
||||
real[r.ID] = r.Path
|
||||
}
|
||||
}
|
||||
var out []Finding
|
||||
for _, r := range m.Resources {
|
||||
p := ""
|
||||
if r.Path != "" {
|
||||
p = norm(r.Path, real)
|
||||
} else if r.Type == "directory" {
|
||||
p = real[r.ID]
|
||||
}
|
||||
if !strings.HasPrefix(p, "/") {
|
||||
continue
|
||||
}
|
||||
for _, s := range system {
|
||||
inside := p == s.Path || strings.HasPrefix(p, s.Path+"/")
|
||||
holds := r.Type == "directory" && strings.HasPrefix(s.Path, p+"/")
|
||||
if !inside && !holds {
|
||||
continue
|
||||
}
|
||||
listed := ""
|
||||
for _, k := range known {
|
||||
if strings.HasPrefix(k, m.Module+" "+r.ID+" ") || k == m.Module+" "+r.ID {
|
||||
listed = k
|
||||
}
|
||||
}
|
||||
if listed != "" {
|
||||
collided[listed] = true
|
||||
break
|
||||
}
|
||||
how := "is or lies inside"
|
||||
if holds {
|
||||
how = "holds"
|
||||
}
|
||||
out = append(out, Finding{5, m.Module, r.ID, fmt.Sprintf("resolves to %q, which %s %s, the directory of the "+
|
||||
"system package %s; place it under /var/lib/mesh/%s (\"place\": \"mesh\") or a path of its own", p, how,
|
||||
s.Path, s.Source, m.Module)})
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func main() {
|
||||
baseFile := flag.String("base", "", "the trunk's copy of "+SystemKnown+": a line it does not hold is refused")
|
||||
flag.Parse()
|
||||
root := "."
|
||||
if flag.NArg() > 0 {
|
||||
root = flag.Arg(0)
|
||||
}
|
||||
var base []string
|
||||
if *baseFile != "" {
|
||||
lines, err := readLines(*baseFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
base = append([]string{}, lines...)
|
||||
}
|
||||
found, err := Check(root, base)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Printf("%s: %s: %s (rule %d, novox/hq ADR 0422 §8, issue 1024)\n", f.Module, f.ID, f.Why, f.Rule)
|
||||
cite := "novox/hq ADR 0422 §8, issue 1024"
|
||||
if f.Rule == 5 {
|
||||
cite = "issue 1841"
|
||||
}
|
||||
fmt.Printf("%s: %s: %s (rule %d, %s)\n", f.Module, f.ID, f.Why, f.Rule, cite)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("owned paths: every directory and archive says how it treats unknown files; no module declares inside another's directory")
|
||||
fmt.Println("owned paths: every directory and archive says how it treats unknown files; no module declares inside another's directory, or in a system package's")
|
||||
}
|
||||
|
||||
+102
-1
@@ -20,7 +20,10 @@ func manifest(t *testing.T, root, module, body string) {
|
||||
|
||||
func check(t *testing.T, root string) []Finding {
|
||||
t.Helper()
|
||||
found, err := Check(root)
|
||||
if _, err := os.Stat(filepath.Join(root, SystemList)); os.IsNotExist(err) {
|
||||
lists(t, root, "/var/lib/pacman pacman: pacman -Qo\n", "")
|
||||
}
|
||||
found, err := Check(root, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -35,6 +38,20 @@ func only(t *testing.T, found []Finding, rule int, module, id string) {
|
||||
}
|
||||
}
|
||||
|
||||
// lists writes the catalogue's list of system directories and its known exceptions.
|
||||
func lists(t *testing.T, root, system, known string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Join(root, "checks", "owned"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, SystemList), []byte(system), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, SystemKnown), []byte(known), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func dir(id, path, rest string) string {
|
||||
return `{"id":"` + id + `","type":"directory","path":"` + path + `"` + rest + `}`
|
||||
}
|
||||
@@ -145,3 +162,87 @@ func TestPlacedDirectoriesOfTwoModulesAreTheirOwn(t *testing.T) {
|
||||
t.Fatalf("refused %+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
// 1841/AC18: a module's directory that resolves to a /var/lib directory a system package owns, lies inside one or
|
||||
// holds one, is refused, whether placed (place ".", or by default beneath the module's own) or stated; a placed
|
||||
// directory saying place "mesh" lies under /var/lib/mesh and passes (issue 1841: pacman's state was /var/lib/pacman).
|
||||
func TestADirectoryOfASystemPackageIsRefused(t *testing.T) {
|
||||
system := "/var/lib/pacman pacman: pacman -Qo\n/var/lib/systemd/coredump systemd: tmpfiles.d/systemd.conf\n"
|
||||
for name, resources := range map[string][]string{
|
||||
"placed-own": {`{"id":"state","type":"directory","place":".","unknown":"program"}`},
|
||||
"placed-beneath": {`{"id":"coredump","type":"directory","unknown":"program"}`},
|
||||
"stated-inside": {dir("d", "/var/lib/pacman/keeper", `,"unknown":"program"`)},
|
||||
"stated-holding": {dir("d", "/var/lib/systemd", `,"unknown":"program"`)},
|
||||
"file-inside": {`{"id":"state","type":"directory","place":"mesh","unknown":"program"}`,
|
||||
`{"id":"f","type":"file","path":"/var/lib/pacman/keeper.json"}`},
|
||||
} {
|
||||
module := map[string]string{"placed-own": "pacman", "placed-beneath": "systemd"}[name]
|
||||
if module == "" {
|
||||
module = "keeper"
|
||||
}
|
||||
root := t.TempDir()
|
||||
lists(t, root, system, "")
|
||||
manifest(t, root, module, mod(module, resources...))
|
||||
found := check(t, root)
|
||||
if len(found) != 1 || found[0].Rule != 5 || found[0].Module != module ||
|
||||
!strings.Contains(found[0].Why, "/var/lib/") || !strings.Contains(found[0].Why, "package") {
|
||||
t.Errorf("%s: found %+v; want one finding of rule 5 for %s naming the path and the package", name, found, module)
|
||||
}
|
||||
}
|
||||
root := t.TempDir()
|
||||
lists(t, root, system, "")
|
||||
manifest(t, root, "pacman", mod("pacman",
|
||||
`{"id":"state","type":"directory","place":"mesh","unknown":"program"}`,
|
||||
`{"id":"f","type":"file","path":"${dir:state}/keeper.json"}`,
|
||||
dir("other", "/var/lib/pacman-other", `,"unknown":"program"`)))
|
||||
if found := check(t, root); len(found) != 0 {
|
||||
t.Fatalf("a directory under /var/lib/mesh/pacman was refused: %+v", found)
|
||||
}
|
||||
}
|
||||
|
||||
// 1841/AC18: a known exception passes; one that no longer collides is refused, so the list only shrinks; one the trunk's
|
||||
// copy does not hold is refused; and an absent list of system directories fails the check rather than passing it.
|
||||
func TestKnownSystemExceptionsOnlyShrink(t *testing.T) {
|
||||
system := "/var/lib/systemd systemd: tmpfiles.d/systemd.conf\n"
|
||||
known := "# comment\nsystemd state issue 1873\n"
|
||||
root := t.TempDir()
|
||||
lists(t, root, system, known)
|
||||
manifest(t, root, "systemd", mod("systemd", `{"id":"state","type":"directory","place":".","unknown":"program"}`))
|
||||
if found := check(t, root); len(found) != 0 {
|
||||
t.Fatalf("a known exception was refused: %+v", found)
|
||||
}
|
||||
if found, err := Check(root, []string{"systemd state issue 1873"}); err != nil || len(found) != 0 {
|
||||
t.Fatalf("a known exception held by the trunk was refused: %+v %v", found, err)
|
||||
}
|
||||
if found, err := Check(root, []string{}); err != nil || len(found) != 1 || found[0].Rule != 5 {
|
||||
t.Fatalf("an exception the trunk does not hold passed: %+v %v", found, err)
|
||||
}
|
||||
|
||||
manifest(t, root, "systemd", mod("systemd", `{"id":"state","type":"directory","place":"mesh","unknown":"program"}`))
|
||||
only(t, check(t, root), 5, "systemd", "state")
|
||||
|
||||
root = t.TempDir()
|
||||
manifest(t, root, "a", mod("a"))
|
||||
if err := os.MkdirAll(filepath.Join(root, "checks", "owned"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, SystemList), []byte("# nothing\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Check(root, nil); err == nil {
|
||||
t.Fatal("an empty list of system directories passed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1841/AC18: the catalogue as it stands passes rule 5 with its own lists.
|
||||
func TestTheCatalogueKeepsOutOfSystemDirectories(t *testing.T) {
|
||||
found, err := Check("../..", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range found {
|
||||
if f.Rule == 5 {
|
||||
t.Errorf("%s: %s: %s", f.Module, f.ID, f.Why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# The directories under /var/lib that a system package owns (rule 5 of checks/owned, issue 1841). A module's
|
||||
# directory, placed or stated, may not be one of these, lie inside one or hold one: the package's program and the
|
||||
# package manager keep their own files there, and the node-engine would judge, chmod or chown the directory.
|
||||
#
|
||||
# One line per directory: "<path> <package>: <how it was read>". Captured on the workstation on 2026-10-22, through
|
||||
# the mesh: pacman's own database by `pacman -Qo` (pacman_owns), the rest from the `d`, `D` and `Q` lines of
|
||||
# /usr/lib/tmpfiles.d (node-reads.search for "/var/lib/"). A directory found to belong to a package is added here.
|
||||
/var/lib/pacman pacman: pacman -Qo /var/lib/pacman/ (pacman 7.1.0)
|
||||
/var/lib/alsa alsa-utils: tmpfiles.d/alsa-utils.conf
|
||||
/var/lib/colord colord: tmpfiles.d/colord.conf
|
||||
/var/lib/containers podman: tmpfiles.d/podman.conf
|
||||
/var/lib/dbus dbus: tmpfiles.d/dbus.conf
|
||||
/var/lib/geoclue geoclue: tmpfiles.d/geoclue.conf
|
||||
/var/lib/lastlog util-linux: tmpfiles.d/lastlog2.conf
|
||||
/var/lib/libuuid util-linux: tmpfiles.d/uuidd.conf
|
||||
/var/lib/machines systemd: tmpfiles.d/systemd-nspawn.conf
|
||||
/var/lib/openvpn/chroot networkmanager-openvpn: tmpfiles.d/nm-openvpn-tmpfiles.conf
|
||||
/var/lib/portables systemd: tmpfiles.d/portables.conf
|
||||
/var/lib/postgres postgresql: tmpfiles.d/postgresql.conf
|
||||
/var/lib/private systemd: tmpfiles.d/systemd.conf
|
||||
/var/lib/rpcbind rpcbind: tmpfiles.d/rpcbind.conf
|
||||
/var/lib/systemd systemd: tmpfiles.d/systemd.conf
|
||||
/var/lib/tpm2-tss/system/keystore tpm2-tss: tmpfiles.d/tpm2-tss-fapi.conf
|
||||
@@ -0,0 +1,5 @@
|
||||
# Modules whose directory lies in a system package's directory today (rule 5 of checks/owned, issue 1841), each with
|
||||
# the issue that moves it out: "<module> <resource id> issue <number>". This list only shrinks: a line whose module
|
||||
# no longer collides is refused, and so is a line the trunk's copy does not hold.
|
||||
postgres state issue 1873
|
||||
systemd state issue 1873
|
||||
+17
-3
@@ -48,7 +48,8 @@
|
||||
# 10. the mail-store seat (novox/hq ADR 0378 §3, issue 800): no manifest but modules/mesh-mail claims it.
|
||||
# 11. owned paths (novox/hq ADR 0422 §8, issue 1024; it holds ~/.ssh for ssh-client as ADR 0415 §6 asked):
|
||||
# every directory and archive says `unknown` (move-out, module or program) and a valid `knows`; `move-out`
|
||||
# only under a home; and no module declares a path inside a directory of another module.
|
||||
# only under a home; no module declares a path inside a directory of another module; and none is, lies in or
|
||||
# holds a directory a system package owns under /var/lib (checks/owned/system, issue 1841).
|
||||
# checks/sshdir (ADR 0415 §6) stays while main runs it; it goes under issue 1035 once main runs checks/owned.
|
||||
# 12. the image builder (novox/hq ADR 0487 §3, issue 1704): only modules/buildah provides rootless-image-build.
|
||||
# 13. the image's place (novox/hq ADR 0487 §1, issue 1707): every image builds from a Dockerfile in its own module's directory.
|
||||
@@ -64,6 +65,8 @@ set -eu
|
||||
(cd checks/imagebuild && go test -count=1 . && go run . ../..)
|
||||
# The image's place (novox/hq ADR 0487 §1): every image builds from a Dockerfile in its own module's directory.
|
||||
(cd checks/imageplace && go test -count=1 . && go run . ../..)
|
||||
# The Node lock files (novox/hq ADR 0469 §5, issue 1868): a TypeScript module that names packages beyond the SDK commits its lock.
|
||||
(cd checks/locks && go test -count=1 . && go run . ../..)
|
||||
# The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
|
||||
(cd checks/agenthome && go test -count=1 . && go run . ../..)
|
||||
# The mail-store seat (novox/hq ADR 0378 §3): only mesh-mail may claim it.
|
||||
@@ -71,8 +74,19 @@ set -eu
|
||||
# ~/.ssh (novox/hq ADR 0415 §6): only ssh-client may name a path under it. Main's merge check runs this line, so
|
||||
# checks/sshdir stays until main runs checks/owned; it goes under issue 1035.
|
||||
(cd checks/sshdir && go test -count=1 . && go run . ../..)
|
||||
# Owned paths (novox/hq ADR 0422 §8): unknown files named per directory; no path inside another module's directory.
|
||||
(cd checks/owned && go test -count=1 . && go run . ../..)
|
||||
# Owned paths (novox/hq ADR 0422 §8): unknown files named per directory; no path inside another module's directory;
|
||||
# none in a system package's directory under /var/lib (issue 1841), the known exceptions only shrinking: judged
|
||||
# against the trunk's copy of checks/owned/system-known at the merge base; a trunk without one is the list's first copy.
|
||||
owned_base=""
|
||||
if merge_base=$(git merge-base HEAD "origin/${MESH_CHECK_BASE:-main}" 2>/dev/null); then
|
||||
if git cat-file -e "$merge_base:checks/owned/system-known" 2>/dev/null; then
|
||||
owned_base=$(mktemp)
|
||||
git show "$merge_base:checks/owned/system-known" >"$owned_base"
|
||||
fi
|
||||
else
|
||||
echo "NOT COMPARED WITH THE TRUNK: no merge base with origin/${MESH_CHECK_BASE:-main}; checks/owned/system-known is held to the list it was first written with"
|
||||
fi
|
||||
(cd checks/owned && go test -count=1 . && if [ -n "$owned_base" ]; then go run . -base "$owned_base" ../..; else go run . ../..; fi)
|
||||
# The secrets (novox/hq issue 342): a password is not a setting, so no setting can replace the vault's. The
|
||||
# known list only shrinks: judged against the trunk's copy at the merge base, a line it does not hold is refused.
|
||||
secrets_base=""
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
@@ -170,20 +171,33 @@ func TestTheLauncherWaitsQuietlyWhileAnotherClipmenudHoldsTheLock(t *testing.T)
|
||||
store := dir + "/cm/clipmenu.6.op"
|
||||
os.MkdirAll(store, 0o700)
|
||||
lock := store + "/session_lock"
|
||||
// the older clipmenud: it holds the lock the way clipmenud does, until it is killed
|
||||
holder := exec.Command("flock", "-x", "-n", lock, "sleep", "60")
|
||||
// the older clipmenud: it holds the lock the way clipmenud does, until it is killed. It says "held" once it has the
|
||||
// lock, so the test goes on only when the lock is held for certain: a holder that is slow to start (the checker
|
||||
// runs thousands of tests at once) must not let the launcher find the lock free (issue 1814).
|
||||
holder := exec.Command("flock", "-x", "-n", lock, "sh", "-c", "echo held; exec sleep 60")
|
||||
holder.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
held, err := holder.StdoutPipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := holder.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// flock's child shares the lock, so the whole group goes
|
||||
release := func() { syscall.Kill(-holder.Process.Pid, syscall.SIGKILL); holder.Wait() }
|
||||
defer release()
|
||||
for i := 0; i < 50; i++ {
|
||||
if probe := exec.Command("flock", "-n", lock, "true"); probe.Run() != nil {
|
||||
break
|
||||
said := make(chan string, 1)
|
||||
go func() {
|
||||
line, _ := bufio.NewReader(held).ReadString('\n')
|
||||
said <- strings.TrimSpace(line)
|
||||
}()
|
||||
select {
|
||||
case line := <-said:
|
||||
if line != "held" {
|
||||
t.Fatalf("the older clipmenud did not take the lock: %q", line)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the older clipmenud did not take the lock within 30 s")
|
||||
}
|
||||
var out syncBuffer
|
||||
cmd := exec.Command("sh", file)
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# The test script compiles the sources beside themselves before the tests import them (package.json, issue 1891).
|
||||
*.js
|
||||
+15
-9
@@ -22,16 +22,18 @@ Every new head of an open pull request is announced as `pull.updated`, once, and
|
||||
minutes, or when the search fails.
|
||||
- **A pull request nobody checks is said — by the delivery's owner, not here.** `gitea_open_pulls` answers every
|
||||
open pull request from the forge itself, with the statuses its base requires and the ones its head has. The
|
||||
`mesh-delivery` module asks it every five minutes, and a head on a base requiring `mesh/merge-gate` or
|
||||
`mesh/repo-check` with neither for ten minutes since it first saw it so is a delivery held past its bound
|
||||
delivery gate asks it every five minutes, and a head on a base requiring `mesh/check` (or hq's
|
||||
`mesh/repo-check`) with none of them for ten minutes since it first saw it so is a delivery held past its bound
|
||||
(`delivery.<id>.stalled`, state `unannounced`), which the controller raises like any other (issue 347).
|
||||
Each of the head's statuses comes with its state and when it was set, so a head whose check started and never
|
||||
answered — a required check pending, or never set beside one that is — is said as a stalled line in the state
|
||||
`unanswered` once that has lasted the delivery table's bound for a check asked and not answered, an hour, counted
|
||||
from its first sight and again only when a check is asked anew (issue 438).
|
||||
- **How to see it work.** A pull request opened more than a minute after its branch was pushed — this fix's own
|
||||
was one, and the announcer it fixes never saw it — has `mesh/merge-gate` pending within a pass of the next sweep
|
||||
at the latest, and `mesh-controller.conditions` names none for it.
|
||||
was one, and the announcer it fixes never saw it — is announced within a pass of the next sweep at the latest, its
|
||||
`mesh/check` is the checker's to set, and `mesh-controller.conditions` names none for it. (Since issue 1301 this
|
||||
module writes `mesh/merge-gate` only where a base still requires it, and none does once the forge requires
|
||||
`mesh/check`.)
|
||||
- **A pull request or issue number that is not one is refused** before the forge is asked: a tool called with a
|
||||
misnamed argument sent `pulls/NaN`.
|
||||
|
||||
@@ -154,15 +156,19 @@ listed repository:
|
||||
naming the description, and a `merge-check.sh` declaring `# mesh-check-toolchain: go` that vets and runs the unit
|
||||
tests, then the integration tests (`-tags integration`), of every Go module in the tree, and says so and passes on
|
||||
a tree with none. The rule is then narrowed to main's rule of ADR 0398 §2 (no push, only `mesh-tools` merges),
|
||||
requiring `mesh/repo-check` and `mesh/merge-gate`, and only then is the repository moved into the organisation.
|
||||
requiring `mesh/check` (ADR 0447 §1, issue 1301), and only then is the repository moved into the organisation.
|
||||
There main's rule is read back and put right once more (the step `rule-after-move`): the forge drops from a merge
|
||||
list a user without access to the repository, and under the setup account `mesh-tools` has none;
|
||||
- one found still under the setup account is finished: the steps it lacks are done, main is never written twice;
|
||||
- one in the organisation is only put right: main's rule narrowed, a missing one of the two statuses added and every
|
||||
status already there kept.
|
||||
- one in the organisation is only put right: main's rule narrowed, `mesh/check` required in place of
|
||||
`mesh/merge-gate` and `mesh/repo-check`, and every other status already there kept. A main that requires
|
||||
`mesh/repo-check` without `mesh/merge-gate` is judged by its own script alone (hq, ADR 0503 §5, until issue 1296)
|
||||
and keeps it; `mesh/check` is never dropped.
|
||||
|
||||
A step the forge refuses stops that repository, and the others go on. A repository taken out of the value is never
|
||||
asked about: nothing is deleted, renamed, moved out or unprotected. The pass keeps what it did per repository in
|
||||
Every other repository of the organisation whose main requires `mesh/merge-gate` has the two old statuses swapped
|
||||
for `mesh/check`, and only its statuses (issue 1301): the switch reaches every repository the merge gate judged, with
|
||||
no list kept by hand. A step the forge refuses stops that repository, and the others go on. Beyond that swap, a
|
||||
repository taken out of the value is never changed: nothing is deleted, renamed, moved out or unprotected. The pass keeps what it did per repository in
|
||||
`repositories-outcome.json`.
|
||||
|
||||
**How a refusal is seen.** `repositories sync` is a run-once process, declared as the module's last resource, so
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
// **mesh/check is written from the checker's owed state, read by a watch** (ADR 0521; R4e round 2 finding R2-1; finding K1
|
||||
// before it; novox/hq issue 1301; ADR 0447 §1, ADR 0503 §3: main requires mesh/check alone).
|
||||
//
|
||||
// No tool writes mesh/check: a tool is told no caller, and the console every agent session speaks through (node-tools)
|
||||
// invokes every tool, so a tool that wrote it could be called by any session to merge past the checker (the fault issue
|
||||
// 293 closed for the two statuses before it). Nor does an event: nothing in the delivery plan orders this module before the
|
||||
// forge bridge, and a durable consumer whose filters change keeps its deliver policy, so an event said before this module
|
||||
// consumed it was lost. The checker keeps what it owes the forge in its state `owed` (ADR 0484 §2), which it announces
|
||||
// readable (ADR 0495 §2-§3), and only the checker writes it. This module watches it: the watch hands over the latest value of
|
||||
// every key when it starts, whenever that is, and then every change. Each key whose act is the status mesh/check is
|
||||
// written on its head, once per revision; a revision older than one already written on the key is dropped.
|
||||
//
|
||||
// A read refused (the grant not yet composed) is said once and asked again on a bound until it holds. A forge that refuses
|
||||
// a status is asked again on a bound, until it writes or a newer revision of the key comes.
|
||||
|
||||
import type { CommitStatus } from "./client.js";
|
||||
import { bindUntilHeld, bindWait } from "./bind.js";
|
||||
import { checkName } from "./guard.js";
|
||||
import { CHECK_STATUS } from "./pulls.js";
|
||||
|
||||
/** The checker's state this module reads, and the act of its keys this module writes. */
|
||||
export const CHECKER_OWED = "mesh-delivery-checker.owed";
|
||||
export const CHECK_ACT = `status=${CHECK_STATUS}`;
|
||||
|
||||
const states = new Set(["pending", "success", "error", "failure", "warning"]);
|
||||
|
||||
/** What writing the check needs of the forge. */
|
||||
export interface CheckForge {
|
||||
setCommitStatus(owner: string, repo: string, sha: string, status: CommitStatus): Promise<void>;
|
||||
}
|
||||
|
||||
/** One change of the checker's owed, as the SDK's state watch hands it. */
|
||||
export interface OwedChange {
|
||||
key: string;
|
||||
op: "put" | "delete";
|
||||
value?: unknown;
|
||||
revision: number;
|
||||
current?: boolean;
|
||||
}
|
||||
|
||||
/** A key part as mesh-sdk go/delivery EscapeName writes it, read back: `=` and two lowercase hex digits per byte. */
|
||||
function unescapeName(part: string): string {
|
||||
if (!/^([-/_a-zA-Z0-9]|=[0-9a-f]{2})+$/.test(part)) throw new Error(`${JSON.stringify(part)} is not an escaped name`);
|
||||
return Buffer.from(part.replace(/=([0-9a-f]{2})/g, (_, h) => String.fromCharCode(parseInt(h, 16))), "latin1").toString("utf8");
|
||||
}
|
||||
|
||||
/** The pull request and head of the checker's owed key (mesh-sdk go/delivery CheckerOwedKey):
|
||||
* `<owner>/<name>.<number>.<head>.status=mesh/check`, owner and name escaped. */
|
||||
export function owedKeyOf(key: string): { owner: string; repo: string; number: number; sha: string } {
|
||||
const parts = String(key ?? "").split(".");
|
||||
if (parts.length !== 4) throw new Error(`${JSON.stringify(key)} is not an owed key, <repository>.<number>.<head>.<act>`);
|
||||
const [repository, number, sha, act] = parts;
|
||||
if (act !== CHECK_ACT) throw new Error(`${JSON.stringify(key)} owes ${JSON.stringify(act)}, not ${CHECK_ACT}`);
|
||||
const m = /^([^/]+)\/([^/]+)$/.exec(repository);
|
||||
if (!m) throw new Error(`${JSON.stringify(key)} names no <owner>/<name>`);
|
||||
if (!/^(0|[1-9][0-9]*)$/.test(number)) throw new Error(`${JSON.stringify(key)} names no pull request number`);
|
||||
if (!/^([0-9a-f]{40}|[0-9a-f]{64})$/i.test(sha)) throw new Error(`${JSON.stringify(key)} names no head's whole sha`);
|
||||
return { owner: checkName("owner", unescapeName(m[1])), repo: checkName("name", unescapeName(m[2])), number: Number(number), sha };
|
||||
}
|
||||
|
||||
/** The status an owed change asks for, checked as mesh-sdk go/delivery DecodeCheckerOwed checks it; or why it is not one. */
|
||||
export function checkStatusOf(change: OwedChange): { owner: string; repo: string; sha: string; revision: number; status: CommitStatus } {
|
||||
if (change.op !== "put") throw new Error(`a ${change.op} owes nothing`);
|
||||
const { owner, repo, sha } = owedKeyOf(change.key);
|
||||
const revision = change.revision;
|
||||
if (typeof revision !== "number" || !Number.isSafeInteger(revision) || revision < 0) {
|
||||
throw new Error(`${JSON.stringify(revision)} is not a revision`);
|
||||
}
|
||||
const v = change.value;
|
||||
if (!v || typeof v !== "object") throw new Error("the value is no owed act");
|
||||
const o = v as Record<string, unknown>;
|
||||
if (o.v !== 1) throw new Error(`the owed act's version ${JSON.stringify(o.v)} is not one this module reads`);
|
||||
if (o.act !== "status" || o.context !== CHECK_STATUS) {
|
||||
throw new Error(`the owed act is ${JSON.stringify(o.act)} ${JSON.stringify(o.context)}, not the status ${CHECK_STATUS}`);
|
||||
}
|
||||
const state = String(o.state ?? "");
|
||||
if (!states.has(state)) throw new Error(`${JSON.stringify(o.state)} is not a status the forge keeps`);
|
||||
let description = String(o.text ?? "").replace(/\s+/g, " ").trim();
|
||||
if (description.length > 140) description = description.slice(0, 139) + "…";
|
||||
const target = typeof o.target === "string" && /^https?:\/\//.test(o.target) ? o.target : undefined;
|
||||
const status: CommitStatus = { state: state as CommitStatus["state"], context: CHECK_STATUS, description,
|
||||
...(target ? { target_url: target } : {}) };
|
||||
return { owner, repo, sha, revision, status };
|
||||
}
|
||||
|
||||
/** The one writer of mesh/check: each owed revision of the checker's, in revision order per key. */
|
||||
export class CheckStatuses {
|
||||
/** The newest revision written of each key, and the newest seen; the oldest keys go past what any forge holds open. */
|
||||
private written = new Map<string, number>();
|
||||
private seen = new Map<string, number>();
|
||||
/** The turn each key's writes wait on, and the asks again still running. */
|
||||
private turns = new Map<string, Promise<unknown>>();
|
||||
private again = new Set<Promise<void>>();
|
||||
private forge: CheckForge;
|
||||
private say: (line: string) => void;
|
||||
private sleep: (ms: number) => Promise<void>;
|
||||
|
||||
constructor(forge: CheckForge, say: (line: string) => void = (l) => console.error(l),
|
||||
sleep: (ms: number) => Promise<void> = (ms) => new Promise((r) => setTimeout(r, ms))) {
|
||||
this.forge = forge;
|
||||
this.say = say;
|
||||
this.sleep = sleep;
|
||||
}
|
||||
|
||||
/** Write the change's status: "written", or "dropped" for one that owes no mesh/check or is older than one written. A
|
||||
* forge that refuses throws. */
|
||||
async apply(change: OwedChange): Promise<"written" | "dropped"> {
|
||||
let asked;
|
||||
try {
|
||||
asked = checkStatusOf(change);
|
||||
} catch (err) {
|
||||
this.say(`[gitea] ${CHECKER_OWED} ${change?.key} at revision ${change?.revision} writes nothing: ${err instanceof Error ? err.message : err}`);
|
||||
return "dropped";
|
||||
}
|
||||
const key = change.key;
|
||||
const seen = this.seen.get(key);
|
||||
if (seen === undefined || asked.revision > seen) this.remember(this.seen, key, asked.revision);
|
||||
// One key's writes take turns, so two at once are weighed one after the other.
|
||||
const before = this.turns.get(key) ?? Promise.resolve();
|
||||
const turn = before.then(async (): Promise<"written" | "dropped"> => {
|
||||
const newest = this.written.get(key);
|
||||
if (newest !== undefined && asked.revision < newest) {
|
||||
this.say(`[gitea] ${asked.owner}/${asked.repo}@${asked.sha.slice(0, 12)}: mesh/check at revision ${asked.revision} is older than ${newest}, already written, and is dropped`);
|
||||
return "dropped";
|
||||
}
|
||||
await this.forge.setCommitStatus(asked.owner, asked.repo, asked.sha, asked.status);
|
||||
this.remember(this.written, key, asked.revision);
|
||||
return "written";
|
||||
});
|
||||
const settled = turn.catch(() => {});
|
||||
this.turns.set(key, settled);
|
||||
void settled.then(() => { if (this.turns.get(key) === settled) this.turns.delete(key); });
|
||||
return turn;
|
||||
}
|
||||
|
||||
/** What the watch hands over: written, or when the forge refuses, asked again on a bound in the background until it is
|
||||
* written or a newer revision of the key was seen. Never throws: a watch's handler that throws loses the change. */
|
||||
async take(change: OwedChange): Promise<void> {
|
||||
try {
|
||||
await this.apply(change);
|
||||
} catch (err) {
|
||||
const run = this.askAgain(change, err);
|
||||
this.again.add(run);
|
||||
void run.finally(() => this.again.delete(run));
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolves when no ask again is running. */
|
||||
async settled(): Promise<void> {
|
||||
while (this.again.size) await Promise.all([...this.again]);
|
||||
}
|
||||
|
||||
private async askAgain(change: OwedChange, first: unknown): Promise<void> {
|
||||
let err = first;
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
const wait = bindWait(attempt);
|
||||
this.say(`[gitea] ${change.key}: mesh/check at revision ${change.revision} could not be written (${err instanceof Error ? err.message : err}); asked again in ${wait / 1000}s`);
|
||||
await this.sleep(wait);
|
||||
const newest = this.seen.get(change.key);
|
||||
if (newest !== undefined && newest > change.revision) return;
|
||||
try {
|
||||
await this.apply(change);
|
||||
return;
|
||||
} catch (e) {
|
||||
err = e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private remember(map: Map<string, number>, key: string, revision: number): void {
|
||||
map.delete(key);
|
||||
map.set(key, revision);
|
||||
if (map.size > 4000) map.delete(map.keys().next().value as string);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Watch the checker's owed with `watch` (the SDK's `state(CHECKER_OWED).watch`), handing each change to `checks`. A watch
|
||||
* refused — the read not granted yet — is said once and asked again on a bound until it holds (bind.ts). Never rejects.
|
||||
*/
|
||||
export async function watchOwed(watch: (handler: (change: OwedChange) => Promise<void>) => Promise<void>, checks: CheckStatuses,
|
||||
sleep?: (ms: number) => Promise<void>, say: (line: string) => void = (l) => console.error(l)): Promise<void> {
|
||||
let refusals = 0;
|
||||
await bindUntilHeld(CHECKER_OWED, () => watch((change) => checks.take(change)), sleep, (line) => {
|
||||
if (/could not be bound/.test(line) && refusals++ > 0) return;
|
||||
say(line);
|
||||
});
|
||||
}
|
||||
+17
-2
@@ -533,7 +533,7 @@ export class GiteaClient {
|
||||
* **A listing the forge cut is no listing** (it says `truncated` on a page with nothing more to read, or the pages
|
||||
* run past a bound): the call fails naming the repository and the commit, never answering the directories it did
|
||||
* see, since a directory left out would be taken for one without a module. Reads only. */
|
||||
async commitTrees(owner: string, repo: string, sha: string): Promise<{ root: string; trees: Record<string, string> }> {
|
||||
async commitTrees(owner: string, repo: string, sha: string, paths?: Record<string, string[]>): Promise<{ root: string; trees: Record<string, string>; paths?: Record<string, Record<string, string>> }> {
|
||||
const base = `${repoPath(owner, repo)}/git/trees/${commitSegment(sha)}?recursive=true`;
|
||||
const cut = (why: string) => new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} was cut (${why}); no trees are answered from a part`);
|
||||
const entries: any[] = [];
|
||||
@@ -560,7 +560,22 @@ export class GiteaClient {
|
||||
if (!tree) throw new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} holds ${e.path} but not the tree of ${dir}`);
|
||||
trees[dir] = tree;
|
||||
}
|
||||
return { root, trees };
|
||||
if (paths === undefined) return { root, trees };
|
||||
// **Named paths** (novox/hq ADR 0499 §3, issue 1758): each asked path's git object id at the commit, a file's or a
|
||||
// directory's, from the same listing, by directory and by path relative to it. A path the listing lacks fails
|
||||
// naming both, and nothing is answered in part.
|
||||
const objectOf = new Map<string, string>();
|
||||
for (const e of entries) if (typeof e?.path === "string" && (e.type === "tree" || e.type === "blob")) objectOf.set(e.path, String(e.sha));
|
||||
const named: Record<string, Record<string, string>> = {};
|
||||
for (const [dir, list] of Object.entries(paths)) {
|
||||
named[dir] = {};
|
||||
for (const p of list) {
|
||||
const id = objectOf.get(dir === "" ? p : `${dir}/${p}`);
|
||||
if (!id) throw new Error(`the path ${p} of ${dir === "" ? "the repository root" : dir} is not in ${owner}/${repo} at ${sha}`);
|
||||
named[dir][p] = id;
|
||||
}
|
||||
}
|
||||
return { root, trees, paths: named };
|
||||
}
|
||||
|
||||
/** Whether a repository holds a file at a commit: true, false for a 404, thrown when the forge cannot say. */
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The tests of issue 1301 (ADR 0447 §1, §5; ADR 0503 §3, §5; lead decisions L4 and L11): main requires `mesh/check`
|
||||
// alone in place of `mesh/merge-gate` and `mesh/repo-check`; a main that requires `mesh/repo-check` without
|
||||
// `mesh/merge-gate` (hq) keeps it; every repository of the organisation whose main requires `mesh/merge-gate` is
|
||||
// switched, listed in the value or not; no other status is dropped.
|
||||
|
||||
func narrowedRule(contexts ...any) map[string]any {
|
||||
return map[string]any{
|
||||
"enable_push": false, "enable_push_whitelist": false, "push_whitelist_usernames": []any{},
|
||||
"enable_merge_whitelist": true, "merge_whitelist_usernames": []any{tools},
|
||||
"enable_status_check": true, "status_check_contexts": contexts,
|
||||
"block_admin_merge_override": true,
|
||||
}
|
||||
}
|
||||
|
||||
// 1301-S1: a listed repository whose main requires the two old statuses is put right to `mesh/check` in their place,
|
||||
// every other status kept; a second pass changes nothing.
|
||||
func TestListedRepositorySwapsTheTwoStatusesForCheck(t *testing.T) {
|
||||
f := newFake()
|
||||
repo := f.add("novox/mesh-delivery", map[string]map[string]any{mainRule: narrowedRule("mesh/merge-gate", "mesh/repo-check", "other/x")}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{"mesh-delivery": {}})
|
||||
got := strs(repo.Rules[mainRule]["status_check_contexts"])
|
||||
if strings.Join(got, ",") != "other/x,mesh/check" {
|
||||
t.Errorf("main requires %v, want other/x and mesh/check", got)
|
||||
}
|
||||
if r := result(o, "mesh-delivery"); r.Did != DidPutRight {
|
||||
t.Errorf("outcome %+v, want put right", r)
|
||||
}
|
||||
before := len(f.writes())
|
||||
run(t, s, Value{"mesh-delivery": {}})
|
||||
if w := f.writes(); len(w) != before {
|
||||
t.Errorf("a second pass wrote %v", w[before:])
|
||||
}
|
||||
}
|
||||
|
||||
// 1301-S1: a repository the program makes requires `mesh/check` alone from its first rule in the organisation on.
|
||||
func TestNewRepositoryRequiresCheckAlone(t *testing.T) {
|
||||
f := newFake()
|
||||
s, _ := syncer(t, f)
|
||||
run(t, s, Value{"fresh": {Description: "d"}})
|
||||
rule := f.repos["novox/fresh"].Rules[mainRule]
|
||||
if got := strs(rule["status_check_contexts"]); rule["enable_status_check"] != true || strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("a new repository's main requires %v, want %s alone", got, CheckStatus)
|
||||
}
|
||||
if readme := f.repos["novox/fresh"].Files["README.md"]; !strings.Contains(readme, CheckStatus) ||
|
||||
strings.Contains(readme, "mesh/merge-gate") || strings.Contains(readme, "mesh/repo-check") {
|
||||
t.Errorf("the README names the statuses of before: %q", readme)
|
||||
}
|
||||
}
|
||||
|
||||
// 1301-S1: mesh/check is never dropped, and a rule that requires it beside a retired status loses the retired one.
|
||||
func TestCheckIsKeptAndARetiredStatusBesideItGoes(t *testing.T) {
|
||||
f := newFake()
|
||||
both := f.add("novox/both", map[string]map[string]any{mainRule: narrowedRule("mesh/check", "mesh/repo-check")}, "main")
|
||||
right := f.add("novox/right", map[string]map[string]any{mainRule: narrowedRule("mesh/check")}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{"both": {}, "right": {}})
|
||||
if got := strs(both.Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("both requires %v, want %s alone", got, CheckStatus)
|
||||
}
|
||||
if got := strs(right.Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("right requires %v", got)
|
||||
}
|
||||
if result(o, "right").Did != DidUnchanged || result(o, "both").Did != DidPutRight {
|
||||
t.Errorf("outcome %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// 1301-S1: a listed repository whose main requires nothing is given mesh/check, not the old two.
|
||||
func TestListedRuleRequiringNothingGetsCheck(t *testing.T) {
|
||||
f := newFake()
|
||||
none := f.add("novox/none", map[string]map[string]any{mainRule: {"enable_push": false}}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
run(t, s, Value{"none": {}})
|
||||
if got := strs(none.Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("none requires %v, want %s", got, CheckStatus)
|
||||
}
|
||||
}
|
||||
|
||||
// 1301-S2 (ADR 0503 §5): a main that requires mesh/repo-check without mesh/merge-gate is judged by its own script
|
||||
// alone (hq): it keeps mesh/repo-check and is not given mesh/check, listed or not.
|
||||
func TestOwnCheckAloneIsKept(t *testing.T) {
|
||||
f := newFake()
|
||||
hq := f.add("novox/hq", map[string]map[string]any{mainRule: narrowedRule("mesh/repo-check")}, "main")
|
||||
lab := f.add("novox/mesh-lab", map[string]map[string]any{mainRule: narrowedRule("mesh/repo-check")}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{"hq": {}})
|
||||
for name, repo := range map[string]*fRepo{"hq": hq, "mesh-lab": lab} {
|
||||
if got := strs(repo.Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != "mesh/repo-check" {
|
||||
t.Errorf("%s requires %v, want mesh/repo-check alone", name, got)
|
||||
}
|
||||
}
|
||||
if result(o, "hq").Did != DidUnchanged {
|
||||
t.Errorf("hq outcome %+v, want unchanged", result(o, "hq"))
|
||||
}
|
||||
for _, w := range f.writes() {
|
||||
t.Errorf("a repository judged by its own check alone was written: %s", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1301-S3 (L4, L11): a repository the value does not list whose main requires mesh/merge-gate has the two old
|
||||
// statuses swapped for mesh/check, and only its statuses: nothing else of its rule is sent. One that requires
|
||||
// neither, or mesh/repo-check alone, is not written to.
|
||||
func TestUnlistedRepositoryRequiringTheGateIsSwitched(t *testing.T) {
|
||||
f := newFake()
|
||||
catalog := f.add("novox/mesh-catalog", map[string]map[string]any{mainRule: {
|
||||
"enable_push": true, "enable_status_check": true, "status_check_contexts": []any{"mesh/merge-gate", "mesh/repo-check"},
|
||||
}}, "main")
|
||||
toolsGo := f.add("novox/mesh-tools-go", map[string]map[string]any{mainRule: narrowedRule("mesh/merge-gate")}, "main")
|
||||
f.add("novox/plain", map[string]map[string]any{mainRule: {"enable_push": false}}, "main")
|
||||
f.add("novox/hq", map[string]map[string]any{mainRule: narrowedRule("mesh/repo-check")}, "main")
|
||||
f.add("novox/norule", nil, "main")
|
||||
f.add("mesh-admin/elsewhere", map[string]map[string]any{mainRule: narrowedRule("mesh/merge-gate")}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{})
|
||||
|
||||
if got := strs(catalog.Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("mesh-catalog requires %v, want %s", got, CheckStatus)
|
||||
}
|
||||
if catalog.Rules[mainRule]["enable_push"] != true {
|
||||
t.Errorf("an unlisted repository's rule was narrowed beyond its statuses: %v", catalog.Rules[mainRule])
|
||||
}
|
||||
if got := strs(toolsGo.Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("mesh-tools-go requires %v, want %s", got, CheckStatus)
|
||||
}
|
||||
want := []string{
|
||||
"PATCH /api/v1/repos/novox/mesh-catalog/branch_protections/main",
|
||||
"PATCH /api/v1/repos/novox/mesh-tools-go/branch_protections/main",
|
||||
}
|
||||
if got := f.writes(); strings.Join(got, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("writes:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
for _, n := range []string{"mesh-catalog", "mesh-tools-go"} {
|
||||
if r := result(o, n); r.Did != DidPutRight {
|
||||
t.Errorf("%s outcome %+v, want put right", n, r)
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"plain", "hq", "norule", "elsewhere"} {
|
||||
if r := result(o, n); r.Name != "" {
|
||||
t.Errorf("%s is in the outcome though nothing was done: %+v", n, r)
|
||||
}
|
||||
}
|
||||
before := len(f.writes())
|
||||
run(t, s, Value{})
|
||||
if w := f.writes(); len(w) != before {
|
||||
t.Errorf("a second pass wrote %v", w[before:])
|
||||
}
|
||||
}
|
||||
|
||||
// 1301-S3: the organisation's repositories are read page by page, so one past the first page is switched too.
|
||||
func TestUnlistedRepositoriesAreReadPastTheFirstPage(t *testing.T) {
|
||||
f := newFake()
|
||||
f.pageSize = 2
|
||||
for _, n := range []string{"a", "b", "c", "d", "e"} {
|
||||
f.add("novox/"+n, map[string]map[string]any{mainRule: narrowedRule("mesh/merge-gate", "mesh/repo-check")}, "main")
|
||||
}
|
||||
s, _ := syncer(t, f)
|
||||
run(t, s, Value{})
|
||||
for _, n := range []string{"a", "b", "c", "d", "e"} {
|
||||
if got := strs(f.repos["novox/"+n].Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("%s requires %v", n, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1301-S3: a refused swap is said with the repository and its step, and the pass goes on to the next.
|
||||
func TestRefusedSwapIsSaid(t *testing.T) {
|
||||
f := newFake()
|
||||
f.add("novox/a", map[string]map[string]any{mainRule: narrowedRule("mesh/merge-gate")}, "main")
|
||||
f.add("novox/b", map[string]map[string]any{mainRule: narrowedRule("mesh/merge-gate")}, "main")
|
||||
f.refuse = func(method, path string) (int, string) {
|
||||
if method == "PATCH" && strings.Contains(path, "/novox/a/") {
|
||||
return 403, "no"
|
||||
}
|
||||
return 0, ""
|
||||
}
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{})
|
||||
if r := result(o, "a"); r.Did != DidRefused || r.Step != StepSwap {
|
||||
t.Errorf("a: %+v, want refused at %s", r, StepSwap)
|
||||
}
|
||||
if r := result(o, "b"); r.Did != DidPutRight {
|
||||
t.Errorf("b: %+v, want put right", r)
|
||||
}
|
||||
}
|
||||
|
||||
// 1301-S3: the organisation's list refused is said, and the listed repositories are still done.
|
||||
func TestOrganisationListRefusedIsSaid(t *testing.T) {
|
||||
f := newFake()
|
||||
f.refuse = func(method, path string) (int, string) {
|
||||
if strings.HasPrefix(path, "/api/v1/orgs/") {
|
||||
return 500, "down"
|
||||
}
|
||||
return 0, ""
|
||||
}
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{"x": {}})
|
||||
if result(o, "x").Did != DidMade {
|
||||
t.Errorf("x: %+v", result(o, "x"))
|
||||
}
|
||||
refusedList := false
|
||||
for _, r := range o.Refused() {
|
||||
refusedList = refusedList || r.Step == StepList
|
||||
}
|
||||
if !refusedList {
|
||||
t.Errorf("the refused list is not in the outcome: %+v", o)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -43,6 +44,8 @@ type fakeForge struct {
|
||||
calls []string // "METHOD /path"
|
||||
// refuse answers a call with a status and a message when it returns a status above zero.
|
||||
refuse func(method, path string) (int, string)
|
||||
// pageSize is how many repositories one page of the organisation's list holds; 50 when zero.
|
||||
pageSize int
|
||||
}
|
||||
|
||||
func newFake() *fakeForge { return &fakeForge{repos: map[string]*fRepo{}} }
|
||||
@@ -84,6 +87,7 @@ var (
|
||||
branchPath = regexp.MustCompile(`^/api/v1/repos/([^/]+)/([^/]+)/branches/([^/]+)$`)
|
||||
filesPath = regexp.MustCompile(`^/api/v1/repos/([^/]+)/([^/]+)/contents$`)
|
||||
movePath = regexp.MustCompile(`^/api/v1/repos/([^/]+)/([^/]+)/transfer$`)
|
||||
orgRepos = regexp.MustCompile(`^/api/v1/orgs/([^/]+)/repos$`)
|
||||
)
|
||||
|
||||
func strs(v any) []string {
|
||||
@@ -167,6 +171,31 @@ func (f *fakeForge) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case p == "/api/v1/version":
|
||||
reply(200, map[string]string{"version": "1.27"})
|
||||
case orgRepos.MatchString(p) && r.Method == http.MethodGet:
|
||||
owner := orgRepos.FindStringSubmatch(p)[1]
|
||||
var names []string
|
||||
for full := range f.repos {
|
||||
if o, n, _ := strings.Cut(full, "/"); o == owner {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
size := f.pageSize
|
||||
if size == 0 {
|
||||
size = 50
|
||||
}
|
||||
if l, _ := strconv.Atoi(r.URL.Query().Get("limit")); l > 0 && l < size {
|
||||
size = l
|
||||
}
|
||||
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
out := []map[string]any{}
|
||||
for i := (page - 1) * size; i < len(names) && i < page*size; i++ {
|
||||
out = append(out, map[string]any{"name": names[i], "full_name": owner + "/" + names[i]})
|
||||
}
|
||||
reply(200, out)
|
||||
case p == "/api/v1/user/repos" && r.Method == http.MethodPost:
|
||||
name := fmt.Sprint(in["name"])
|
||||
if f.repos[admin+"/"+name] != nil {
|
||||
@@ -312,12 +341,3 @@ func result(o Outcome, name string) Result {
|
||||
}
|
||||
return Result{}
|
||||
}
|
||||
|
||||
func has(list []string, want string) bool {
|
||||
for _, s := range list {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import "strings"
|
||||
const MergeCheck = `#!/bin/sh
|
||||
# mesh-check-toolchain: go
|
||||
#
|
||||
# This repository's own check, mesh/repo-check (novox/hq ADR 0238 §4), written with its first commit (ADR 0428 §4):
|
||||
# This repository's own check (novox/hq ADR 0238 §4), run by the checker within mesh/check (ADR 0447), written with its first commit (ADR 0428 §4):
|
||||
# every Go module in the tree is vetted, its unit tests run, and then its integration tests (-tags integration).
|
||||
# A tree with no Go module has nothing to test, and the check says so and passes.
|
||||
set -eu
|
||||
@@ -37,7 +37,7 @@ func FirstCommit(name string, r Repo) map[string]string {
|
||||
}
|
||||
readme := "# " + name + "\n\n" + purpose + "\n\n" +
|
||||
"Made by the mesh from the gitea module's config value `repositories` (novox/hq ADR 0428). Main takes no push: " +
|
||||
"a change reaches it by a pull request whose `mesh/repo-check` (this repository's `merge-check.sh`) and " +
|
||||
"`mesh/merge-gate` passed.\n"
|
||||
"a change reaches it by a pull request whose `mesh/check` passed: the checker's verdict on its head merged onto " +
|
||||
"main, this repository's `merge-check.sh` among its tests (novox/hq ADR 0447).\n"
|
||||
return map[string]string{"README.md": readme, "merge-check.sh": MergeCheck}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
// again whenever the value changes: it reads the file the mesh renders from that value and, with the setup login,
|
||||
// makes every listed repository that does not exist under that login's own account, protects its main, writes
|
||||
// main's first commit, narrows main's rule and only then moves it into the organisation; one that exists only has
|
||||
// its main rule put right. A repository the value does not list is never touched. It keeps what it did, per
|
||||
// its main rule put right, requiring mesh/check in place of the two statuses it replaces (issue 1301). A repository
|
||||
// the value does not list is touched only to swap those two for mesh/check when its main requires mesh/merge-gate. It keeps what it did, per
|
||||
// repository, in the outcome file, says each change on stdout in one line and never a secret, and exits 1 when a
|
||||
// step was refused, its last stderr lines naming each (outcome.go Report): the node-engine's gate on a run-once
|
||||
// step (ADR 0136) reports it and runs it again on every apply until it passes.
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestNewRepositoryRefusesTheAdministratorsOverride(t *testing.T) {
|
||||
}
|
||||
|
||||
// 1141/AC2: a narrowed main rule that still lets an administrator override is not right: it is put right, its
|
||||
// statuses kept, and a second pass changes nothing.
|
||||
// statuses kept (the two old ones as mesh/check, issue 1301), and a second pass changes nothing.
|
||||
func TestNarrowedRuleLettingTheAdministratorOverrideIsPutRight(t *testing.T) {
|
||||
f := newFake()
|
||||
repo := f.add("novox/mesh-delivery", map[string]map[string]any{mainRule: {
|
||||
@@ -33,10 +33,8 @@ func TestNarrowedRuleLettingTheAdministratorOverrideIsPutRight(t *testing.T) {
|
||||
if rule["block_admin_merge_override"] != true {
|
||||
t.Errorf("the administrator's override is still allowed: %v", rule)
|
||||
}
|
||||
for _, c := range []string{"mesh/merge-gate", "mesh/repo-check", "other/x"} {
|
||||
if !has(strs(rule["status_check_contexts"]), c) {
|
||||
t.Errorf("a status was dropped: %v", rule["status_check_contexts"])
|
||||
}
|
||||
if c := strs(rule["status_check_contexts"]); strings.Join(c, ",") != "other/x,mesh/check" {
|
||||
t.Errorf("statuses %v, want other/x kept and mesh/check for the two it replaces (issue 1301)", c)
|
||||
}
|
||||
if r := result(o, "mesh-delivery"); r.Did != DidPutRight {
|
||||
t.Errorf("outcome %+v, want put right", r)
|
||||
|
||||
@@ -26,9 +26,47 @@ const (
|
||||
MainBranch = "main"
|
||||
)
|
||||
|
||||
// RequiredStatuses are what main requires in every listed repository from its first commit (ADR 0428 §5): the
|
||||
// repository's own check and the module graph's gate. A status already on the rule is kept beside them.
|
||||
var RequiredStatuses = []string{"mesh/merge-gate", "mesh/repo-check"}
|
||||
// CheckStatus is the one status main requires (ADR 0447 §1, issue 1301): the checker's verdict on a pull request's
|
||||
// head merged onto its trunk's head. It replaces GateStatus and OwnCheckStatus, and is never dropped.
|
||||
const CheckStatus = "mesh/check"
|
||||
|
||||
// GateStatus and OwnCheckStatus are the two statuses CheckStatus replaces: the controller's merge gate and the
|
||||
// repository's own merge-check.sh (ADR 0238 §4, ADR 0428 §5 before ADR 0447 §1).
|
||||
const (
|
||||
GateStatus = "mesh/merge-gate"
|
||||
OwnCheckStatus = "mesh/repo-check"
|
||||
)
|
||||
|
||||
// wanted is what a main rule requiring have should require: have with GateStatus and OwnCheckStatus replaced by
|
||||
// CheckStatus, every other status kept in its order (ADR 0447 §1). A rule requiring OwnCheckStatus with neither
|
||||
// GateStatus nor CheckStatus is judged by its own script alone, as hq is until issue 1296 settles how hq is
|
||||
// checked (ADR 0503 §5): the checker writes no status there (ADR 0484 §8), so it is left as it is. This reads the
|
||||
// rule, not a list of names (lead decision L11 of R4e).
|
||||
func wanted(have []string) []string {
|
||||
if !has(have, GateStatus) && !has(have, CheckStatus) && has(have, OwnCheckStatus) {
|
||||
return have
|
||||
}
|
||||
out := []string{}
|
||||
for _, s := range have {
|
||||
if s != GateStatus && s != OwnCheckStatus && s != CheckStatus {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return append(out, CheckStatus)
|
||||
}
|
||||
|
||||
func has(list []string, want string) bool {
|
||||
for _, s := range list {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func sameList(a, b []string) bool {
|
||||
return strings.Join(a, "\x00") == strings.Join(b, "\x00")
|
||||
}
|
||||
|
||||
// The steps of one repository, as an outcome names a refused one (ADR 0428 §6).
|
||||
const (
|
||||
@@ -42,6 +80,10 @@ const (
|
||||
// StepRuleAfterMove reads main's rule back once the repository is in the organisation and puts it right: the
|
||||
// forge drops from a merge whitelist a user without access, and under the setup account the tools' have none.
|
||||
StepRuleAfterMove = "rule-after-move"
|
||||
// StepList reads the organisation's repositories, for those the value does not list (issue 1301).
|
||||
StepList = "list"
|
||||
// StepSwap swaps the two old statuses for CheckStatus on a repository the value does not list (issue 1301).
|
||||
StepSwap = "swap"
|
||||
)
|
||||
|
||||
// What a pass did with one repository.
|
||||
@@ -157,9 +199,73 @@ func (s *Syncer) Sync(ctx context.Context, v Value) (Outcome, error) {
|
||||
}
|
||||
out.Repositories = append(out.Repositories, r)
|
||||
}
|
||||
for _, r := range s.switchUnlisted(ctx, v) {
|
||||
if r.Did == DidRefused {
|
||||
s.say("refused: %s/%s at the step %s: %s", s.Org, r.Name, r.Step, r.Said)
|
||||
} else {
|
||||
s.say("%s: %s/%s (%s in place of %s and %s)", r.Did, s.Org, r.Name, CheckStatus, GateStatus, OwnCheckStatus)
|
||||
}
|
||||
out.Repositories = append(out.Repositories, r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// orgPage is how many repositories one read of the organisation's list asks for.
|
||||
const orgPage = 50
|
||||
|
||||
// switchUnlisted swaps the two old statuses for CheckStatus on main's rule of every repository of the organisation
|
||||
// the value does not list whose main requires GateStatus (issue 1301, lead decisions L4 and L11 of R4e): the
|
||||
// repositories the merge gate judges are switched in the delivery that has the checker write mesh/check (ADR 0447
|
||||
// §5), with no list kept by hand. Only the statuses of a rule covering main are sent; nothing else of a rule the
|
||||
// value does not hold is changed (ADR 0428 §7 for all of it but the statuses the mesh itself requires). A main that
|
||||
// does not require GateStatus gets no call. It answers only what it did: a repository switched, or refused.
|
||||
func (s *Syncer) switchUnlisted(ctx context.Context, v Value) []Result {
|
||||
var names []string
|
||||
for page := 1; ; page++ {
|
||||
var repos []struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
q := "?limit=" + fmt.Sprint(orgPage) + "&page=" + fmt.Sprint(page)
|
||||
if err := s.Forge.GetJSON(ctx, "/api/v1/orgs/"+url.PathEscape(s.Org)+"/repos"+q, &repos); err != nil {
|
||||
return []Result{refused("(the organisation's repositories)", StepList, err)}
|
||||
}
|
||||
for _, r := range repos {
|
||||
if _, listed := v[r.Name]; !listed && r.Name != "" {
|
||||
names = append(names, r.Name)
|
||||
}
|
||||
}
|
||||
if len(repos) == 0 || page > 1000 {
|
||||
break
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
var out []Result
|
||||
for _, n := range names {
|
||||
rules, err := s.rules(ctx, s.Org, n)
|
||||
if err != nil {
|
||||
out = append(out, refused(n, StepRead, err))
|
||||
continue
|
||||
}
|
||||
var did *Result
|
||||
for _, r := range rules {
|
||||
if ok, _ := path.Match(ruleName(r), MainBranch); !ok || !has(required(r), GateStatus) {
|
||||
continue
|
||||
}
|
||||
body := map[string]any{"enable_status_check": true, "status_check_contexts": wanted(required(r))}
|
||||
if err := s.Forge.SendJSON(ctx, http.MethodPatch,
|
||||
repoAPI(s.Org, n)+"/branch_protections/"+url.PathEscape(ruleName(r)), body, nil); err != nil {
|
||||
did = &Result{Name: n, Did: DidRefused, Step: StepSwap, Said: err.Error()}
|
||||
break
|
||||
}
|
||||
did = &Result{Name: n, Did: DidPutRight}
|
||||
}
|
||||
if did != nil {
|
||||
out = append(out, *did)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// waitForForge is cmd/agent-teams' (teams.go waitForForge): the forge starting with the module answers late.
|
||||
func (s *Syncer) waitForForge(ctx context.Context) error {
|
||||
deadline := time.Now().Add(s.WaitFor)
|
||||
@@ -309,7 +415,7 @@ func strList(v any) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// statuses is what a rule requires, with RequiredStatuses added and none dropped.
|
||||
// statuses is what the rules require together, as wanted makes it.
|
||||
func statuses(rules ...rule) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
@@ -320,16 +426,19 @@ func statuses(rules ...rule) []string {
|
||||
}
|
||||
}
|
||||
for _, r := range rules {
|
||||
if r != nil && r["enable_status_check"] == true {
|
||||
for _, s := range strList(r["status_check_contexts"]) {
|
||||
add(s)
|
||||
}
|
||||
for _, s := range required(r) {
|
||||
add(s)
|
||||
}
|
||||
}
|
||||
for _, s := range RequiredStatuses {
|
||||
add(s)
|
||||
return wanted(out)
|
||||
}
|
||||
|
||||
// required is what a rule requires: its statuses when it checks them, else none.
|
||||
func required(r rule) []string {
|
||||
if r == nil || r["enable_status_check"] != true {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
return strList(r["status_check_contexts"])
|
||||
}
|
||||
|
||||
// narrowed is main's rule of ADR 0398 §2 (protection.ts mainRuleBody): no push, only the tools' account merges,
|
||||
@@ -344,7 +453,7 @@ func narrowed(r rule, toolsAccount string, also ...rule) map[string]any {
|
||||
}
|
||||
}
|
||||
|
||||
// isRight says whether a rule already is narrowed and requires both statuses.
|
||||
// isRight says whether a rule already is narrowed and requires what wanted makes of it.
|
||||
func isRight(r rule, toolsAccount string) bool {
|
||||
if r["enable_push"] != false || r["enable_push_whitelist"] == true || r["enable_merge_whitelist"] != true {
|
||||
return false
|
||||
@@ -352,26 +461,15 @@ func isRight(r rule, toolsAccount string) bool {
|
||||
if m := strList(r["merge_whitelist_usernames"]); len(m) != 1 || m[0] != toolsAccount {
|
||||
return false
|
||||
}
|
||||
return r["enable_status_check"] == true && hasAll(strList(r["status_check_contexts"])) &&
|
||||
have := strList(r["status_check_contexts"])
|
||||
return r["enable_status_check"] == true && len(have) > 0 && sameList(have, wanted(have)) &&
|
||||
r["block_admin_merge_override"] == true
|
||||
}
|
||||
|
||||
func hasAll(have []string) bool {
|
||||
for _, want := range RequiredStatuses {
|
||||
found := false
|
||||
for _, h := range have {
|
||||
found = found || h == want
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// putRight makes main's rule in an organisation's repository the narrowed one, keeping every status it requires.
|
||||
// putRight makes main's rule in an organisation's repository the narrowed one, keeping every status it requires but
|
||||
// the two CheckStatus replaces (ADR 0447 §1).
|
||||
// A repository with no rule named main gets one, carrying what a glob rule covering main asks (client.ts
|
||||
// ensureMainMerges, issue 948). It never takes a status away and never unprotects (ADR 0428 §5, §7).
|
||||
// ensureMainMerges, issue 948). It takes away no other status and never unprotects (ADR 0428 §5, §7).
|
||||
func (s *Syncer) putRight(ctx context.Context, owner, name string) (bool, error) {
|
||||
rules, err := s.rules(ctx, owner, name)
|
||||
if err != nil {
|
||||
|
||||
@@ -17,8 +17,8 @@ import (
|
||||
const mainRule = "main"
|
||||
|
||||
// 1124/AC2: a listed repository that does not exist is made under the setup account, given main's rule letting that
|
||||
// login alone push, given its first commit, narrowed to no push and the tools' merge requiring mesh/repo-check (and
|
||||
// mesh/merge-gate, 1124/AC5), and only then moved into the organisation. Nothing reaches the organisation before the
|
||||
// login alone push, given its first commit, narrowed to no push and the tools' merge requiring mesh/check (1124/AC5
|
||||
// as issue 1301 changes it), and only then moved into the organisation. Nothing reaches the organisation before the
|
||||
// narrowed rule, and nothing else is pushed to main.
|
||||
func TestNewRepositoryIsMadeInOrder(t *testing.T) {
|
||||
f := newFake()
|
||||
@@ -67,8 +67,8 @@ func TestNewRepositoryIsMadeInOrder(t *testing.T) {
|
||||
t.Errorf("main's merge whitelist is not the tools' account alone: %v", rule)
|
||||
}
|
||||
checks := strs(rule["status_check_contexts"])
|
||||
if rule["enable_status_check"] != true || !has(checks, "mesh/repo-check") || !has(checks, "mesh/merge-gate") {
|
||||
t.Errorf("main does not require mesh/repo-check and mesh/merge-gate (1124/AC5): %v", rule)
|
||||
if rule["enable_status_check"] != true || strings.Join(checks, ",") != CheckStatus {
|
||||
t.Errorf("main does not require mesh/check alone (1124/AC5, issue 1301): %v", rule)
|
||||
}
|
||||
if r := result(o, "mesh-delivery"); r.Did != DidMade || r.Step != "" {
|
||||
t.Errorf("outcome %+v, want made", r)
|
||||
@@ -87,7 +87,7 @@ func TestNewRepositoryIsMadeInOrder(t *testing.T) {
|
||||
|
||||
// 1124/AC2: the forge drops from a merge whitelist a user without access to the repository, and under the setup
|
||||
// account the tools' account has none; so once the repository is in the organisation its main rule is read back and
|
||||
// put right: the tools' account the only one that merges, both statuses required.
|
||||
// put right: the tools' account the only one that merges, mesh/check required.
|
||||
func TestOrganisationRuleListsTheToolsAfterTheMove(t *testing.T) {
|
||||
f := newFake()
|
||||
s, _ := syncer(t, f)
|
||||
@@ -104,8 +104,8 @@ func TestOrganisationRuleListsTheToolsAfterTheMove(t *testing.T) {
|
||||
t.Errorf("after the move main takes a push: %v", rule)
|
||||
}
|
||||
checks := strs(rule["status_check_contexts"])
|
||||
if rule["enable_status_check"] != true || !has(checks, "mesh/repo-check") || !has(checks, "mesh/merge-gate") {
|
||||
t.Errorf("after the move main does not require both statuses: %v", rule)
|
||||
if rule["enable_status_check"] != true || strings.Join(checks, ",") != CheckStatus {
|
||||
t.Errorf("after the move main does not require mesh/check alone: %v", rule)
|
||||
}
|
||||
if r := result(o, "mesh-delivery"); r.Did != DidMade {
|
||||
t.Errorf("outcome %+v, want made", r)
|
||||
@@ -136,7 +136,8 @@ func TestAdminRuleLetsTheSetupLoginAlonePush(t *testing.T) {
|
||||
}
|
||||
|
||||
// 1124/AC3: a listed repository that exists in the organisation is not made again, only its main rule put right;
|
||||
// a rule still letting the setup login push is narrowed, its statuses kept (1124/AC5).
|
||||
// a rule still letting the setup login push is narrowed, its statuses kept but the two mesh/check replaces (1124/AC5,
|
||||
// issue 1301).
|
||||
func TestExistingOrganisationRepositoryIsOnlyPutRight(t *testing.T) {
|
||||
f := newFake()
|
||||
hq := f.add("novox/hq", map[string]map[string]any{mainRule: {
|
||||
@@ -157,14 +158,11 @@ func TestExistingOrganisationRepositoryIsOnlyPutRight(t *testing.T) {
|
||||
t.Errorf("hq's rule not narrowed: %v", rule)
|
||||
}
|
||||
checks := strs(rule["status_check_contexts"])
|
||||
for _, c := range []string{"other/x", "mesh/merge-gate", "mesh/repo-check"} {
|
||||
if !has(checks, c) {
|
||||
t.Errorf("hq's rule lacks %s: %v", c, checks)
|
||||
}
|
||||
if strings.Join(checks, ",") != "other/x,mesh/check" {
|
||||
t.Errorf("hq's rule requires %v, want other/x and mesh/check", checks)
|
||||
}
|
||||
made := bare.Rules[mainRule]
|
||||
if made == nil || made["enable_push"] != false || !has(strs(made["status_check_contexts"]), "mesh/repo-check") ||
|
||||
!has(strs(made["status_check_contexts"]), "mesh/merge-gate") {
|
||||
if made == nil || made["enable_push"] != false || strings.Join(strs(made["status_check_contexts"]), ",") != CheckStatus {
|
||||
t.Errorf("bare's rule: %v", made)
|
||||
}
|
||||
if hq.Commits != 0 || bare.Commits != 0 {
|
||||
@@ -214,8 +212,8 @@ func TestRepositoryUnderTheSetupAccountIsFinished(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1124/AC5: putting a rule right adds a missing one of mesh/repo-check and mesh/merge-gate and keeps every status
|
||||
// already there; no pass drops one.
|
||||
// 1124/AC5 as issue 1301 changes it: putting a rule right adds mesh/check when the rule requires mesh/merge-gate,
|
||||
// and keeps every other status there; a rule requiring mesh/repo-check alone keeps it (ADR 0503 §5).
|
||||
func TestStatusesAreAddedAndNeverDropped(t *testing.T) {
|
||||
f := newFake()
|
||||
only := f.add("novox/only", map[string]map[string]any{mainRule: {
|
||||
@@ -224,15 +222,15 @@ func TestStatusesAreAddedAndNeverDropped(t *testing.T) {
|
||||
}}, "main")
|
||||
more := f.add("novox/more", map[string]map[string]any{mainRule: {
|
||||
"enable_push": false, "enable_merge_whitelist": true, "merge_whitelist_usernames": []any{tools},
|
||||
"enable_status_check": true, "status_check_contexts": []any{"mesh/merge-gate", "mesh/repo-check", "mesh/delivery"},
|
||||
"enable_status_check": true, "status_check_contexts": []any{"mesh/delivery", "mesh/check"},
|
||||
"block_admin_merge_override": true, // right since issue 1141
|
||||
}}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{"only": {}, "more": {}})
|
||||
if c := strs(only.Rules[mainRule]["status_check_contexts"]); !has(c, "mesh/merge-gate") || !has(c, "mesh/repo-check") {
|
||||
if c := strs(only.Rules[mainRule]["status_check_contexts"]); strings.Join(c, ",") != "mesh/repo-check" {
|
||||
t.Errorf("only: %v", c)
|
||||
}
|
||||
if c := strs(more.Rules[mainRule]["status_check_contexts"]); len(c) != 3 || !has(c, "mesh/delivery") {
|
||||
if c := strs(more.Rules[mainRule]["status_check_contexts"]); len(c) != 2 || !has(c, "mesh/delivery") {
|
||||
t.Errorf("more lost a status: %v", c)
|
||||
}
|
||||
for _, w := range f.writes() {
|
||||
@@ -314,7 +312,8 @@ func TestRefusedStepStopsThatRepositoryOnly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1124/AC7: a repository taken out of the value gets no call at all.
|
||||
// 1124/AC7: a repository taken out of the value is not changed: its rules are read (issue 1301 switches one whose
|
||||
// main requires mesh/merge-gate), nothing is written, and one still under the setup account is not asked about.
|
||||
func TestRepositoryOutOfTheValueIsNotTouched(t *testing.T) {
|
||||
f := newFake()
|
||||
f.add("novox/old", map[string]map[string]any{mainRule: {"enable_push": true}}, "main")
|
||||
@@ -323,7 +322,7 @@ func TestRepositoryOutOfTheValueIsNotTouched(t *testing.T) {
|
||||
run(t, s, Value{})
|
||||
run(t, s, Value{"new": {}})
|
||||
for _, c := range f.all() {
|
||||
if strings.Contains(c, "old") {
|
||||
if strings.Contains(c, "older") || (strings.Contains(c, "old") && !strings.HasPrefix(c, "GET ")) {
|
||||
t.Errorf("a repository out of the value was asked: %s", c)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,9 @@ import { promisify } from "node:util";
|
||||
import type { ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import type { CommitStatus } from "./client.js";
|
||||
import { checkName } from "./guard.js";
|
||||
import { CHECK_STATUS } from "./pulls.js";
|
||||
|
||||
export { CHECK_STATUS };
|
||||
|
||||
/** How a command is run: docker on the forge's machine, or a test's. */
|
||||
export type Runner = (file: string, args: string[]) => Promise<{ stdout: string; code: number }>;
|
||||
@@ -91,11 +94,18 @@ const states = new Set(["pending", "success", "error", "failure", "warning"]);
|
||||
/** The merge check's contexts (pulls.ts): a branch's protection may require them. */
|
||||
const mergeCheckContexts = new Set(["mesh/merge-gate", "mesh/repo-check"]);
|
||||
|
||||
|
||||
/** A status mesh-delivery asks for, checked: one of the forge's states, a context of the mesh's own, a
|
||||
* bounded description. */
|
||||
export function deliveryStatus(context: string, state: string, description: string, target?: string) {
|
||||
if (!/^mesh\/[a-z-]+$/.test(context)) throw new Error(`${context} is not a status of the mesh's own`);
|
||||
if (!states.has(state)) throw new Error(`${state} is not a status the forge keeps`);
|
||||
// mesh/check is the checker's verdict and the one status main requires (ADR 0447 §1): no hand sets it in any
|
||||
// state, as issue 293 closed for the two before it. No tool writes it, since a tool is told no caller (R4e finding K1):
|
||||
// check.ts writes it from the checker's owed state (ADR 0521).
|
||||
if (context === CHECK_STATUS) {
|
||||
throw new Error(`${CHECK_STATUS} is the checker's status: it is written from mesh-delivery-checker.owed alone, never by a tool`);
|
||||
}
|
||||
// The merge check's statuses are its verdict's, set when the controller says it, and a branch may require
|
||||
// them: never set to a verdict by hand, so nothing passes — or fails — a merge past the check (novox/hq
|
||||
// issue 293). Pending is no verdict: it is what this holder says on every new head until the verdict comes,
|
||||
@@ -126,7 +136,7 @@ export interface StatusWriters {
|
||||
export function commitStatusTool(writers: StatusWriters): ToolDefinition {
|
||||
return {
|
||||
name: "gitea_commit_status",
|
||||
description: "Set one of the mesh's statuses on a commit (mesh/delivery, mesh/delivery-group): pending, success, error, failure or warning, a short description, and the page it links to. The merge check's statuses (mesh/merge-gate, mesh/repo-check) take pending only — a recheck putting them back until the fresh verdict sets them — on the head's whole sha, in their turn after the check's earlier writes.",
|
||||
description: "Set one of the mesh's statuses on a commit (mesh/delivery, mesh/delivery-group): pending, success, error, failure or warning, a short description, and the page it links to. mesh/check, the status main requires, is refused: no tool writes it: it is written from the checker's own state. hq's mesh/repo-check (and mesh/merge-gate, kept for a revert) take pending only — a recheck putting them back until the fresh verdict sets them — on the head's whole sha, in their turn after the check's earlier writes.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
|
||||
@@ -212,7 +212,8 @@ export class GateWriter {
|
||||
let since = h.pendingAt;
|
||||
if (since === undefined) {
|
||||
const gate = await this.forge.gateOf(owner, repo, sha);
|
||||
if (gate && gate.state !== "pending") return;
|
||||
// No gate on the forge: none was written (its base does not require it, issue 1301), and nothing is said.
|
||||
if (!gate || gate.state !== "pending") return;
|
||||
since = gate?.updated_at ? Date.parse(gate.updated_at) : undefined;
|
||||
}
|
||||
if (since !== undefined && at.getTime() < since - CLOCK_SLACK_MS) return;
|
||||
|
||||
@@ -3,7 +3,7 @@ module gitea
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
+38
-10
@@ -12,7 +12,13 @@
|
||||
// Consumes mesh-controller.checked — a pull request's merge check, judged — and sets it as the head
|
||||
// commit's statuses: `mesh/merge-gate`, the modules of the mesh's graph the change touches, and
|
||||
// `mesh/repo-check`, the repository's own merge-check.sh (novox/hq ADR 0237 as amended); with a comment
|
||||
// saying why when the gate is not a pass or the repository's own check failed.
|
||||
// saying why when the gate is not a pass or the repository's own check failed. Of the two, only those the base
|
||||
// still requires are written (issue 1301): main requires `mesh/check` — the checker's — everywhere but hq, whose main
|
||||
// requires `mesh/repo-check` alone (ADR 0503 §5).
|
||||
//
|
||||
// Reads mesh-delivery-checker.owed — what the checker owes the forge, mesh/check on each head it judged — by a watch, and
|
||||
// writes it (check.ts; ADR 0521, R4e finding R2-1). No tool writes mesh/check (finding K1), and no event: the watch hands
|
||||
// over the latest value of every key whenever this module starts.
|
||||
//
|
||||
// Consumes node-build-agent.started — a build machine took an ask — and says, on the head a pending check is of,
|
||||
// which machine builds it since when (novox/hq issue 446). This module is the one writer of mesh/merge-gate.
|
||||
@@ -38,12 +44,15 @@
|
||||
|
||||
import { emit, on } from "@novox/mesh-sdk/events";
|
||||
import { registerModuleTools } from "@novox/mesh-sdk/tools";
|
||||
import { statesAnnouncement } from "./kept.js";
|
||||
import { commitStatusTool } from "./delivery.js";
|
||||
import { CHECKER_OWED, CheckStatuses, watchOwed, type OwedChange } from "./check.js";
|
||||
import { state } from "@novox/mesh-sdk/state";
|
||||
import { announce, keepWatermark, readWatermark } from "./announce.js";
|
||||
import { mergeTool } from "./merge.js";
|
||||
import { GiteaAdmin, GiteaClient, movedSince } from "./client.js";
|
||||
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, composedGate, groupGateStatus, headsToAnnounce, statusesFor, type Announced, type Checked,
|
||||
repoCheckFacts, reposToLook } from "./pulls.js";
|
||||
import { CHECK_CONTEXT, CHECK_STATUS, REPO_CHECK_CONTEXT, commentFor, composedGate, groupGateStatus, headsToAnnounce, maintained,
|
||||
requiredOf, statusesFor, writesGate, type Announced, type Checked, repoCheckFacts, reposToLook } from "./pulls.js";
|
||||
import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
|
||||
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
|
||||
import { keepSiblings, noteComposed, noteVerdict, readSiblings, siblingsFile, siblingsOf, weighSiblings } from "./siblings.js";
|
||||
@@ -247,8 +256,11 @@ async function announceHead(client: GiteaClient, repo: RepoOf, pull: OpenOf): Pr
|
||||
// Said, so an operator knows the mesh was asked to check it.
|
||||
console.log(`[gitea] announced ${key} at ${sha.slice(0, 8)} to be checked before it merges`);
|
||||
// Pending until the verdict comes, so the pull request says a check is running rather than nothing — and
|
||||
// where it is: in the build queue from now, until a build machine takes it (novox/hq issue 446).
|
||||
await gates.pending(repo.owner, repo.name, sha);
|
||||
// where it is: in the build queue from now, until a build machine takes it (novox/hq issue 446). Only where the base
|
||||
// still requires mesh/merge-gate (issue 1301): a main that requires mesh/check is the checker's, and hq's requires
|
||||
// mesh/repo-check alone; a rule that cannot be read is said pending, as before.
|
||||
const rule = await client.appliedProtection(repo.owner, repo.name, String(pull.base || "main")).catch(() => undefined);
|
||||
if (writesGate(requiredOf(rule))) await gates.pending(repo.owner, repo.name, sha);
|
||||
return true;
|
||||
}
|
||||
type OpenOf = Awaited<ReturnType<GiteaClient["listOpenPullsSince"]>>[number];
|
||||
@@ -257,7 +269,7 @@ type OpenOf = Awaited<ReturnType<GiteaClient["listOpenPullsSince"]>>[number];
|
||||
async function alreadyChecked(client: GiteaClient, repo: RepoOf, pull: OpenOf): Promise<boolean> {
|
||||
try {
|
||||
const checks = await client.headChecks(repo.owner, repo.name, String(pull.head_sha));
|
||||
return CHECK_CONTEXT in checks || REPO_CHECK_CONTEXT in checks;
|
||||
return CHECK_CONTEXT in checks || REPO_CHECK_CONTEXT in checks || CHECK_STATUS in checks;
|
||||
} catch {
|
||||
return false; // the forge cannot say: announced, and the bus's message id collapses it if it was
|
||||
}
|
||||
@@ -394,16 +406,20 @@ async function setVerdict(client: GiteaClient, event: { body: unknown }): Promis
|
||||
// (novox/hq ADR 0323 §4): a check asked before the trunk moved. It is said so, pending with the reason, in place of
|
||||
// its pass or failure; a trunk that cannot be read lets the verdict stand, and the merge tool compares the commits.
|
||||
const trunk = base ?? c.plan?.base;
|
||||
// Only the statuses the base still requires are written (issue 1301): on hq, mesh/repo-check alone.
|
||||
const required = trunk ? requiredOf(await client.appliedProtection(c.owner, c.repo, trunk).catch(() => undefined)) : undefined;
|
||||
const kept = maintained(required);
|
||||
if (c.trunk && trunk) {
|
||||
const head = await client.branchHead(c.owner, c.repo, trunk).catch(() => undefined);
|
||||
if (head && !sameCommit(c.trunk, head)) {
|
||||
const description = outOfDateWords(c.trunk, head);
|
||||
for (const context of [...(composedGate(c) ? [] : [CHECK_CONTEXT]), ...(c["repo-check"] ? [REPO_CHECK_CONTEXT] : [])]) {
|
||||
for (const context of [...(composedGate(c) ? [] : [CHECK_CONTEXT]), ...(c["repo-check"] ? [REPO_CHECK_CONTEXT] : [])]
|
||||
.filter((x) => kept.includes(x))) {
|
||||
await client.setCommitStatus(c.owner, c.repo, c.commit, { state: "pending", context, description,
|
||||
...(target ? { target_url: target } : {}) });
|
||||
}
|
||||
// Its account is kept and shown all the same (ADR 0323 §7): what it judged, and that it is out of date.
|
||||
const said = commentFor(c, await repoCheckFacts(client, c, trunk));
|
||||
const said = commentFor(c, await repoCheckFacts(client, c, trunk), required);
|
||||
if (c.number && !composedGate(c)) await client.addComment(c.owner, c.repo, c.number, lateVerdictComment(c, said, description));
|
||||
console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}, ` +
|
||||
`${description}`);
|
||||
@@ -412,10 +428,10 @@ async function setVerdict(client: GiteaClient, event: { body: unknown }): Promis
|
||||
}
|
||||
const facts = await repoCheckFacts(client, c, base ?? c.plan?.base);
|
||||
const gateNow = composedGate(c) ? (await client.headChecks(c.owner, c.repo, c.commit).catch(() => ({} as Record<string, { description?: string }>)))[CHECK_CONTEXT]?.description : undefined;
|
||||
for (const status of statusesFor(c, facts, gateNow)) {
|
||||
for (const status of statusesFor(c, facts, gateNow, required)) {
|
||||
await client.setCommitStatus(c.owner, c.repo, c.commit, target ? { ...status, target_url: target } : status);
|
||||
}
|
||||
const comment = composedGate(c) ? null : commentFor(c, facts);
|
||||
const comment = composedGate(c) ? null : commentFor(c, facts, required);
|
||||
if (comment && c.number) await client.addComment(c.owner, c.repo, c.number, comment);
|
||||
console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}`);
|
||||
}
|
||||
@@ -465,6 +481,11 @@ const gates = new GateWriter({
|
||||
headChecks: (owner, repo, sha) => (gitea as GiteaClient).headChecks(owner, repo, sha),
|
||||
});
|
||||
|
||||
// The one writer of mesh/check, from the checker's owed (check.ts; ADR 0521).
|
||||
const checks = new CheckStatuses({
|
||||
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
||||
});
|
||||
|
||||
// **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own
|
||||
// process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to
|
||||
// reach, nothing is served, as the tools' bundle serves nothing then.
|
||||
@@ -473,6 +494,10 @@ const gates = new GateWriter({
|
||||
const pace = new Pace();
|
||||
if (!gitea) pace.unheard("the forge is not watched: no way to a token", false);
|
||||
|
||||
// The states this module keeps and which others may read, said on the bus before it serves (novox/hq ADR 0495 clause 2).
|
||||
// Said with `emit`, not the SDK's `keeps`, which the toolchain's SDK does not have yet: switch to `keeps` once that SDK is 0.1.15 or later (issue 1852).
|
||||
await emit("states", statesAnnouncement(process.env.MESH_MODULE || "gitea", process.env.MESH_NODE));
|
||||
|
||||
// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729).
|
||||
registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({
|
||||
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
||||
@@ -605,4 +630,7 @@ if (gitea) {
|
||||
})],
|
||||
["node-build-agent.started", () => on("node-build-agent.started", buildStarted)],
|
||||
], () => console.log("[gitea] watching for new repositories and merged pull requests"));
|
||||
// The checker's mesh/check, from its owed state (check.ts; ADR 0521): a read not granted yet is said once and asked again,
|
||||
// apart from the binds above, so neither waits for the other.
|
||||
void watchOwed((handler) => state<OwedChange["value"]>(CHECKER_OWED).watch((change) => handler(change)), checks);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// The states this module keeps on the bus and whether other modules may read them (novox/hq ADR 0495 clause 2; issue
|
||||
// 1790): the module says it itself, from its code, as it starts (index.ts calls keeps with this list), and a reader is
|
||||
// granted what it said (clause 3). A state is readable when another module's manifest reads it; none reads gitea's.
|
||||
// test/kept.test.ts holds the list to the manifest's `state`.
|
||||
export const KEPT: readonly { readonly name: string; readonly readable?: boolean }[] = [
|
||||
{ name: "announced" },
|
||||
{ name: "bound" },
|
||||
];
|
||||
|
||||
/** The announcement of these states, in the layout mesh-sdk's go/announce reads (novox/hq ADR 0495 clause 2): version 1,
|
||||
* the module, the machine when known, and the states sorted by name, `readable` only where true. It is said as the event
|
||||
* `states`, which is the subject mesh.mod.gitea.event.states. gitea says it with the SDK's `emit` because the
|
||||
* toolchain's SDK, which judges R4e's group, predates the SDK's `keeps` (0.1.15); switch to `keeps` once the toolchain's
|
||||
* SDK is 0.1.15 or later (issue 1852). */
|
||||
export function statesAnnouncement(module: string, node: string | undefined) {
|
||||
const states = [...KEPT]
|
||||
.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
|
||||
.map((k) => (k.readable ? { name: k.name, readable: true } : { name: k.name }));
|
||||
return { v: 1, module, ...(node ? { node } : {}), states };
|
||||
}
|
||||
+41
-10
@@ -4,13 +4,18 @@
|
||||
// when a verdict is judged against an earlier trunk commit than the trunk's head, the tool asks the head's check and
|
||||
// puts its statuses back to pending through that writer, so where the check runs is said and no second check is asked.
|
||||
// Each bundle the runtime serves is its own process; from the tools' bundle the pending bypassed the writer's chain.
|
||||
//
|
||||
// **It judges on the status the base requires** (issue 1301; ADR 0447 §1, §4; ADR 0503 §5): mesh/check, the checker's,
|
||||
// whose line names the trunk it judged ("pass <head12> on <trunk12>"); where a base does not require it, the controller's
|
||||
// status it still requires (hq's mesh/repo-check), as before. A mesh/check of another trunk is refused and nothing is
|
||||
// asked: the checker sets it out of date on the trunk's move and checks it again.
|
||||
|
||||
import { announce } from "./announce.js";
|
||||
import type { ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import type { CommitStatus, GiteaClient } from "./client.js";
|
||||
import { mergeableUnder, newestByContext } from "./checks.js";
|
||||
import { MERGE_METHODS, Refused, checkName } from "./guard.js";
|
||||
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js";
|
||||
import { CHECK_CONTEXT, CHECK_STATUS, REPO_CHECK_CONTEXT, maintained, requiredOf } from "./pulls.js";
|
||||
import { approvedHeadsOf, groupRefusal, memberKey, namesSiblings, notApproved, readSiblings, siblingOutOfDate, siblingsFile } from "./siblings.js";
|
||||
import { announcementOf, askedWords, outOfDateWords, sameCommit, tookFile, tookWords, twelve, verdictAtTrunk, waitsForComposed } from "./trunk.js";
|
||||
|
||||
@@ -32,7 +37,7 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
||||
return {
|
||||
name: "gitea_merge_pull_request",
|
||||
description: "Merge a pull request at the head you name — the commit you reviewed — when that is still its head " +
|
||||
"and every status its base branch's protection requires succeeded there (mesh/merge-gate, mesh/repo-check), " +
|
||||
"and every status its base branch's protection requires succeeded there (mesh/check; hq's mesh/repo-check), " +
|
||||
"judged against the trunk's head as it is now, optionally deleting the source branch afterwards. Refused " +
|
||||
"otherwise, with nothing merged: no tool merges past a check, and none merges a head that moved after it was " +
|
||||
"reviewed. A verdict judged against an earlier trunk commit is refused, naming both, and the head's check against " +
|
||||
@@ -108,11 +113,33 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
||||
// head makes — never again while one is on its way. The caller asks again at the reviewed head once it passes.
|
||||
const trunkHead = await gitea.branchHead(owner, repo, pull.base);
|
||||
const checks = await gitea.headChecks(owner, repo, pull.head_sha);
|
||||
const at = verdictAtTrunk(checks, trunkHead);
|
||||
const required = requiredOf(rule) ?? [];
|
||||
if (required.includes(CHECK_STATUS)) {
|
||||
// **mesh/check is the checker's** (ADR 0447 §4): it is judged here against the trunk's head, and refused when it
|
||||
// is not current; nothing is written or asked — the checker checks a head again when its trunk moves.
|
||||
const check = checks[CHECK_STATUS];
|
||||
const at = verdictAtTrunk(checks, trunkHead, CHECK_STATUS);
|
||||
if (!check || check.state === "pending") {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its ${CHECK_STATUS} is ` +
|
||||
`${check ? `pending (${check.description ?? ""})` : "not set yet"}; the checker answers it. Ask again at ` +
|
||||
`${twelve(pull.head_sha)} once it passes against ${pull.base}'s head ${twelve(trunkHead)}. Nothing was merged`);
|
||||
}
|
||||
if (!at.current) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its ${CHECK_STATUS} ` +
|
||||
`${at.judged ? `was judged on trunk ${twelve(at.judged)}` : "names no trunk commit it was judged on"}, and ` +
|
||||
`${pull.base} is now ${twelve(trunkHead)}. The checker checks it again against the trunk's head (novox/hq ` +
|
||||
`ADR 0447); this tool asks nothing. Ask again at ${twelve(pull.head_sha)} once it passes. Nothing was merged`);
|
||||
}
|
||||
}
|
||||
// Where the base still requires the controller's statuses (hq's mesh/repo-check; mesh/merge-gate only on a
|
||||
// revert), the first it requires judges, as before (ADR 0323 §5).
|
||||
const kept = maintained(required);
|
||||
const judge = kept[0] ?? CHECK_CONTEXT;
|
||||
const at = kept.length ? verdictAtTrunk(checks, trunkHead, judge) : { current: true, onItsWay: false };
|
||||
if (!at.current) {
|
||||
// A verdict put out of date by a sibling's move says so in its own words (ADR 0382 §5); it is asked again as a
|
||||
// trunk's move is, and a failed ask puts those words back, not the trunk's.
|
||||
const siblingWords = siblingOutOfDate(checks[CHECK_CONTEXT]?.description) ? checks[CHECK_CONTEXT]?.description : undefined;
|
||||
const siblingWords = siblingOutOfDate(checks[judge]?.description) ? checks[judge]?.description : undefined;
|
||||
const judged = siblingWords ? `was put out of date by a sibling's move (${siblingWords})`
|
||||
: at.judged ? `was judged against trunk ${twelve(at.judged)}` : "names no trunk commit it was judged against";
|
||||
const took = tookWords(tookFile(deps.stateDir), `${owner}/${repo}`);
|
||||
@@ -120,21 +147,22 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
||||
// **A member that waits for its group's composed check is not announced again** (novox/hq issue 1106): its own
|
||||
// check can only say "composed" once more, so an ask here brought a verdict naming no trunk and the next merge was
|
||||
// refused the same way for ever. The composed check is mesh-delivery's to ask; this says whose verdict is awaited.
|
||||
if (waitsForComposed(checks[CHECK_CONTEXT])) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its merge gate ${checks[CHECK_CONTEXT]?.description}; the ` +
|
||||
if (waitsForComposed(checks[judge])) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its merge gate ${checks[judge]?.description}; the ` +
|
||||
`verdict that names its trunk commit is its group's composed check's, and mesh-delivery asks it, this tool asks no check ` +
|
||||
`(main is now ${now}). Ask again once the group's composed check has passed. Nothing was merged`);
|
||||
}
|
||||
if (at.onItsWay) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` +
|
||||
`${now}. Its check is already asked and not yet answered (${checks[CHECK_CONTEXT]?.description}), so it is not ` +
|
||||
`${now}. Its check is already asked and not yet answered (${checks[judge]?.description}), so it is not ` +
|
||||
`asked again; ${took}. Ask again at ${head} once both statuses pass: this tool never merges by itself ` +
|
||||
`(novox/hq ADR 0323). A check that never answers is asked again with mesh-delivery.recheck. Nothing was merged`);
|
||||
}
|
||||
// Pending first, so a second ask finds the check on its way; then the head is announced as a new head is. An
|
||||
// announcement that cannot be made puts the statuses back to out of date, so the next ask asks again rather
|
||||
// than wait for ever on a check nobody asked.
|
||||
const contexts = [CHECK_CONTEXT, ...(checks[REPO_CHECK_CONTEXT] ? [REPO_CHECK_CONTEXT] : [])];
|
||||
// The statuses the base requires of the controller's (issue 1301): hq's mesh/repo-check alone.
|
||||
const contexts = kept.filter((c) => c === judge || (c === REPO_CHECK_CONTEXT && checks[REPO_CHECK_CONTEXT]));
|
||||
// Through the module's one writer of the merge check (gate.ts, novox/hq issue 456): the pending opens a new
|
||||
// check of the head, so where it runs is said ("building on …") and its verdict is not weighed as older.
|
||||
const say = async (description: string) => {
|
||||
@@ -167,9 +195,12 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
||||
// **A member judged beside its siblings merges only with its group passed and reviewed** (novox/hq ADR 0382 §6): the
|
||||
// siblings its verdict names are the holder's record (siblings.ts); a status that says it read siblings with no record
|
||||
// of them is no verdict to merge on, never read as none.
|
||||
const heard = readSiblings(siblingsFile(deps.stateDir))[memberKey(owner, repo, number)];
|
||||
// Under mesh/check (issue 1301) the group is the checker's one verdict on every member's head (ADR 0447 §2) and the
|
||||
// Approve's heads above; a record the controller's verdicts left before the switch judges nothing there.
|
||||
const heard = required.includes(CHECK_STATUS) ? undefined : readSiblings(siblingsFile(deps.stateDir))[memberKey(owner, repo, number)];
|
||||
const siblings = heard && sameCommit(heard.head, pull.head_sha) ? heard.siblings : [];
|
||||
if (!siblings.length && namesSiblings(statuses.find((s) => s.context === CHECK_CONTEXT)?.description)) {
|
||||
if (!siblings.length && !required.includes(CHECK_STATUS) &&
|
||||
namesSiblings(statuses.find((s) => s.context === CHECK_CONTEXT)?.description)) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict read siblings, and which ones is not held ` +
|
||||
`here, so the group cannot be weighed. Ask its check again at ${twelve(pull.head_sha)}. Nothing was merged`);
|
||||
}
|
||||
|
||||
@@ -172,7 +172,7 @@
|
||||
},
|
||||
"repositories": {
|
||||
"type": "object",
|
||||
"description": "The repositories of the organisation novox the mesh holds (ADR 0428): each by name, {\"<name>\": {\"private\": true, \"description\": \"...\"}}. One that does not exist is made by the module's setup login under its own account, its main protected and given a first commit (a README naming the description and a merge-check.sh), its rule narrowed to no push and the tools' merge requiring mesh/repo-check and mesh/merge-gate, and only then moved into the organisation; one that exists only has main's rule put right, its statuses kept. A repository taken out of the value is never touched.",
|
||||
"description": "The repositories of the organisation novox the mesh holds (ADR 0428): each by name, {\"<name>\": {\"private\": true, \"description\": \"...\"}}. One that does not exist is made by the module's setup login under its own account, its main protected and given a first commit (a README naming the description and a merge-check.sh), its rule narrowed to no push and the tools' merge requiring mesh/check, and only then moved into the organisation; one that exists only has main's rule put right, mesh/check in place of mesh/merge-gate and mesh/repo-check and its other statuses kept (a main requiring mesh/repo-check alone keeps it). Every other repository whose main requires mesh/merge-gate has the two swapped for mesh/check; nothing else of a repository out of the value is touched.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
}
|
||||
@@ -543,6 +543,7 @@
|
||||
"ssh-client.keys",
|
||||
"mesh-build-go.keys",
|
||||
"mesh-build-node.keys",
|
||||
"mesh-build-image.keys"
|
||||
"mesh-build-image.keys",
|
||||
"mesh-delivery-checker.owed"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "tsc client.ts token.ts index.ts provisioner/index.ts provisioner/teams.ts tools/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist",
|
||||
"test": "npm run build && node --test --experimental-strip-types 'test/*.test.ts'"
|
||||
"test": "npm run build && tsc --module NodeNext --moduleResolution NodeNext --target ES2022 --types node --rootDir . --outDir . $(find . -name '*.ts' ! -name '*.d.ts' ! -path './test/*' ! -path './node_modules/*' ! -path './dist/*') && node --test --experimental-strip-types 'test/*.test.ts'"
|
||||
},
|
||||
"dependencies": {
|
||||
"@novox/mesh-sdk": "^0.1.1"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// A branch's protection (novox/hq ADR 0237): what makes a pull request wait for the mesh's merge check —
|
||||
// `mesh/merge-gate`, the module graph's gate, and `mesh/repo-check`, the repository's own tests — before it
|
||||
// may merge. Pure, so it is tested without a forge; the client does the asking (client.ts).
|
||||
// A branch's protection (novox/hq ADR 0237): what makes a pull request wait for the mesh's check — `mesh/check`,
|
||||
// the checker's verdict on the head merged onto its trunk (ADR 0447 §1; hq's `mesh/repo-check` until issue 1296) —
|
||||
// before it may merge. Pure, so it is tested without a forge; the client does the asking (client.ts).
|
||||
|
||||
import { TOOLS_ACCOUNT } from "./tools-account.js";
|
||||
|
||||
@@ -24,7 +24,7 @@ export interface BranchProtection {
|
||||
/** What a branch's protection is set to. A field not given is left as the rule has it (or the forge's
|
||||
* default on a new rule), except pushes, which a new rule refuses unless `push` says otherwise. */
|
||||
export interface ProtectionWanted {
|
||||
/** The statuses a pull request must have succeeded before it merges, e.g. mesh/merge-gate. Empty: none. */
|
||||
/** The statuses a pull request must have succeeded before it merges, e.g. mesh/check. Empty: none. */
|
||||
statusChecks: string[];
|
||||
/** Whether a person may push to the branch directly. */
|
||||
push?: boolean;
|
||||
|
||||
+45
-7
@@ -19,6 +19,34 @@ export const CHECK_CONTEXT = "mesh/merge-gate";
|
||||
/** The status context of the repository's own merge-check.sh, the check's second layer (novox/hq ADR 0237). */
|
||||
export const REPO_CHECK_CONTEXT = "mesh/repo-check";
|
||||
|
||||
/** The one status main requires (novox/hq ADR 0447 §1, issue 1301): the checker's verdict, written by this module from the
|
||||
* checker's owed state (check.ts, ADR 0521). Its line names the head and the trunk commit it judged: "pass <head12> on <trunk12> …". */
|
||||
export const CHECK_STATUS = "mesh/check";
|
||||
|
||||
/** The statuses a branch's rule requires, from the rule the forge applies to it: none when it is unprotected or checks
|
||||
* no status; undefined when the rule could not be read. */
|
||||
export function requiredOf(rule: { enable_status_check?: boolean; status_check_contexts?: string[] | null } | null | undefined): string[] | undefined {
|
||||
if (rule === undefined) return undefined;
|
||||
if (!rule || !rule.enable_status_check) return [];
|
||||
return (rule.status_check_contexts ?? []).filter(Boolean);
|
||||
}
|
||||
|
||||
/**
|
||||
* **The controller's statuses are written only where a base still requires them** (issue 1301; ADR 0503 §3, §5): of
|
||||
* mesh/merge-gate and mesh/repo-check, those the base requires — hq's mesh/repo-check alone once the forge requires
|
||||
* mesh/check everywhere else, so no mesh/merge-gate stands on a head that nothing will set. A base whose rule could not
|
||||
* be read keeps both, as before: a status nothing requires blocks nothing, and one that is required is never left out.
|
||||
*/
|
||||
export function maintained(required: string[] | undefined): string[] {
|
||||
const both = [CHECK_CONTEXT, REPO_CHECK_CONTEXT];
|
||||
return required === undefined ? both : both.filter((c) => required.includes(c));
|
||||
}
|
||||
|
||||
/** Whether mesh/merge-gate is written on a head of this base (maintained). */
|
||||
export function writesGate(required: string[] | undefined): boolean {
|
||||
return maintained(required).includes(CHECK_CONTEXT);
|
||||
}
|
||||
|
||||
/**
|
||||
* The pending words a head's merge check is announced with (novox/hq issue 446): where the check is, and since when,
|
||||
* in the time the operator reads. They stood as "the mesh is mapping this head onto its module graph" for the whole
|
||||
@@ -257,8 +285,14 @@ export function composedGate(c: Checked): boolean {
|
||||
return c.gate?.verdict === "composed";
|
||||
}
|
||||
|
||||
/** Every status a verdict sets: the gate's, and the repository's own check's when it was said. */
|
||||
export function statusesFor(c: Checked, facts: RepoCheckFacts = {}, gateNow?: string): CommitStatus[] {
|
||||
/** Every status a verdict sets: the gate's, and the repository's own check's when it was said; of them, only those the
|
||||
* base requires (`required`, the base's rule: maintained). */
|
||||
export function statusesFor(c: Checked, facts: RepoCheckFacts = {}, gateNow?: string, required?: string[]): CommitStatus[] {
|
||||
const kept = maintained(required);
|
||||
return allStatusesFor(c, facts, gateNow).filter((s) => kept.includes(s.context));
|
||||
}
|
||||
|
||||
function allStatusesFor(c: Checked, facts: RepoCheckFacts = {}, gateNow?: string): CommitStatus[] {
|
||||
if (composedGate(c)) {
|
||||
// A member whose gate the group's composed check judges (ADR 0403 §1): only its repository's check is its own to
|
||||
// set. The gate stays pending, saying it waits for the group, unless a composed verdict already stands on it.
|
||||
@@ -292,11 +326,13 @@ function judgedFirst(c: Checked, status: CommitStatus): CommitStatus {
|
||||
* change that builds something (novox/hq ADR 0238), and why, when the gate is not a pass or the
|
||||
* repository's own check failed or could not run. A repository with no merge-check.sh, touching nothing,
|
||||
* is said by its statuses alone, not by a comment on every push. */
|
||||
export function commentFor(c: Checked, facts: RepoCheckFacts = {}): string | null {
|
||||
export function commentFor(c: Checked, facts: RepoCheckFacts = {}, required?: string[]): string | null {
|
||||
const gate = c.gate ?? { verdict: c.verdict, summary: c.summary };
|
||||
// A base that no longer requires the gate (issue 1301) is told nothing of it: its verdict is the repository's own check.
|
||||
const withGate = writesGate(required);
|
||||
const repo = c["repo-check"];
|
||||
const wrong = repoWrong(repo, facts);
|
||||
if (gate.verdict === "pass" && !wrong && !builds(c.plan)) return null;
|
||||
if ((!withGate || gate.verdict === "pass") && !wrong && !builds(c.plan)) return null;
|
||||
const lines = [`**Merge check** at \`${c.commit.slice(0, 8)}\``, ""];
|
||||
if (c.trunk) {
|
||||
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and what was read beside it.
|
||||
@@ -312,9 +348,11 @@ export function commentFor(c: Checked, facts: RepoCheckFacts = {}): string | nul
|
||||
} else if (c.siblings_unknown) {
|
||||
lines.splice(c.trunk ? 2 : 1, 0, `Siblings could not be learned: ${c.siblings_unknown}`);
|
||||
}
|
||||
lines.push(`- \`${CHECK_CONTEXT}\`: **${saidAs(gate.verdict).toUpperCase()}** — ${gate.summary}` +
|
||||
(gate.modules?.length ? ` (modules: ${gate.modules.join(", ")}` +
|
||||
(gate.dependents?.length ? `; built after them: ${gate.dependents.join(", ")}` : "") + ")" : ""));
|
||||
if (withGate) {
|
||||
lines.push(`- \`${CHECK_CONTEXT}\`: **${saidAs(gate.verdict).toUpperCase()}** — ${gate.summary}` +
|
||||
(gate.modules?.length ? ` (modules: ${gate.modules.join(", ")}` +
|
||||
(gate.dependents?.length ? `; built after them: ${gate.dependents.join(", ")}` : "") + ")" : ""));
|
||||
}
|
||||
if (repo) {
|
||||
const st = repoStatus(repo, facts);
|
||||
lines.push(`- \`${REPO_CHECK_CONTEXT}\`: ` + (repo.verdict === "warning" && facts.defined === false
|
||||
|
||||
@@ -36,3 +36,76 @@ test("importing the bundle resolves while the broker keeps refusing the binds",
|
||||
clearTimeout(timer);
|
||||
assert.match(out, /imported/, `the import did not resolve within 3 s while a bind was refused; stderr: ${err.slice(0, 600)}`);
|
||||
});
|
||||
|
||||
// **The bundle writes mesh/check from the checker's owed state, and serves no tool that writes it** (ADR 0521; R4e round 2
|
||||
// finding R2-1; finding K1). The bundle is imported whole, as the runtime imports it, while its read of the checker's owed
|
||||
// is refused (the grant not yet composed): the watch is asked again, and once granted it is handed the bucket as the bus
|
||||
// keeps it, the latest revision of each key, then an older revision delivered late. The forge stand-in sees the latest
|
||||
// written, and nothing after it.
|
||||
test("a bundle whose read of the checker's owed is granted late writes the latest mesh/check, and serves no gitea_check_status", async () => {
|
||||
const index = pathToFileURL(join(here, "..", "dist", "index.js")).href;
|
||||
const sha = "0123456789abcdef0123456789abcdef01234567";
|
||||
const script = `
|
||||
const http = await import("node:http");
|
||||
const statuses = [];
|
||||
const forge = http.createServer((req, res) => {
|
||||
let b = ""; req.on("data", (d) => (b += d)); req.on("end", () => {
|
||||
if (req.method === "POST" && /\\/statuses\\//.test(req.url)) statuses.push(req.url + " " + b);
|
||||
res.setHeader("content-type", "application/json"); res.end(req.method === "GET" ? "[]" : "{}");
|
||||
});
|
||||
});
|
||||
await new Promise((r) => forge.listen(0, "127.0.0.1", r));
|
||||
process.env.MESH_GITEA_URL = "http://127.0.0.1:" + forge.address().port;
|
||||
const bound = [];
|
||||
const { useBroker } = await import("@novox/mesh-sdk/messaging");
|
||||
useBroker(() => ({
|
||||
subscribe: async (pattern) => { bound.push(pattern); return () => {}; },
|
||||
publish: async () => {}, request: async () => { throw new Error("no responders"); },
|
||||
handle: async () => () => {}, close: async () => {},
|
||||
}));
|
||||
const key = "novox/mesh-sdk.12.${sha}.status=mesh/check";
|
||||
const owed = (state, text) => ({ v: 1, act: "status", context: "mesh/check", state, text, target: "https://view.example/c" });
|
||||
const watched = [];
|
||||
let granted = 0;
|
||||
const { useStateChannel } = await import("@novox/mesh-sdk/state");
|
||||
useStateChannel({
|
||||
ask: async () => null,
|
||||
watch: async (params, deliver) => {
|
||||
watched.push(params.state);
|
||||
if (watched.length === 1) throw new Error('gitea keeps and reads no state called "mesh-delivery-checker.owed"');
|
||||
granted++;
|
||||
// Two revisions were written while the read was refused; the bucket hands the latest, then a late older one.
|
||||
await deliver({ state: params.state, key, op: "put", value: owed("success", "pass at 4"), revision: 4, current: true });
|
||||
await deliver({ state: params.state, key, op: "put", value: owed("pending", "pending at 3"), revision: 3, current: false });
|
||||
},
|
||||
});
|
||||
await import(${JSON.stringify(index)});
|
||||
const { collectTools } = await import("@novox/mesh-sdk/tools");
|
||||
const tools = collectTools().flatMap((r) => r.tools.map((t) => t.name));
|
||||
for (let i = 0; i < 300 && !(granted && statuses.length); i++) await new Promise((r) => setTimeout(r, 20));
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
console.log(JSON.stringify({ tools, bound, watched, statuses }));
|
||||
process.exit(0);`;
|
||||
const env = { ...process.env, MESH_GITEA_TOKEN: "x" };
|
||||
for (const k of ["MESH_GITEA_STATE_DIR", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD", "MESH_SERVED_MODULE", "MESH_GITEA_URL"]) delete env[k];
|
||||
const child = spawn(process.execPath, ["--input-type=module", "-e", script], { cwd: join(here, ".."), env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let out = "";
|
||||
let err = "";
|
||||
child.stdout.on("data", (d) => (out += d));
|
||||
child.stderr.on("data", (d) => (err += d));
|
||||
const ended = new Promise<void>((r) => child.on("exit", () => r()));
|
||||
const timer = setTimeout(() => child.kill("SIGKILL"), 15000);
|
||||
await ended;
|
||||
clearTimeout(timer);
|
||||
const line = out.split("\n").find((l) => l.startsWith("{"));
|
||||
assert.ok(line, `the bundle said nothing; stdout: ${out.slice(0, 400)} stderr: ${err.slice(0, 600)}`);
|
||||
const got = JSON.parse(line!) as { tools: string[]; bound: string[]; watched: string[]; statuses: string[] };
|
||||
assert.ok(got.tools.includes("gitea_commit_status"), `the bundle's tools: ${got.tools}`);
|
||||
assert.ok(!got.tools.includes("gitea_check_status"), "no tool writes mesh/check");
|
||||
assert.ok(!got.bound.some((p) => p.includes("check.status")), `bound: ${got.bound}`);
|
||||
assert.deepEqual(got.watched, ["mesh-delivery-checker.owed", "mesh-delivery-checker.owed"], "refused, then asked again");
|
||||
assert.equal(got.statuses.length, 1, `statuses written: ${got.statuses}`);
|
||||
assert.match(got.statuses[0], new RegExp(`/repos/novox/mesh-sdk/statuses/${sha} .*"context":"mesh/check"`));
|
||||
assert.match(got.statuses[0], /"state":"success".*pass at 4/);
|
||||
assert.match(err, /mesh-delivery-checker\.owed could not be bound/, "the refusal is said");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
// **mesh/check is written from the checker's owed state, read by a watch** (ADR 0521; R4e round 2 finding R2-1; finding K1
|
||||
// before it). No tool writes mesh/check, since a tool is told no caller and node-tools invokes every tool. Nor does an event:
|
||||
// nothing in the delivery plan orders this module before the forge bridge, and a durable consumer whose filters change keeps
|
||||
// its deliver policy, so an event said before this module consumed it was lost. A watch of the checker's state hands over
|
||||
// the latest value of every key when it starts, whenever that is.
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
const sha = "0123456789abcdef0123456789abcdef01234567";
|
||||
const key = (s = sha, repo = "novox/mesh-sdk", n = 12) => `${repo}.${n}.${s}.status=mesh/check`;
|
||||
const owed = (over: Record<string, unknown> = {}) => ({ v: 1, act: "status", context: "mesh/check", state: "success",
|
||||
text: "pass 0123456789ab on aaaaaaaaaaaa", target: "https://view.example/checks/g.1", ...over });
|
||||
const put = (k: string, value: unknown, revision: number, current = false) =>
|
||||
({ state: "mesh-delivery-checker.owed", key: k, op: "put" as const, value, revision, current });
|
||||
|
||||
function forge() {
|
||||
const sent: string[] = [];
|
||||
let failing = 0;
|
||||
return {
|
||||
sent,
|
||||
fail: (n: number) => { failing = n; },
|
||||
setCommitStatus: async (o: string, r: string, s: string, st: any) => {
|
||||
if (failing > 0) { failing--; throw new Error("Gitea API /repos/x/statuses: 502 bad gateway"); }
|
||||
sent.push(`${o}/${r}@${s.slice(0, 8)} ${st.context} ${st.state} ${st.description}${st.target_url ? " " + st.target_url : ""}`);
|
||||
},
|
||||
};
|
||||
}
|
||||
const noSleep = async () => {};
|
||||
|
||||
test("the checker's owed is what this module reads, and it consumes no check.status", async () => {
|
||||
const { CHECKER_OWED } = await import("../check.ts");
|
||||
assert.equal(CHECKER_OWED, "mesh-delivery-checker.owed");
|
||||
const manifest = JSON.parse(readFileSync(new URL("../module.json", import.meta.url), "utf8"));
|
||||
assert.ok(manifest.reads.includes(CHECKER_OWED), "gitea's manifest reads the checker's owed");
|
||||
assert.ok(!manifest.consumes.some((c: string) => c.includes("check.status")), "no check.status is consumed");
|
||||
});
|
||||
|
||||
test("an owed mesh/check writes the status on its head: its state, its line cut and its link", async () => {
|
||||
const { CheckStatuses } = await import("../check.ts");
|
||||
const f = forge();
|
||||
const checks = new CheckStatuses(f, () => {}, noSleep);
|
||||
assert.equal(await checks.apply(put(key(), owed(), 7, true)), "written");
|
||||
const other = sha.replace(/0/g, "f");
|
||||
assert.equal(await checks.apply(put(key(other, "novox/my=2erepo"), owed({ state: "pending", text: "y".repeat(300), target: "javascript:x" }), 8)), "written");
|
||||
assert.equal(f.sent[0], "novox/mesh-sdk@01234567 mesh/check success pass 0123456789ab on aaaaaaaaaaaa https://view.example/checks/g.1");
|
||||
assert.ok(f.sent[1].startsWith("novox/my.repo@f1234567 mesh/check pending yyyy"), f.sent[1]);
|
||||
assert.ok(f.sent[1].length < 200, "the line is cut as every status's is");
|
||||
assert.ok(!f.sent[1].includes("javascript"), "a link that is no web page is not written");
|
||||
});
|
||||
|
||||
test("a key or value that is no owed mesh/check, and a delete, write nothing", async () => {
|
||||
const { CheckStatuses } = await import("../check.ts");
|
||||
const f = forge();
|
||||
const checks = new CheckStatuses(f, () => {}, noSleep);
|
||||
const cases = [
|
||||
put(key(sha.slice(0, 12)), owed(), 1), put(`novox/mesh-sdk.12.${sha}.status=mesh/merge-gate`, owed(), 1),
|
||||
put(`novox/mesh-sdk.12.${sha}.view`, owed(), 1), put(`nokey`, owed(), 1), put(`../x/y.12.${sha}.status=mesh/check`, owed(), 1),
|
||||
put(key(), owed({ context: "mesh/delivery" }), 1), put(key(), owed({ act: "view" }), 1), put(key(), owed({ state: "green" }), 1),
|
||||
put(key(), owed({ v: 2 }), 1), put(key(), null, 1), put(key(), "text", 1), put(key(), owed(), -1),
|
||||
{ state: "mesh-delivery-checker.owed", key: key(), op: "delete" as const, revision: 2, current: false },
|
||||
];
|
||||
for (const c of cases) assert.equal(await checks.apply(c as any), "dropped", JSON.stringify(c));
|
||||
assert.deepEqual(f.sent, []);
|
||||
});
|
||||
|
||||
test("a revision older than one written on the key is dropped; the same one is written again", async () => {
|
||||
const { CheckStatuses } = await import("../check.ts");
|
||||
const f = forge();
|
||||
const checks = new CheckStatuses(f, () => {}, noSleep);
|
||||
await checks.apply(put(key(), owed({ state: "success" }), 10));
|
||||
assert.equal(await checks.apply(put(key(), owed({ state: "pending" }), 9)), "dropped", "a stale pending never hides the verdict");
|
||||
assert.equal(await checks.apply(put(key(), owed({ state: "success" }), 10)), "written");
|
||||
assert.equal(await checks.apply(put(key(sha.replace(/1/g, "2")), owed({ state: "pending" }), 3)), "written", "another key is its own");
|
||||
assert.equal(await checks.apply(put(key(), owed({ state: "pending" }), 11)), "written");
|
||||
assert.deepEqual(f.sent.map((s) => s.split(" ").slice(0, 3).join(" ")), ["novox/mesh-sdk@01234567 mesh/check success",
|
||||
"novox/mesh-sdk@01234567 mesh/check success", "novox/mesh-sdk@02234567 mesh/check pending", "novox/mesh-sdk@01234567 mesh/check pending"]);
|
||||
});
|
||||
|
||||
test("a forge that refuses is asked again until it writes, unless a newer revision of the key came meanwhile", async () => {
|
||||
const { CheckStatuses } = await import("../check.ts");
|
||||
const f = forge();
|
||||
const said: string[] = [];
|
||||
const waits: number[] = [];
|
||||
const checks = new CheckStatuses(f, (l) => said.push(l), async (ms) => { waits.push(ms); });
|
||||
f.fail(2);
|
||||
await checks.take(put(key(), owed({ state: "success" }), 5));
|
||||
await checks.settled();
|
||||
assert.deepEqual(f.sent.map((s) => s.split(" ")[2]), ["success"], "written on the third ask");
|
||||
assert.deepEqual(waits, [2_000, 5_000]);
|
||||
assert.match(said[0], /502.*asked again/);
|
||||
// A refusal whose key moved on meanwhile is not asked again: the newer revision is written, the older never after it.
|
||||
const g = forge();
|
||||
let release: () => void = () => {};
|
||||
const gate = new Promise<void>((r) => { release = r; });
|
||||
const later = new CheckStatuses(g, () => {}, () => gate);
|
||||
g.fail(1);
|
||||
await later.take(put(key(), owed({ state: "pending" }), 6));
|
||||
await later.take(put(key(), owed({ state: "success" }), 7));
|
||||
release();
|
||||
await later.settled();
|
||||
assert.deepEqual(g.sent.map((s) => s.split(" ")[2]), ["success"]);
|
||||
});
|
||||
|
||||
test("a watch refused at first is asked again, said once, and once granted writes the latest revision of each key", async () => {
|
||||
const { CheckStatuses, watchOwed } = await import("../check.ts");
|
||||
const f = forge();
|
||||
const said: string[] = [];
|
||||
const checks = new CheckStatuses(f, () => {}, noSleep);
|
||||
// The checker's bucket as the bus keeps it: two revisions of one key written while the read was refused.
|
||||
const kept = new Map<string, { value: unknown; revision: number }>();
|
||||
kept.set(key(), { value: owed({ state: "pending" }), revision: 3 });
|
||||
kept.set(key(), { value: owed({ state: "success" }), revision: 4 });
|
||||
let asked = 0;
|
||||
const watch = async (handler: (c: any) => Promise<void>) => {
|
||||
asked++;
|
||||
if (asked < 3) throw new Error("gitea keeps and reads no state called \"mesh-delivery-checker.owed\"");
|
||||
for (const [k, e] of kept) await handler(put(k, e.value, e.revision, true));
|
||||
};
|
||||
await watchOwed(watch, checks, noSleep, (l) => said.push(l));
|
||||
await checks.settled();
|
||||
assert.equal(asked, 3);
|
||||
assert.deepEqual(f.sent.map((s) => s.split(" ")[2]), ["success"], "only the latest revision is written");
|
||||
assert.equal(said.filter((l) => /could not be bound/.test(l)).length, 1, `said once: ${said.join(" | ")}`);
|
||||
assert.ok(said.some((l) => /bound after 3 attempts/.test(l)), said.join(" | "));
|
||||
});
|
||||
|
||||
test("no tool writes mesh/check: gitea_check_status is gone and gitea_commit_status refuses it", async () => {
|
||||
const delivery = await import("../delivery.ts");
|
||||
assert.equal((delivery as Record<string, unknown>).checkStatusTool, undefined, "delivery.ts serves no gitea_check_status");
|
||||
assert.throws(() => delivery.deliveryStatus("mesh/check", "success", "x"), /mesh-delivery-checker\.owed/);
|
||||
const index = readFileSync(new URL("../index.ts", import.meta.url), "utf8");
|
||||
assert.ok(!/gitea_check_status|checkStatusTool/.test(index), "index.ts serves no gitea_check_status");
|
||||
});
|
||||
@@ -171,3 +171,53 @@ test("the root tree is git's hash of the root's own entries, never the listing's
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("named paths: each asked path's git object id at the commit, a file's or a directory's, by directory (1758 S1, ADR 0499 §3)", async () => {
|
||||
const { client, server } = await serve([{ sha: COMMIT, url: "u", tree: LISTING, truncated: false, page: 1, total_count: LISTING.length }]);
|
||||
try {
|
||||
const got = await client.commitTrees("novox", "hq", COMMIT, { "modules/a": ["main.go", "module.json"], "modules/b": ["c"], "": ["README.md", "modules/d"] });
|
||||
assert.deepEqual(got.paths, {
|
||||
"modules/a": { "main.go": h("6"), "module.json": h("5") },
|
||||
"modules/b": { c: h("8") },
|
||||
"": { "README.md": h("1"), "modules/d": h("a") },
|
||||
});
|
||||
assert.deepEqual(got.trees, { "": ROOT, "modules/a": h("4"), "modules/b/c": h("8") });
|
||||
assert.equal((await client.commitTrees("novox", "hq", COMMIT)).paths, undefined, "no paths asked, none answered");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("a named path the listing lacks fails naming the directory and the path, and answers nothing", async () => {
|
||||
const { client, server } = await serve([{ sha: COMMIT, url: "u", tree: LISTING, truncated: false, page: 1, total_count: LISTING.length }]);
|
||||
try {
|
||||
await assert.rejects(client.commitTrees("novox", "hq", COMMIT, { "modules/a": ["main.go", "gone.go"] }), (err: Error) => {
|
||||
assert.match(err.message, /modules\/a/);
|
||||
assert.match(err.message, /gone\.go/);
|
||||
return true;
|
||||
});
|
||||
for (const bad of [[""], ["../x"], ["/abs"], ["a//b"]]) {
|
||||
await assert.rejects(client.commitTrees("novox", "hq", COMMIT, { "modules/a": bad }), /./, JSON.stringify(bad));
|
||||
}
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("gitea_commit_trees takes paths as {directory: [paths]} and refuses another shape before the forge is asked", async () => {
|
||||
const { getGiteaTools } = await import("../tools/index.ts");
|
||||
const { client, server, asked } = await serve([{ sha: COMMIT, url: "u", tree: LISTING, truncated: false, page: 1, total_count: LISTING.length }]);
|
||||
try {
|
||||
const tool = getGiteaTools(client).find((t) => t.name === "gitea_commit_trees")!;
|
||||
assert.ok("paths" in tool.input, "the tool names the argument");
|
||||
const got: any = await tool.run({ owner: "novox", repo: "hq", sha: COMMIT, paths: { "modules/a": ["main.go"] } } as any);
|
||||
assert.deepEqual(got.paths, { "modules/a": { "main.go": h("6") } });
|
||||
const before = asked.length;
|
||||
for (const paths of ["modules/a", ["main.go"], { "modules/a": "main.go" }, { "modules/a": [1] }]) {
|
||||
await assert.rejects(tool.run({ owner: "novox", repo: "hq", sha: COMMIT, paths } as any), /paths/, JSON.stringify(paths));
|
||||
}
|
||||
assert.equal(asked.length, before, "the forge was asked for a refused call");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -61,3 +61,21 @@ test("one view per pull request, found by its marker; only the mesh's statuses",
|
||||
assert.equal(deliveryStatus("mesh/repo-check", "pending", "checking again: main moved on").context, "mesh/repo-check");
|
||||
assert.equal(deliveryStatus("mesh/delivery", "success", "x", "javascript:alert(1)").target_url, undefined);
|
||||
});
|
||||
|
||||
// Issue 1301-S4 (ADR 0447 §1; R4e findings K1 and R2-1): main requires mesh/check, so no hand may set it. The status tool
|
||||
// every caller may use refuses it in every state; no tool writes it at all: it is written from the checker's owed state
|
||||
// (check.ts, test/check-owed.test.ts; ADR 0521).
|
||||
test("mesh/check is refused by the status tool in every state", async () => {
|
||||
const { commitStatusTool, deliveryStatus, CHECK_STATUS } = await import("../delivery.ts");
|
||||
assert.equal(CHECK_STATUS, "mesh/check");
|
||||
const sha = "0123456789abcdef0123456789abcdef01234567";
|
||||
const sent: string[] = [];
|
||||
const setCommitStatus = async (o: string, r: string, s: string, st: any) => void sent.push(`${o}/${r}@${s} ${st.context}`);
|
||||
const gates = { rechecked: async () => { throw new Error("not the gate writer's"); } };
|
||||
for (const state of ["pending", "success", "failure", "error", "warning"]) {
|
||||
assert.throws(() => deliveryStatus("mesh/check", state, "x"), /mesh\/check is the checker's status/);
|
||||
await assert.rejects(commitStatusTool({ setCommitStatus, gates } as any).run({ owner: "novox", repo: "mesh-sdk", sha,
|
||||
context: "mesh/check", state, description: "by hand" }), /mesh-delivery-checker\.owed/);
|
||||
}
|
||||
assert.deepEqual(sent, [], "nothing was sent for a hand-set mesh/check");
|
||||
});
|
||||
|
||||
@@ -322,3 +322,13 @@ test("a repository as the mesh clones it is told from another", async () => {
|
||||
assert.equal(clonedAs("http://127.0.0.1:20000/novox/mesh-hq.git", "novox", "hq"), false);
|
||||
assert.equal(clonedAs("http://127.0.0.1:20000/novox/mesh-controller.git", "novox", "hq"), false);
|
||||
});
|
||||
|
||||
// Issue 1301: a build machine's start on a head that holds no mesh/merge-gate (its base requires none, so none was written)
|
||||
// says nothing: no "building on" status appears on a main that does not require the gate.
|
||||
test("a start on a head with no gate on the forge writes nothing", async () => {
|
||||
const { GateWriter } = await import("../gate.ts");
|
||||
const forge = aForge();
|
||||
const gates = new GateWriter(forge, announcedAt);
|
||||
assert.equal(await gates.started("novox", "hq", SHA, startedOnShanks), false);
|
||||
assert.equal(forge.kept["mesh/merge-gate"], undefined);
|
||||
});
|
||||
|
||||
@@ -568,6 +568,7 @@ test("no tool's caller text becomes another path or another user: `..` and `?sud
|
||||
["gitea_get_file", { ...repo, path: "README.md?sudo=jschoubben" }],
|
||||
["gitea_commit_status", { ...repo, sha: "abc1234?sudo=jschoubben", context: "mesh/delivery", state: "success" }],
|
||||
["gitea_commit_status", { ...repo, sha: "../../../admin/users", context: "mesh/delivery", state: "success" }],
|
||||
["gitea_commit_status", { ...repo, sha: "0123456789abcdef0123456789abcdef01234567", context: "mesh/check", state: "success" }],
|
||||
["gitea_commit_statuses", { ...repo, sha: ".." }],
|
||||
["gitea_commit_statuses", { ...repo, sha: "main?sudo=jschoubben" }],
|
||||
["gitea_branch_protection_set", { ...repo, branch: "..", status_checks: "mesh/merge-gate" }],
|
||||
@@ -704,3 +705,53 @@ test("a group of two merges in one pass: the sibling's merge of the composed hea
|
||||
assert.equal(done.merged, true);
|
||||
assert.equal(state.emitted.length, 0, "no check was asked");
|
||||
});
|
||||
|
||||
// **Issue 1301 (ADR 0447 §1, §4; ADR 0503 §3, §5): the merge tool judges on the status the base requires.** A main that
|
||||
// requires mesh/check is judged by the checker's line ("pass <head12> on <trunk12> …"): it merges on a pass that names the
|
||||
// trunk's head, and refuses one that names an older trunk, naming both, writing nothing and asking nothing — a re-check on
|
||||
// a trunk's move is the checker's. No mesh/merge-gate is read or written there.
|
||||
test("gitea_merge_pull_request judges a main requiring mesh/check on mesh/check alone, and asks nothing of it", async () => {
|
||||
const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED });
|
||||
state.rules = [{ ...protectedMain(), status_check_contexts: ["mesh/check"] }];
|
||||
const line = (trunk: string, s = "success", words = "pass") =>
|
||||
[{ context: "mesh/check", state: s, description: `${words} ${REVIEWED.slice(0, 12)} on ${trunk.slice(0, 12)} | unit 12 run` }];
|
||||
const before = state.emitted.length;
|
||||
|
||||
state.statuses = line(MOVED);
|
||||
await assert.rejects(merge(), (err: Error) => {
|
||||
assert.match(err.message, /mesh\/check/);
|
||||
assert.match(err.message, new RegExp(`${MOVED.slice(0, 12)}.*${TRUNK.slice(0, 12)}`));
|
||||
assert.match(err.message, /checker checks it again/i);
|
||||
assert.match(err.message, /Nothing was merged/);
|
||||
return true;
|
||||
});
|
||||
state.statuses = line(TRUNK, "pending", "checking");
|
||||
await assert.rejects(merge(), /mesh\/check is pending/);
|
||||
state.statuses = judgedAt(TRUNK); // the old two, current: not what this main requires
|
||||
await assert.rejects(merge(), /mesh\/check/);
|
||||
assert.deepEqual(writes(), [], "nothing is written: mesh/check is the checker's");
|
||||
assert.equal(state.emitted.length, before, "no check is asked");
|
||||
assert.equal(state.merged, false);
|
||||
|
||||
state.statuses = line(TRUNK);
|
||||
assert.equal(((await merge()) as Record<string, unknown>).merged, true);
|
||||
});
|
||||
|
||||
// hq's main requires mesh/repo-check alone (ADR 0503 §5): judged on its words, and an out-of-date one is asked again on
|
||||
// mesh/repo-check alone; no mesh/merge-gate is written on it.
|
||||
test("gitea_merge_pull_request judges a main requiring mesh/repo-check alone on it, and writes no mesh/merge-gate", async () => {
|
||||
const { keepSiblings, siblingsFile } = await import("../dist/siblings.js");
|
||||
keepSiblings(siblingsFile(stateDir), {}); // no sibling record left by the tests before
|
||||
const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED });
|
||||
state.rules = [{ ...protectedMain(), status_check_contexts: ["mesh/repo-check"] }];
|
||||
state.statuses = [judgedAt(MOVED)[1]];
|
||||
const before = state.emitted.length;
|
||||
await assert.rejects(merge(), /judged against trunk/);
|
||||
assert.deepEqual(state.sent.filter((s) => s.path.includes("/statuses/")).map((s) => `${s.body.context} ${s.body.state}`),
|
||||
["mesh/repo-check pending"]);
|
||||
assert.equal(state.emitted.length, before + 1, "its check is asked once");
|
||||
await assert.rejects(merge(), /already asked/);
|
||||
state.sent = [];
|
||||
state.statuses = [judgedAt(TRUNK)[1]];
|
||||
assert.equal(((await merge()) as Record<string, unknown>).merged, true);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
// 1790: the states the module announces are the states its manifest declares, no more, no fewer; none is readable
|
||||
// because no other module's manifest reads one of them.
|
||||
test("the states the module announces are the states its manifest declares", async () => {
|
||||
const { KEPT } = await import("../kept.ts");
|
||||
const manifest = JSON.parse(readFileSync(new URL("../module.json", import.meta.url), "utf8"));
|
||||
const declared = (manifest.state as (string | { name: string })[]).map((s) => (typeof s === "string" ? s : s.name)).sort();
|
||||
assert.deepEqual(KEPT.map((k) => k.name).sort(), declared);
|
||||
assert.ok(KEPT.every((k) => !k.readable), "no other module's manifest reads a state of gitea");
|
||||
});
|
||||
|
||||
test("the announcement has the layout mesh-sdk's go/announce reads: version 1, the module, the node, the states sorted by name", async () => {
|
||||
const { statesAnnouncement } = await import("../kept.ts");
|
||||
assert.deepEqual(statesAnnouncement("gitea", "ace"), { v: 1, module: "gitea", node: "ace", states: [{ name: "announced" }, { name: "bound" }] });
|
||||
assert.deepEqual(statesAnnouncement("gitea", undefined), { v: 1, module: "gitea", states: [{ name: "announced" }, { name: "bound" }] });
|
||||
});
|
||||
|
||||
test("index.ts says the announcement as the event `states` before the tools are served", () => {
|
||||
const source = readFileSync(new URL("../index.ts", import.meta.url), "utf8");
|
||||
assert.match(source, /await emit\("states", statesAnnouncement\(/);
|
||||
assert.ok(source.indexOf('await emit("states"') < source.indexOf('registerModuleTools("gitea"'), "announced before the tools are served");
|
||||
});
|
||||
@@ -192,3 +192,36 @@ test("a member's own verdict with a composed gate layer leaves the gate to the c
|
||||
// A verdict with an ordinary gate is as before.
|
||||
assert.equal(composedGate({ ...c, gate: { verdict: "pass", summary: "x" } }), false);
|
||||
});
|
||||
|
||||
// Issue 1301 (ADR 0503 §3, §5): gitea writes the controller's statuses only where the base still requires them. hq's main
|
||||
// requires mesh/repo-check alone: its verdict sets mesh/repo-check and no mesh/merge-gate, and its comment names no gate.
|
||||
test("a base's required statuses decide which of the controller's statuses are written", async () => {
|
||||
const { maintained, writesGate, statusesFor, commentFor, CHECK_STATUS } = await import("../pulls.ts");
|
||||
assert.equal(CHECK_STATUS, "mesh/check");
|
||||
assert.deepEqual(maintained(undefined), ["mesh/merge-gate", "mesh/repo-check"], "a rule not read: as before");
|
||||
assert.deepEqual(maintained(["mesh/repo-check"]), ["mesh/repo-check"]);
|
||||
assert.deepEqual(maintained(["mesh/check", "other/x"]), []);
|
||||
assert.deepEqual(maintained([]), []);
|
||||
assert.equal(writesGate(undefined), true);
|
||||
assert.equal(writesGate(["mesh/repo-check"]), false);
|
||||
assert.equal(writesGate(["mesh/check"]), false);
|
||||
assert.equal(writesGate(["mesh/merge-gate", "mesh/repo-check"]), true);
|
||||
const c = { owner: "novox", repo: "hq", number: 5, commit: "a".repeat(40), id: "x", verdict: "warning", summary: "no script",
|
||||
trunk: "b".repeat(12), "repo-check": { verdict: "pass", summary: "ok" } } as any;
|
||||
const hq = statusesFor(c, {}, undefined, ["mesh/repo-check"]);
|
||||
assert.deepEqual(hq.map((s: any) => s.context), ["mesh/repo-check"]);
|
||||
assert.match(hq[0].description, /merged into trunk bbbbbbbbbbbb/);
|
||||
assert.deepEqual(statusesFor(c).map((s: any) => s.context), ["mesh/merge-gate", "mesh/repo-check"], "unread: as before");
|
||||
assert.deepEqual(statusesFor(c, {}, undefined, ["mesh/check"]), []);
|
||||
const comment = commentFor(c, {}, ["mesh/repo-check"]);
|
||||
assert.ok(comment === null || !comment.includes("mesh/merge-gate"), comment ?? "");
|
||||
});
|
||||
|
||||
test("a rule the forge gives with no contexts (null) requires nothing, and an unread rule is unknown", async () => {
|
||||
const { requiredOf } = await import("../pulls.ts");
|
||||
// The forge's BranchProtection says status_check_contexts may be null (protection.ts); requiredOf takes it as it comes.
|
||||
assert.deepEqual(requiredOf({ enable_status_check: true, status_check_contexts: null }), []);
|
||||
assert.deepEqual(requiredOf({ enable_status_check: true, status_check_contexts: ["mesh/check", ""] }), ["mesh/check"]);
|
||||
assert.deepEqual(requiredOf(null), []);
|
||||
assert.equal(requiredOf(undefined), undefined);
|
||||
});
|
||||
|
||||
@@ -199,3 +199,21 @@ test("a release branch's move puts every open pull request into it out of date,
|
||||
assert.equal(verdictAtTrunk({ "mesh/merge-gate": { state: "success",
|
||||
description: "head x merged into trunk eeeeeeeeeeee — pass" } }, MOVED).current, false);
|
||||
});
|
||||
|
||||
// Issue 1301: a trunk's move puts out of date only the controller's statuses the base still requires; on hq that is
|
||||
// mesh/repo-check alone, judged by its own words; on a main requiring mesh/check, none (the checker's, ADR 0447 §4).
|
||||
test("a trunk's move writes only the statuses the base still requires", async () => {
|
||||
const { outOfDateStatuses, trunkNamed } = await import("../trunk.ts");
|
||||
const old = "1".repeat(40), now = "2".repeat(40);
|
||||
const repo = { state: "success", description: `head ${"a".repeat(12)} merged into trunk ${old.slice(0, 12)} — pass: y` };
|
||||
const gate = { state: "success", description: `head ${"a".repeat(12)} merged into trunk ${old.slice(0, 12)} — pass: x` };
|
||||
assert.deepEqual(outOfDateStatuses({ "mesh/repo-check": repo }, now, ["mesh/repo-check"]).map((s: any) => s.context), ["mesh/repo-check"]);
|
||||
assert.deepEqual(outOfDateStatuses({ "mesh/merge-gate": gate, "mesh/repo-check": repo }, now, ["mesh/repo-check"]).map((s: any) => s.context),
|
||||
["mesh/repo-check"]);
|
||||
assert.deepEqual(outOfDateStatuses({ "mesh/merge-gate": gate, "mesh/repo-check": repo }, now, ["mesh/check"]), []);
|
||||
assert.deepEqual(outOfDateStatuses({ "mesh/merge-gate": gate, "mesh/repo-check": repo }, now).map((s: any) => s.context),
|
||||
["mesh/merge-gate", "mesh/repo-check"], "a rule not read: as before");
|
||||
// The checker's line names its trunk as "<head12> on <trunk12>".
|
||||
assert.equal(trunkNamed(`pass ${"a".repeat(12)} on ${old.slice(0, 12)} | unit 3 run`), old.slice(0, 12));
|
||||
assert.equal(trunkNamed("building on shanks since 10:00"), undefined);
|
||||
});
|
||||
|
||||
@@ -350,3 +350,14 @@ test("a pull request's edit event carries its description, and no other pull req
|
||||
assert.equal(eventOf("pull_request", { action, pull_request: pull, repository }).body.body, undefined, action);
|
||||
}
|
||||
});
|
||||
|
||||
// Issue 1302: a status event carries the status's words, so the delivery gate reads mesh/check's line (the head and the trunk
|
||||
// it judged) from the event itself.
|
||||
test("a status event carries its context, state and words", async () => {
|
||||
const { eventOf } = await import("../webhook.ts");
|
||||
const e = eventOf("status", { sha: "a".repeat(40), context: "mesh/check", state: "success",
|
||||
description: "pass aaaaaaaaaaaa on bbbbbbbbbbbb | unit 3 run, 0 kept", repository: { name: "x", owner: { login: "novox" } } });
|
||||
assert.equal(e.name, "forge.status");
|
||||
assert.equal((e.body as any).context, "mesh/check");
|
||||
assert.equal((e.body as any).description, "pass aaaaaaaaaaaa on bbbbbbbbbbbb | unit 3 run, 0 kept");
|
||||
});
|
||||
|
||||
@@ -60,6 +60,22 @@ function namesChecked(tools: ToolDefinition[]): ToolDefinition[] {
|
||||
}));
|
||||
}
|
||||
|
||||
/** The `paths` argument of gitea_commit_trees: {directory: [path relative to it]}, each path plain (no empty, `.` or `..`
|
||||
* segment, not absolute); anything else is refused before the forge is asked. Absent is absent. */
|
||||
function namedPaths(raw: unknown): Record<string, string[]> | undefined {
|
||||
if (raw === undefined || raw === null) return undefined;
|
||||
if (typeof raw !== "object" || Array.isArray(raw)) throw new Error("paths is {directory: [path, ...]}");
|
||||
const out: Record<string, string[]> = {};
|
||||
for (const [dir, list] of Object.entries(raw as Record<string, unknown>)) {
|
||||
if (!Array.isArray(list) || list.some((p) => typeof p !== "string")) throw new Error(`paths of ${dir || "the root"} is a list of paths`);
|
||||
for (const p of list as string[]) {
|
||||
if (p === "" || p.split("/").some((seg) => seg === "" || seg === "." || seg === "..")) throw new Error(`paths: ${JSON.stringify(p)} of ${dir || "the root"} is not a plain relative path`);
|
||||
}
|
||||
out[dir] = list as string[];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function getGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] {
|
||||
return namesChecked(allGiteaTools(gitea, admin));
|
||||
}
|
||||
@@ -395,7 +411,7 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
// ---- Branch protection (novox/hq ADR 0237) ----
|
||||
{
|
||||
name: "gitea_branch_protection_get",
|
||||
description: "A repository's branch protection: the rule for one branch (null when it has none) or every rule — whether direct pushes are refused, which commit statuses a pull request must have succeeded to merge (e.g. mesh/merge-gate), approvals, and whether an administrator may merge past them.",
|
||||
description: "A repository's branch protection: the rule for one branch (null when it has none) or every rule — whether direct pushes are refused, which commit statuses a pull request must have succeeded to merge (e.g. mesh/check), approvals, and whether an administrator may merge past them.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
@@ -413,12 +429,12 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
},
|
||||
{
|
||||
name: "gitea_branch_protection_set",
|
||||
description: "Only ever stronger (novox/hq ADR 0266): a change that drops a required status, lets a person push directly or lets an administrator merge past a status is refused, and on main so is a change that drops the merge whitelist (only mesh-tools merges there) or widens it (novox/hq ADR 0398 §2, issue 948) — that is the operator's, at the forge. Make a branch require commit statuses before a pull request merges into it — the mesh's merge check sets `mesh/merge-gate` (the module graph's gate) and `mesh/repo-check` (the repository's own tests). Edits the branch's rule (its required statuses replaced by these, everything else kept unless given) or creates one, which refuses direct pushes unless push=true. Answers the rule before and after.",
|
||||
description: "Only ever stronger (novox/hq ADR 0266): a change that drops a required status, lets a person push directly or lets an administrator merge past a status is refused, and on main so is a change that drops the merge whitelist (only mesh-tools merges there) or widens it (novox/hq ADR 0398 §2, issue 948) — that is the operator's, at the forge. Make a branch require commit statuses before a pull request merges into it — the mesh's checker sets `mesh/check` (ADR 0447), and the repositories program requires it on every main. Edits the branch's rule (its required statuses replaced by these, everything else kept unless given) or creates one, which refuses direct pushes unless push=true. Answers the rule before and after.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
branch: { type: "string", description: "the branch to protect, e.g. main" },
|
||||
status_checks: { type: "string", description: "comma-separated statuses required to merge, e.g. mesh/merge-gate,mesh/repo-check; empty requires none" },
|
||||
status_checks: { type: "string", description: "comma-separated statuses required to merge, e.g. mesh/check; empty requires none" },
|
||||
push: { type: "boolean", description: "whether a person may push to the branch directly (a new rule refuses it when not given)" },
|
||||
block_admin_override: { type: "boolean", description: "stop an administrator merging past a status that has not succeeded (left as it is when not given)" },
|
||||
},
|
||||
@@ -478,7 +494,7 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
},
|
||||
{
|
||||
name: "gitea_commit_statuses",
|
||||
description: "Read a commit's statuses — or a pull request's head's — as the forge keeps them: the newest of each context (mesh/merge-gate, mesh/repo-check, mesh/delivery, …) with its state, description, link, when it was set and by whom; and whether the branch it merges into would let it merge, every status that branch's protection requires being a success. Reads only.",
|
||||
description: "Read a commit's statuses — or a pull request's head's — as the forge keeps them: the newest of each context (mesh/check, mesh/delivery, …) with its state, description, link, when it was set and by whom; and whether the branch it merges into would let it merge, every status that branch's protection requires being a success. Reads only.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
@@ -525,18 +541,19 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
},
|
||||
{
|
||||
name: "gitea_commit_trees",
|
||||
description: "A commit's trees from the forge's one recursive listing: the root tree, and the tree of every directory that holds a module.json (the root as an empty name), each by its git hash. What the forge bridge puts in a merge's message (novox/hq ADR 0437 §5). The commit is a full 40-hex sha. A listing the forge cut is refused, never answered in part. Reads only.",
|
||||
description: "A commit's trees from the forge's one recursive listing: the root tree, and the tree of every directory that holds a module.json (the root as an empty name), each by its git hash. What the forge bridge puts in a merge's message (novox/hq ADR 0437 §5). With paths, {directory: [path relative to it]}, the answer also carries paths, {directory: {path: git object id}} of each named file or directory at the commit (novox/hq ADR 0499 §3); a path the commit lacks fails naming the directory and the path. The commit is a full 40-hex sha. A listing the forge cut is refused, never answered in part. Reads only.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
sha: { type: "string", description: "the commit, its full 40-hex sha" },
|
||||
paths: { type: "object", description: "optional, {directory: [path relative to it]}: the git object id of each named file or directory is answered under paths; the root directory is the empty name" },
|
||||
},
|
||||
run: async (args) => {
|
||||
const owner = String(args.owner ?? "").trim(), repo = String(args.repo ?? "").trim();
|
||||
const sha = String(args.sha ?? "").trim();
|
||||
if (!owner || !repo) throw new Error("name the repository: owner and repo");
|
||||
if (!/^[0-9a-f]{40}$/i.test(sha)) throw new Error("sha is a commit's full 40-hex sha");
|
||||
return gitea.commitTrees(owner, repo, sha);
|
||||
return gitea.commitTrees(owner, repo, sha, namedPaths(args.paths));
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
+24
-12
@@ -12,12 +12,17 @@
|
||||
// the moment it is asked, merges nothing when they differ, asks the head's check once — the same announcement a
|
||||
// new head makes — and says how long the last check of that repository took. It never merges by itself after.
|
||||
//
|
||||
// Since issue 1301 (ADR 0447 §4) a main that requires mesh/check is the checker's: its line names the trunk commit it judged
|
||||
// ("pass <head12> on <trunk12>"), the checker sets it out of date when the trunk moves, and the merge tool refuses a verdict
|
||||
// of another trunk and asks nothing. The two statuses above are kept only where a base still requires them (hq's
|
||||
// mesh/repo-check, pulls.ts maintained).
|
||||
//
|
||||
// Pure functions and two small files of the module's own state, so this is tested without a forge.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import type { CommitStatus, GiteaClient, GiteaPull, HeadCheck } from "./client.js";
|
||||
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, WAITS_FOR_COMPOSED } from "./pulls.js";
|
||||
import { CHECK_CONTEXT, WAITS_FOR_COMPOSED, maintained, requiredOf } from "./pulls.js";
|
||||
|
||||
/** A commit as a verdict names it: its first twelve characters. */
|
||||
export function twelve(sha: string | undefined): string {
|
||||
@@ -40,7 +45,9 @@ export function judgedWords(head: string, trunk: string): string {
|
||||
* verdict put out of date, the one it had judged ("judged against trunk …"). Undefined when it names none, as a
|
||||
* verdict of a build seat from before ADR 0323 does. */
|
||||
export function trunkNamed(description: string | undefined): string | undefined {
|
||||
const m = /(?:merged into|judged against) trunk ([0-9a-f]{7,64})\b/i.exec(description ?? "");
|
||||
const m = /(?:merged into|judged against) trunk ([0-9a-f]{7,64})\b/i.exec(description ?? "")
|
||||
// The checker's mesh/check line (ADR 0447 §3): "<head12> on <trunk12>".
|
||||
?? /\b[0-9a-f]{12} on ([0-9a-f]{12})\b/i.exec(description ?? "");
|
||||
return m ? m[1].toLowerCase() : undefined;
|
||||
}
|
||||
|
||||
@@ -70,15 +77,15 @@ export function waitsForComposed(gate: HeadCheck | undefined): boolean {
|
||||
* gate, and the repository's own check when the head has one — a pending repo-check a verdict will not set would
|
||||
* stand for ever (novox/hq issue 305's rule).
|
||||
*/
|
||||
export function outOfDateStatuses(checks: Record<string, HeadCheck>, trunkHead: string): CommitStatus[] {
|
||||
const gate = checks[CHECK_CONTEXT];
|
||||
if (!gate || gate.state === "pending") return [];
|
||||
const judged = trunkNamed(gate.description);
|
||||
export function outOfDateStatuses(checks: Record<string, HeadCheck>, trunkHead: string, required?: string[]): CommitStatus[] {
|
||||
// Only the statuses the base still requires (issue 1301): the first of them the head holds judges.
|
||||
const kept = maintained(required).filter((c) => checks[c]);
|
||||
const judge = kept.length ? checks[kept[0]] : undefined;
|
||||
if (!judge || judge.state === "pending") return [];
|
||||
const judged = trunkNamed(judge.description);
|
||||
if (judged && sameCommit(judged, trunkHead)) return [];
|
||||
const description = outOfDateWords(judged, trunkHead);
|
||||
const out: CommitStatus[] = [{ state: "pending", context: CHECK_CONTEXT, description }];
|
||||
if (checks[REPO_CHECK_CONTEXT]) out.push({ state: "pending", context: REPO_CHECK_CONTEXT, description });
|
||||
return out;
|
||||
return kept.map((context) => ({ state: "pending" as const, context, description }));
|
||||
}
|
||||
|
||||
/** Where the merge tool stands with a head's verdict and the trunk's head (ADR 0323 §5). */
|
||||
@@ -91,8 +98,8 @@ export interface VerdictAtTrunk {
|
||||
judged?: string;
|
||||
}
|
||||
|
||||
export function verdictAtTrunk(checks: Record<string, HeadCheck>, trunkHead: string): VerdictAtTrunk {
|
||||
const gate = checks[CHECK_CONTEXT];
|
||||
export function verdictAtTrunk(checks: Record<string, HeadCheck>, trunkHead: string, context = CHECK_CONTEXT): VerdictAtTrunk {
|
||||
const gate = checks[context];
|
||||
const judged = trunkNamed(gate?.description);
|
||||
if (checkOnItsWay(gate)) return { current: false, onItsWay: true, judged };
|
||||
return { current: !!gate && gate.state !== "pending" && !!judged && sameCommit(judged, trunkHead), onItsWay: false, judged };
|
||||
@@ -119,6 +126,9 @@ export function lateVerdictComment(c: { commit: string; trunk?: string; verdict:
|
||||
export interface TrunkForge {
|
||||
listOpenPulls(owner: string, repo: string): Promise<GiteaPull[]>;
|
||||
branchHead(owner: string, repo: string, branch: string): Promise<string>;
|
||||
/** The rule the forge applies to a branch: which of the controller's statuses are kept there (issue 1301). Not given,
|
||||
* or not read: both, as before. */
|
||||
appliedProtection?(owner: string, repo: string, branch: string): Promise<{ enable_status_check?: boolean; status_check_contexts?: string[] | null } | null>;
|
||||
}
|
||||
|
||||
/** What it writes with: the merge check's one writer (gate.ts), which decides in the head's own turn. */
|
||||
@@ -147,11 +157,13 @@ export async function weighTrunks(forge: TrunkForge, gates: OutOfDateWriter, tru
|
||||
if (before && sameCommit(before, head)) continue;
|
||||
if (before) {
|
||||
let put = 0;
|
||||
const required = forge.appliedProtection
|
||||
? requiredOf(await forge.appliedProtection(owner, name, base).catch(() => undefined)) : undefined;
|
||||
for (const pull of open.filter((p) => p.base === base)) {
|
||||
// The trunk is read again in the head's own turn: a verdict judged against a head newer than the pass's first
|
||||
// read — the trunk moved again meanwhile, and a check answered — is not put out of date.
|
||||
const written = await gates.outOfDate(owner, name, String(pull.head_sha),
|
||||
async (checks) => outOfDateStatuses(checks, await forge.branchHead(owner, name, base)), pull.html_url || undefined);
|
||||
async (checks) => outOfDateStatuses(checks, await forge.branchHead(owner, name, base), required), pull.html_url || undefined);
|
||||
if (written.length) put++;
|
||||
}
|
||||
console.log(`[gitea] ${owner}/${name}'s ${base} moved from ${twelve(before)} to ${twelve(head)}: ${put} open pull ` +
|
||||
|
||||
@@ -127,7 +127,9 @@ function forgeEventOf(said: string, payload: any): ForgeEvent {
|
||||
return { name: kind, body: { ...base, ref: str(payload.ref), ref_type: str(payload.ref_type), sha: str(payload.sha) } };
|
||||
}
|
||||
if (kind === "status") {
|
||||
return { name: "status", body: { ...base, sha: str(payload.sha), context: str(payload.context), state: str(payload.state) } };
|
||||
// Its words too (issue 1302): mesh/check's line names the head and the trunk it judged, which the delivery gate reads.
|
||||
return { name: "status", body: { ...base, sha: str(payload.sha), context: str(payload.context), state: str(payload.state),
|
||||
description: str(payload.description) } };
|
||||
}
|
||||
const name = family === "other" ? `other.${kind.replace(/[^a-z0-9_-]/gi, "_") || "unknown"}` :
|
||||
["fork"].includes(family) ? family : `${family}.${action ?? "changed"}`;
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -204,41 +203,37 @@ func TestTheLoginManagersEntryRunsTheSessionsStart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Every module of the catalogue that contributes window-manager lines (novox/hq ADR 0212) is placed in
|
||||
// the main file as the controller places them — in module order, each module's under a line naming it
|
||||
// — and loaded as i3 would load them on a machine with all of them assigned: no two bind one key, and
|
||||
// every line parses.
|
||||
// The window-manager lines the catalogue's modules contribute to the main file (novox/hq ADR 0212) are kept in
|
||||
// testdata/window-manager-lines.json, one entry per contributing module in module order, as the controller places
|
||||
// them. The test reads that file, never the other modules' manifests: a module may not read a manifest (ADR 0495
|
||||
// §1), and a test that reads its siblings' fails wherever they are not beside it (issue 1814). The main file and
|
||||
// those lines are placed together — each module's under a line naming it — and loaded as i3 would load them on a
|
||||
// machine with all of them assigned: no two bind one key, and every line parses.
|
||||
func TestTheMainFileAndEveryModulesContributionLoadTogether(t *testing.T) {
|
||||
i3, err := exec.LookPath("i3")
|
||||
if err != nil {
|
||||
t.Skip("no i3 here to check with")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
manifests, _ := filepath.Glob(filepath.Join("..", "..", "..", "*", "module.json"))
|
||||
sort.Strings(manifests)
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "window-manager-lines.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var contributions []struct {
|
||||
Module string `json:"module"`
|
||||
Content []string `json:"content"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &contributions); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var placed strings.Builder
|
||||
var found []string
|
||||
for _, p := range manifests {
|
||||
raw, _ := os.ReadFile(p)
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Contributions []struct {
|
||||
Seat, Kind, Content string
|
||||
} `json:"contributions"`
|
||||
}
|
||||
json.Unmarshal(raw, &m)
|
||||
named := false
|
||||
for _, c := range m.Contributions {
|
||||
if c.Seat != "node-display-session" || c.Kind != "config" {
|
||||
continue
|
||||
}
|
||||
if !named {
|
||||
placed.WriteString("# " + m.Module + "\n")
|
||||
named = true
|
||||
found = append(found, m.Module)
|
||||
}
|
||||
placed.WriteString(c.Content)
|
||||
if !strings.HasSuffix(c.Content, "\n") {
|
||||
for _, c := range contributions {
|
||||
placed.WriteString("# " + c.Module + "\n")
|
||||
found = append(found, c.Module)
|
||||
for _, content := range c.Content {
|
||||
placed.WriteString(content)
|
||||
if !strings.HasSuffix(content, "\n") {
|
||||
placed.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
[
|
||||
{
|
||||
"module": "asus-zephyrus-g14",
|
||||
"content": [
|
||||
"# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the\n# mesh: replaced at every push. Once the controller places contributions to node-display-session\n# (ADR 0210), these become the module's contribution instead of a file in i3's directory.\n#\n# The keys the firmware turns into ordinary key presses. The vendor keys that reach no X client are\n# triggerhappy's (/etc/triggerhappy/triggers.d/asus-g14.conf): M4 and Fn+F4/F5 for media, Fn+F7/F8 for\n# the panel, Fn+F10 for the touchpad. The keyboard backlight (Fn+F2/F3) is the firmware's and asusd's.\n\n# Fn+F6, the screenshot key: the firmware sends Super+Shift+S. Released before it runs, because the\n# screenshot grabs the pointer to select a region, which fails while the key is still held.\nbindsym --release $mod+Shift+s exec --no-startup-id $XDG_CONFIG_HOME/i3/scripts/screenshot.sh\n\n# Fn+F9, the display key: the firmware sends Super+P. Odd workspaces to the panel, even ones to the\n# external output.\nbindsym $mod+p exec --no-startup-id /usr/local/lib/example-laptop/bin/zephyrus-display order\n\n# The keyboard backlight's level, shown when it changes.\nexec --no-startup-id /usr/local/lib/example-laptop/bin/zephyrus-kbd-notify\n# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the\n# mesh: replaced at every push. Once the controller places contributions to node-display-session\n# (ADR 0210), these become the module's contribution instead of a file in i3's directory.\n#\n# The model's panel and touchpad.\n\n# The internal panel is the primary output, where the bars' tray goes. Which monitors are on and where\n# is the display server's (autorandr, run at every session start).\nexec --no-startup-id /usr/local/lib/example-laptop/bin/zephyrus-display primary\n\n# The touchpad's settings again, by hand. X applies them itself whenever the device appears.\nbindsym $mod+Shift+x exec --no-startup-id /usr/local/lib/example-laptop/bin/zephyrus-touchpad reset\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "clipmenu",
|
||||
"content": [
|
||||
"# The clipboard's history key (module clipmenu, novox/hq ADR 0208). Owned by the mesh: replaced at\n# every push. clipmenu shows the history through `dmenu`, the node's dmenu-compatible command, which\n# the holder of node-launcher answers (rofi on the workstations); the chosen entry is put back on the\n# clipboard.\nbindsym $mod+period exec --no-startup-id clipmenu -p Clipboard\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "dunst",
|
||||
"content": [
|
||||
"# The notifications' key (module dunst, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# Super+N opens the menu (rofi) of the actions and links of every notification on screen, so one is\n# chosen from the keyboard as a tap chooses it on the notification.\nbindsym $mod+n exec --no-startup-id dunstctl context\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "feh",
|
||||
"content": [
|
||||
"# The wallpaper's key (module feh, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# It puts the declared wallpaper back, after a monitor change or a wallpaper set for the session.\nbindsym $mod+Shift+b exec --no-startup-id ~/.fehbg\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "i3status-rust",
|
||||
"content": [
|
||||
"# The bars (module i3status-rust, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# i3 reads this file through its configuration's `include ~/.config/i3/config.d/*.conf`. The bottom\n# bar shows the machine; the top bar the focused window and the tray, on the primary output. The\n# face is the monospace one every desktop module names.\nbar {\n font pango:JetBrainsMono Nerd Font 11\n position bottom\n status_command i3status-rs ~/.config/i3status-rust/bottom-bar.toml\n tray_output none\n colors {\n separator #ffffff\n background #000000\n statusline #ffffff\n # The text on an accent-coloured button is dark: light text on the accent was barely\n # legible.\n focused_workspace #de5200 #de5200 #000000\n active_workspace #de5200 #de5200 #000000\n inactive_workspace #000000 #000000 #ffffff\n urgent_workspace #2f343a #900000 #ffffff\n }\n}\n\nbar {\n font pango:JetBrainsMono Nerd Font 11\n position top\n status_command i3status-rs ~/.config/i3status-rust/top-bar.toml\n workspace_buttons no\n tray_output primary\n colors {\n separator #ffffff\n background #000000\n statusline #ffffff\n }\n}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "polychromatic",
|
||||
"content": [
|
||||
"# The Razer peripherals' tray (module polychromatic, novox/hq ADR 0208, ADR 0212). Owned by the mesh:\n# replaced at every push. This line is the tray's one start, at the session's start (exec, not\n# exec_always, so a reload starts nothing). The package's own login helper starts it too while the\n# application's setting \"Start the tray applet when I log on\" is ticked; polychromatic_check names\n# that as a second start.\nexec --no-startup-id polychromatic-tray-applet\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "rofi",
|
||||
"content": [
|
||||
"# The launcher's key bindings (module rofi, novox/hq ADR 0208). Owned by the mesh: replaced at every\n# push. i3 reads this file through its configuration's `include ~/.config/i3/config.d/*.conf`, after\n# the variables it sets, so $mod is i3's.\nbindsym $mod+d exec --no-startup-id ~/.local/bin/rofi-launch drun\nbindsym $mod+t exec --no-startup-id ~/.local/bin/rofi-launch run\nbindsym $mod+Shift+t exec --no-startup-id ~/.local/bin/rofi-launch sudo\nbindsym $mod+Shift+w exec --no-startup-id ~/.local/bin/rofi-launch window\nbindsym $mod+Escape exec --no-startup-id ~/.local/bin/rofi-powermenu\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "slack",
|
||||
"content": [
|
||||
"# Slack (module slack, novox/hq ADR 0208, ADR 0212). Owned by the mesh: replaced at every push.\n# Its one start: at login, to the tray (-s), with the arguments of the package's own desktop entry.\n# No XDG autostart entry starts it as well; slack_check names one if it appears.\nexec --no-startup-id /usr/bin/slack --gtk-version=3 -s\n# The chat window. Slack owns four X windows, and the only one i3 manages has the class \"slack\" in\n# lower case; the three of class \"Slack\" (the packaged entry's StartupWMClass) are its unmanaged\n# helpers and its tray icon, so a rule on class=\"Slack\" matches nothing. window_role keeps a call or\n# screen-share window out of the rules. $ws3 is i3's, in scope here.\nfor_window [class=\"(?i)^slack$\" window_role=\"browser-window\"] move to workspace $ws3\nfor_window [class=\"(?i)^slack$\" window_role=\"browser-window\"] floating disable\nfor_window [class=\"(?i)^slack$\" window_role=\"browser-window\"] border pixel 2\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,8 +1,9 @@
|
||||
# kernel
|
||||
|
||||
The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). Today it holds one:
|
||||
**transparent huge pages**, set per machine. The rest of the row (the kernel package, microcode, boot
|
||||
entries) is not built yet.
|
||||
The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). It holds two things:
|
||||
**transparent huge pages**, set per machine, and **the fallback kernel** (`linux-lts` with its headers) and
|
||||
its place on the boot menu (novox/hq ADR 0508 §9 and §10). The default kernel's package, and microcode, are
|
||||
not built yet.
|
||||
|
||||
## Why (novox/hq issue 746)
|
||||
|
||||
@@ -35,16 +36,34 @@ under the new mode. A failed rotation is said in the unit's journal and does not
|
||||
rotation is made while the machine is still starting (`systemctl is-system-running` says `initializing`
|
||||
or `starting`): nothing is in huge pages yet, and the persistent journal opens after this unit.
|
||||
|
||||
## The fallback kernel (novox/hq ADR 0508, issue 1841)
|
||||
|
||||
Every machine keeps a second, older kernel to start from, so a kernel upgrade that does not boot is not the
|
||||
end of the machine. The module declares the packages `linux-lts` and `linux-lts-headers` (so every DKMS module is
|
||||
built for the fallback kernel too) after its boot-entries service, so the entry is in place before the kernel is.
|
||||
|
||||
| | what | notes |
|
||||
|---|---|---|
|
||||
| file | `${dir:state}/boot-entries` | the script. It reads from the machine which loader booted it: the EFI variable `LoaderEntrySelected` present means systemd-boot, a `/boot/grub/grub.cfg` without it means GRUB, neither fails the unit loudly |
|
||||
| | systemd-boot | writes `/boot/loader/entries/mesh-linux-lts.conf` from `/proc/cmdline` without `initrd=` and `BOOT_IMAGE=`, with the microcode images present in `/boot` and the fallback initramfs as initrd lines; `loader.conf` and the default entry are never touched |
|
||||
| | GRUB | runs `grub-mkconfig -o /boot/grub/grub.cfg` only when the set of `/boot/vmlinuz-*` differs from the kernels the menu names, with `GRUB_TOP_LEVEL` naming the kernel the machine booted (`BOOT_IMAGE`; else the non-fallback kernel), since GRUB lists kernels in reverse version order and would otherwise make the fallback kernel the default; `/etc/default/grub` still overrides it when it sets one |
|
||||
| | room | while `/boot/vmlinuz-linux-lts` is not there yet, less than 96 MiB free on `/boot` fails the unit and says so (the fallback kernel writes about 72 MB, issue 1836) |
|
||||
| | a kernel-copying hook | a hook in `/etc/pacman.d/hooks` that the mesh did not write, triggered by kernel images and copying to one fixed file name in `/boot`, would put the fallback kernel in the default kernel's place: it is moved into `${dir:state}/taken-over/` and named in the unit's journal. Each kernel is then placed under its own package's name by the distribution's hooks |
|
||||
| file | `/etc/systemd/system/mesh-kernel-boot-entries.service` | oneshot, sandboxed: the file system read-only but for `/boot`, `/etc/pacman.d/hooks` and the module's state; no home, no network |
|
||||
| file | `/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook` | runs the script after a kernel image is installed, upgraded or removed |
|
||||
| packages | `linux-lts`, `linux-lts-headers` | declared after the service and the hook |
|
||||
|
||||
## Tools
|
||||
|
||||
| tool | | answers |
|
||||
|---|---|---|
|
||||
| `kernel_boot` | r | the loader (`systemd-boot`, `grub` or `none`), the menu's entries, the kernels in `/boot`, the entry booted (systemd-boot) or the kernel image booted (GRUB), whether the fallback kernel is on the menu with its kernel in `/boot`, and the room free on `/boot` in KiB |
|
||||
| `kernel_huge_pages` | r | the mode, defrag and shared-memory mode; each per-size control and what `inherit` resolves to; huge pages by kind from `/proc/meminfo` (MiB); the units holding the most file pages in huge folios (`memory.stat` `file_thp`) with their dirty and writeback bytes, khugepaged's collapses and the bytes their cgroup wrote (`io.stat`, the most any one device saw: a mirror and its members each count the same write) |
|
||||
|
||||
## Tests
|
||||
|
||||
`go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the
|
||||
manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); and the script as the mesh
|
||||
manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); the boot-entries script run with `sh` over stand-in roots (a systemd-boot entries directory, a GRUB menu, a hook that copies kernels to one fixed name): the entry from the booted command line, a menu generated again only when the kernels changed, no loader, no room, the taken-over hook; `kernel_boot` over the same roots; and the huge-pages script as the mesh
|
||||
writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once,
|
||||
rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a
|
||||
reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
// kernel_boot: which loader booted the machine, its entries, the kernels in /boot, the entry it booted and
|
||||
// whether the fallback kernel is on the menu (novox/hq ADR 0508 §9, issue 1841). Reads only.
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"unicode/utf16"
|
||||
)
|
||||
|
||||
const loaderSelected = "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f"
|
||||
|
||||
// BootState is what kernel_boot answers.
|
||||
type BootState struct {
|
||||
Loader string `json:"loader"`
|
||||
Entries []string `json:"entries"`
|
||||
Kernels []string `json:"kernels"`
|
||||
Booted string `json:"booted"`
|
||||
FallbackPresent bool `json:"fallback_present"`
|
||||
BootFreeKiB int64 `json:"boot_free_kib"`
|
||||
}
|
||||
|
||||
var grubKernel = regexp.MustCompile(`(?m)^\s*linux\s+(?:/boot)?/(vmlinuz-\S+)`)
|
||||
var grubTitle = regexp.MustCompile(`(?m)^\s*menuentry\s+'([^']*)'`)
|
||||
var bootImage = regexp.MustCompile(`BOOT_IMAGE=(?:/boot)?/?(\S+)`)
|
||||
|
||||
// Boot reads the machine, the loader the way the boot-entries script does: the loader's EFI variable means
|
||||
// systemd-boot, a grub.cfg without it GRUB.
|
||||
func (m *Machine) Boot() (*BootState, error) {
|
||||
b := &BootState{Loader: "none", Entries: []string{}, Kernels: []string{}}
|
||||
files, _ := filepath.Glob(m.path("boot/vmlinuz-*"))
|
||||
for _, f := range files {
|
||||
b.Kernels = append(b.Kernels, filepath.Base(f))
|
||||
}
|
||||
sort.Strings(b.Kernels)
|
||||
hasKernel := func(name string) bool {
|
||||
for _, k := range b.Kernels {
|
||||
if k == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
var s syscall.Statfs_t
|
||||
if err := syscall.Statfs(m.path("boot"), &s); err == nil {
|
||||
b.BootFreeKiB = int64(s.Bavail) * int64(s.Bsize) / 1024
|
||||
}
|
||||
if raw, err := os.ReadFile(m.path(loaderSelected)); err == nil {
|
||||
b.Loader = "systemd-boot"
|
||||
if len(raw) > 4 {
|
||||
var u []uint16
|
||||
for i := 4; i+1 < len(raw); i += 2 {
|
||||
u = append(u, uint16(raw[i])|uint16(raw[i+1])<<8)
|
||||
}
|
||||
b.Booted = strings.TrimRight(string(utf16.Decode(u)), "\x00")
|
||||
}
|
||||
files, _ := filepath.Glob(m.path("boot/loader/entries/*.conf"))
|
||||
for _, f := range files {
|
||||
b.Entries = append(b.Entries, filepath.Base(f))
|
||||
}
|
||||
sort.Strings(b.Entries)
|
||||
if raw, err := os.ReadFile(m.path("boot/loader/entries/mesh-linux-lts.conf")); err == nil {
|
||||
b.FallbackPresent = strings.Contains(string(raw), "/vmlinuz-linux-lts") && hasKernel("vmlinuz-linux-lts")
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
if raw, err := os.ReadFile(m.path("boot/grub/grub.cfg")); err == nil {
|
||||
b.Loader = "grub"
|
||||
for _, t := range grubTitle.FindAllStringSubmatch(string(raw), -1) {
|
||||
b.Entries = append(b.Entries, t[1])
|
||||
}
|
||||
for _, k := range grubKernel.FindAllStringSubmatch(string(raw), -1) {
|
||||
if k[1] == "vmlinuz-linux-lts" && hasKernel("vmlinuz-linux-lts") {
|
||||
b.FallbackPresent = true
|
||||
}
|
||||
}
|
||||
if cmd, err := m.read("proc/cmdline"); err == nil {
|
||||
if i := bootImage.FindStringSubmatch(cmd); i != nil {
|
||||
b.Booted = i[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
// kernel_boot over stand-in roots: a systemd-boot machine with and without the fallback entry, a GRUB machine
|
||||
// with and without the fallback kernel in its menu, and a machine with neither (novox/hq ADR 0508 §9, issue 1841).
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const selectedVar = "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f"
|
||||
|
||||
func put(t *testing.T, root, p, content string) {
|
||||
t.Helper()
|
||||
full := filepath.Join(root, p)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write(t, full, content, 0o644)
|
||||
}
|
||||
|
||||
func bootOf(t *testing.T, root string) *BootState {
|
||||
t.Helper()
|
||||
got, err := (&Machine{Root: root}).Boot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func TestKernelBootReadsASystemdBootMachineWithItsFallbackEntry(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
put(t, root, selectedVar, "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00")
|
||||
put(t, root, "boot/loader/entries/arch.conf", defaultEntry)
|
||||
put(t, root, "boot/loader/entries/mesh-linux-lts.conf", "title Arch Linux (linux-lts, fallback)\nlinux /vmlinuz-linux-lts\n")
|
||||
put(t, root, "boot/vmlinuz-linux", "k")
|
||||
put(t, root, "boot/vmlinuz-linux-lts", "k")
|
||||
got := bootOf(t, root)
|
||||
if got.Loader != "systemd-boot" || got.Booted != "arch.conf" || !got.FallbackPresent {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if len(got.Entries) != 2 || got.Entries[0] != "arch.conf" || got.Entries[1] != "mesh-linux-lts.conf" {
|
||||
t.Fatalf("entries: %v", got.Entries)
|
||||
}
|
||||
if len(got.Kernels) != 2 || got.Kernels[0] != "vmlinuz-linux" || got.Kernels[1] != "vmlinuz-linux-lts" {
|
||||
t.Fatalf("kernels: %v", got.Kernels)
|
||||
}
|
||||
if got.BootFreeKiB <= 0 {
|
||||
t.Fatalf("free: %d", got.BootFreeKiB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKernelBootSaysASystemdBootMachineHasNoFallbackYet(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
put(t, root, selectedVar, "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00")
|
||||
put(t, root, "boot/loader/entries/arch.conf", defaultEntry)
|
||||
put(t, root, "boot/vmlinuz-linux", "k")
|
||||
// The entry file alone is not the fallback: the kernel it names must be in /boot.
|
||||
put(t, root, "boot/loader/entries/mesh-linux-lts.conf", "linux /vmlinuz-linux-lts\n")
|
||||
if got := bootOf(t, root); got.FallbackPresent {
|
||||
t.Fatalf("a fallback entry without its kernel: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKernelBootReadsAGrubMachine(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
put(t, root, "boot/grub/grub.cfg", grubMenu)
|
||||
put(t, root, "boot/vmlinuz-linux", "k")
|
||||
put(t, root, "proc/cmdline", bootedCmdline)
|
||||
got := bootOf(t, root)
|
||||
if got.Loader != "grub" || got.FallbackPresent || got.Booted != "vmlinuz-linux" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if len(got.Entries) != 2 || got.Entries[0] != "Arch Linux" {
|
||||
t.Fatalf("entries: %v", got.Entries)
|
||||
}
|
||||
put(t, root, "boot/vmlinuz-linux-lts", "k")
|
||||
put(t, root, "boot/grub/grub.cfg", grubMenu+"menuentry 'Arch Linux LTS' {\n\tlinux\t/boot/vmlinuz-linux-lts root=x rw\n}\n")
|
||||
if got := bootOf(t, root); !got.FallbackPresent {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKernelBootSaysWhenThereIsNoLoader(t *testing.T) {
|
||||
got := bootOf(t, t.TempDir())
|
||||
if got.Loader != "none" || got.FallbackPresent || len(got.Entries) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package main
|
||||
|
||||
// The boot-entries script as the mesh writes it, run with sh over a stand-in root (novox/hq ADR 0508 §9 and
|
||||
// §10, issue 1841). The fixtures are the shapes read from machines: a systemd-boot entries directory, a GRUB
|
||||
// menu, a hook that copies every kernel image to the default kernel's name, and the command line a machine
|
||||
// booted with.
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const bootedCmdline = "initrd=\\initramfs-linux.img BOOT_IMAGE=/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet loglevel=3 initrd=\\intel-ucode.img\n"
|
||||
|
||||
const defaultEntry = "title Arch Linux\nlinux /vmlinuz-linux\ninitrd /intel-ucode.img\ninitrd /initramfs-linux.img\noptions root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet\n"
|
||||
|
||||
const loaderConf = "default arch.conf\ntimeout 3\nconsole-mode max\n"
|
||||
|
||||
// grubMenu is the part of a generated grub.cfg that names kernels, for a machine holding one kernel.
|
||||
const grubMenu = `menuentry 'Arch Linux' --class arch {
|
||||
set root='hd0,gpt2'
|
||||
linux /boot/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet
|
||||
initrd /boot/initramfs-linux.img
|
||||
}
|
||||
menuentry 'Arch Linux, with fallback initramfs' --class arch {
|
||||
linux /boot/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet
|
||||
initrd /boot/initramfs-linux-fallback.img
|
||||
}
|
||||
`
|
||||
|
||||
// kernelCopyingHook copies every installed kernel image to the default kernel's name on the boot partition.
|
||||
const kernelCopyingHook = `[Trigger]
|
||||
Operation = Install
|
||||
Operation = Upgrade
|
||||
Type = Path
|
||||
Target = usr/lib/modules/*/vmlinuz
|
||||
|
||||
[Action]
|
||||
Description = Copying the kernel to the default name
|
||||
When = PostTransaction
|
||||
Exec = /usr/bin/sh -c '/usr/bin/cp /usr/lib/modules/$(uname -r)/vmlinuz /boot/vmlinuz-linux'
|
||||
`
|
||||
|
||||
// otherHook is a hook of the distribution's shape that has nothing to do with kernel images.
|
||||
const otherHook = `[Trigger]
|
||||
Operation = Upgrade
|
||||
Type = Package
|
||||
Target = pacman-mirrorlist
|
||||
|
||||
[Action]
|
||||
Description = Updating the mirror list
|
||||
When = PostTransaction
|
||||
Exec = /usr/bin/cp /etc/pacman.d/mirrorlist.pacnew /etc/pacman.d/mirrorlist
|
||||
`
|
||||
|
||||
type bootBed struct {
|
||||
script, root, state, bin, mkconfig string
|
||||
need string
|
||||
}
|
||||
|
||||
func newBootBed(t *testing.T) bootBed {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
b := bootBed{script: filepath.Join(dir, "boot-entries"), root: filepath.Join(dir, "root"), state: filepath.Join(dir, "state"),
|
||||
bin: filepath.Join(dir, "bin"), mkconfig: filepath.Join(dir, "mkconfig")}
|
||||
content := manifest(t).resource(t, "boot-entries").text(t)
|
||||
content = strings.ReplaceAll(content, "${dir:state}", b.state)
|
||||
if strings.Contains(content, "${") {
|
||||
t.Fatalf("a substitution the test does not make is left in the script")
|
||||
}
|
||||
for _, d := range []string{b.root + "/boot", b.root + "/proc", b.root + "/etc/pacman.d/hooks", b.state, b.bin} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write(t, b.script, content, 0o755)
|
||||
write(t, b.root+"/proc/cmdline", bootedCmdline, 0o644)
|
||||
// the stand-in grub-mkconfig records the kernel it was told to put first, then its arguments.
|
||||
write(t, filepath.Join(b.bin, "grub-mkconfig"), "#!/bin/sh\necho \"top=$GRUB_TOP_LEVEL $@\" >>"+b.mkconfig+"\n", 0o755)
|
||||
return b
|
||||
}
|
||||
|
||||
func (b bootBed) put(t *testing.T, p, content string) {
|
||||
t.Helper()
|
||||
full := filepath.Join(b.root, p)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write(t, full, content, 0o644)
|
||||
}
|
||||
|
||||
func (b bootBed) systemdBoot(t *testing.T) {
|
||||
b.put(t, "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f", "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00")
|
||||
b.put(t, "boot/loader/loader.conf", loaderConf)
|
||||
b.put(t, "boot/loader/entries/arch.conf", defaultEntry)
|
||||
b.put(t, "boot/vmlinuz-linux", "k")
|
||||
}
|
||||
|
||||
func (b bootBed) run(t *testing.T) (string, error) {
|
||||
t.Helper()
|
||||
args := []string{b.script, "run", b.root, b.state}
|
||||
if b.need != "" {
|
||||
args = append(args, b.need)
|
||||
}
|
||||
cmd := exec.Command("sh", args...)
|
||||
cmd.Env = append(os.Environ(), "PATH="+b.bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
out, err := cmd.CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
func (b bootBed) read(t *testing.T, p string) string {
|
||||
t.Helper()
|
||||
raw, _ := os.ReadFile(filepath.Join(b.root, p))
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func TestASystemdBootMachineGetsTheFallbackEntryFromItsBootedCommandLine(t *testing.T) {
|
||||
b := newBootBed(t)
|
||||
b.systemdBoot(t)
|
||||
b.put(t, "boot/intel-ucode.img", "u")
|
||||
out, err := b.run(t)
|
||||
if err != nil || !strings.Contains(out, "wrote") {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
want := "title Arch Linux (linux-lts, fallback)\nlinux /vmlinuz-linux-lts\ninitrd /intel-ucode.img\ninitrd /initramfs-linux-lts.img\n" +
|
||||
"options root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet loglevel=3\n"
|
||||
if got := b.read(t, "boot/loader/entries/mesh-linux-lts.conf"); got != want {
|
||||
t.Fatalf("entry:\n%s\nwant:\n%s", got, want)
|
||||
}
|
||||
// The default entry and the loader's own file are as they were.
|
||||
if b.read(t, "boot/loader/entries/arch.conf") != defaultEntry || b.read(t, "boot/loader/loader.conf") != loaderConf {
|
||||
t.Fatalf("the default entry or loader.conf was touched")
|
||||
}
|
||||
if out, err := b.run(t); err != nil || !strings.Contains(out, "as it was") {
|
||||
t.Fatalf("again: %v %s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithoutMicrocodeThereIsNoMicrocodeLine(t *testing.T) {
|
||||
b := newBootBed(t)
|
||||
b.systemdBoot(t)
|
||||
if out, err := b.run(t); err != nil {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
if got := b.read(t, "boot/loader/entries/mesh-linux-lts.conf"); strings.Contains(got, "ucode") || !strings.Contains(got, "initrd /initramfs-linux-lts.img") {
|
||||
t.Fatalf("entry: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGrubMachineIsGeneratedAgainOnlyWhenItsKernelsChanged(t *testing.T) {
|
||||
b := newBootBed(t)
|
||||
b.put(t, "boot/grub/grub.cfg", grubMenu)
|
||||
b.put(t, "boot/vmlinuz-linux", "k")
|
||||
if out, err := b.run(t); err != nil || !strings.Contains(out, "left as it is") {
|
||||
t.Fatalf("same: %v %s", err, out)
|
||||
}
|
||||
if _, err := os.Stat(b.mkconfig); err == nil {
|
||||
t.Fatalf("the menu was generated again with the same kernels")
|
||||
}
|
||||
b.put(t, "boot/vmlinuz-linux-lts", "k")
|
||||
if out, err := b.run(t); err != nil || !strings.Contains(out, "generated again") {
|
||||
t.Fatalf("changed: %v %s", err, out)
|
||||
}
|
||||
raw, _ := os.ReadFile(b.mkconfig)
|
||||
if string(raw) != "top=/boot/vmlinuz-linux -o "+filepath.Join(b.root, "boot/grub/grub.cfg")+"\n" {
|
||||
t.Fatalf("grub-mkconfig was asked %q", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0508 §9: GRUB's menu lists kernels in reverse version order, so linux-lts would come first and become the
|
||||
// default entry (GRUB_DEFAULT=0); the kernel the machine booted is put first, and the fallback kernel never.
|
||||
func TestGrubKeepsTheBootedKernelFirstAndNeverTheFallback(t *testing.T) {
|
||||
for name, c := range map[string]struct{ cmdline, want string }{
|
||||
"booted from the default kernel": {"BOOT_IMAGE=/vmlinuz-linux root=UUID=x rw\n", "top=/boot/vmlinuz-linux"},
|
||||
"booted from a grub device path": {"BOOT_IMAGE=(hd0,msdos1)/vmlinuz-linux-zen root=UUID=x rw\n", "top=/boot/vmlinuz-linux-zen"},
|
||||
"booted from the fallback kernel": {"BOOT_IMAGE=/vmlinuz-linux-lts root=UUID=x rw\n", "top=/boot/vmlinuz-linux"},
|
||||
"no image on the command line": {"root=UUID=x rw\n", "top=/boot/vmlinuz-linux"},
|
||||
"an image no longer in /boot": {"BOOT_IMAGE=/vmlinuz-linux-gone root=UUID=x rw\n", "top=/boot/vmlinuz-linux"},
|
||||
} {
|
||||
b := newBootBed(t)
|
||||
b.put(t, "boot/grub/grub.cfg", grubMenu)
|
||||
b.put(t, "proc/cmdline", c.cmdline)
|
||||
for _, k := range []string{"vmlinuz-linux", "vmlinuz-linux-lts", "vmlinuz-linux-zen"} {
|
||||
b.put(t, "boot/"+k, "k")
|
||||
}
|
||||
if out, err := b.run(t); err != nil {
|
||||
t.Fatalf("%s: %v %s", name, err, out)
|
||||
}
|
||||
raw, _ := os.ReadFile(b.mkconfig)
|
||||
if !strings.HasPrefix(string(raw), c.want+" -o ") {
|
||||
t.Errorf("%s: grub-mkconfig was asked %q, want %s first", name, raw, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMachineWithNoLoaderFailsLoudly(t *testing.T) {
|
||||
b := newBootBed(t)
|
||||
out, err := b.run(t)
|
||||
if err == nil || !strings.Contains(out, "no boot loader found") {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABootPartitionWithoutRoomFailsBeforeAnythingIsWritten(t *testing.T) {
|
||||
b := newBootBed(t)
|
||||
b.systemdBoot(t)
|
||||
b.need = "999999999999"
|
||||
out, err := b.run(t)
|
||||
if err == nil || !strings.Contains(out, "free room there before the kernel is installed") {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
if b.read(t, "boot/loader/entries/mesh-linux-lts.conf") != "" {
|
||||
t.Fatalf("an entry was written")
|
||||
}
|
||||
// Once the fallback kernel is in place the room is not asked for again.
|
||||
b.put(t, "boot/vmlinuz-linux-lts", "k")
|
||||
if out, err := b.run(t); err != nil {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHookThatCopiesKernelsToOneFixedNameIsMovedIntoTheStateAndNamed(t *testing.T) {
|
||||
b := newBootBed(t)
|
||||
b.systemdBoot(t)
|
||||
b.put(t, "etc/pacman.d/hooks/60-copy-kernel.hook", kernelCopyingHook)
|
||||
b.put(t, "etc/pacman.d/hooks/70-mirrorlist.hook", otherHook)
|
||||
b.put(t, "etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook", "[Trigger]\nTarget = usr/lib/modules/*/vmlinuz\n[Action]\nExec = /bin/cp x /boot/y\n")
|
||||
out, err := b.run(t)
|
||||
if err != nil || !strings.Contains(out, "taken over: the pacman hook 60-copy-kernel.hook") {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
if b.read(t, "etc/pacman.d/hooks/60-copy-kernel.hook") != "" {
|
||||
t.Fatalf("the copying hook is still in the hook directory")
|
||||
}
|
||||
if raw, _ := os.ReadFile(filepath.Join(b.state, "taken-over", "60-copy-kernel.hook")); string(raw) != kernelCopyingHook {
|
||||
t.Fatalf("moved hook: %q", raw)
|
||||
}
|
||||
if b.read(t, "etc/pacman.d/hooks/70-mirrorlist.hook") != otherHook || b.read(t, "etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook") == "" {
|
||||
t.Fatalf("a hook that copies no kernel was moved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBootEntriesUnitAndHookAreTheModules(t *testing.T) {
|
||||
m := manifest(t)
|
||||
unit := m.resource(t, "boot-unit").text(t)
|
||||
for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/boot-entries run", "ProtectSystem=strict",
|
||||
"ReadWritePaths=/boot ${dir:state} /etc/pacman.d/hooks", "NoNewPrivileges=yes", "WantedBy=multi-user.target"} {
|
||||
if !strings.Contains(unit, line+"\n") {
|
||||
t.Errorf("the unit lacks %q", line)
|
||||
}
|
||||
}
|
||||
hook := m.resource(t, "boot-hook")
|
||||
text := hook.text(t)
|
||||
if hook["path"] != "/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook" || !strings.Contains(text, "Target = usr/lib/modules/*/vmlinuz\n") ||
|
||||
!strings.Contains(text, "When = PostTransaction\n") || !strings.Contains(text, "Exec = ${dir:state}/boot-entries run\n") {
|
||||
t.Fatalf("the hook: %v\n%s", hook["path"], text)
|
||||
}
|
||||
apply := m.resource(t, "boot-apply")
|
||||
if apply["unit"] != "mesh-kernel-boot-entries.service" || apply["state"] != "running" || apply["boot"] != "enabled" {
|
||||
t.Fatalf("apply: %v", apply)
|
||||
}
|
||||
}
|
||||
|
||||
// The fallback kernel and its headers are declared after the boot-entries service, so the entry is placed
|
||||
// before the kernel is installed and every DKMS module is built for it (ADR 0508 §9).
|
||||
func TestTheFallbackKernelAndItsHeadersAreDeclaredAfterTheBootEntriesUnit(t *testing.T) {
|
||||
m := manifest(t)
|
||||
at := map[string]int{}
|
||||
for i, r := range m.Resources {
|
||||
at[r["id"].(string)] = i
|
||||
}
|
||||
for id, pkg := range map[string]string{"linux-lts": "linux-lts", "linux-lts-headers": "linux-lts-headers"} {
|
||||
r := m.resource(t, id)
|
||||
if r["type"] != "package" || r["package"] != pkg || r["absent"] == true {
|
||||
t.Errorf("%s: %v", id, r)
|
||||
}
|
||||
if at[id] < at["boot-apply"] || at[id] < at["boot-hook"] {
|
||||
t.Errorf("%s is declared before the boot-entries unit and hook", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -46,5 +46,15 @@ func tools(m *Machine) []stdio.Tool {
|
||||
return m.HugePages(n)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "kernel_boot",
|
||||
Description: "How this machine boots and whether the fallback kernel is on its menu (novox/hq ADR 0508 §9): the loader that booted it " +
|
||||
"(systemd-boot, read from the EFI variable it sets, grub, or none), the menu's entries, the kernels in /boot, the entry " +
|
||||
"the machine booted (systemd-boot) or the kernel image it booted (grub), whether the fallback kernel linux-lts is on " +
|
||||
"the menu with its kernel in /boot, and the room free on /boot in KiB (the fallback kernel needs about 72 MB). " +
|
||||
"Replaces ls /boot, bootctl list, reading grub.cfg and df /boot. (r)",
|
||||
Input: schema(map[string]any{}),
|
||||
Run: func(args map[string]any) (any, error) { return m.Boot() },
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) {
|
||||
for _, r := range m.Resources {
|
||||
ids = append(ids, r["id"].(string))
|
||||
}
|
||||
if strings.Join(ids, " ") != "state huge-pages unit apply" {
|
||||
if strings.Join(ids, " ") != "state huge-pages unit apply boot-entries boot-unit boot-hook boot-apply linux-lts linux-lts-headers" {
|
||||
t.Fatalf("order: %v", ids)
|
||||
}
|
||||
script := m.resource(t, "huge-pages")
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# Module kernel, novox/hq ADR 0508 §9, issue 1841. After a kernel image is installed, upgraded or removed, the
|
||||
# fallback kernel's boot entry (or the GRUB menu) is placed again. Written by the mesh; change the module.
|
||||
[Trigger]
|
||||
Operation = Install
|
||||
Operation = Upgrade
|
||||
Operation = Remove
|
||||
Type = Path
|
||||
Target = usr/lib/modules/*/vmlinuz
|
||||
|
||||
[Action]
|
||||
Description = Placing the fallback kernel on the boot menu
|
||||
When = PostTransaction
|
||||
Exec = ${dir:state}/boot-entries run
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/bin/sh
|
||||
# The fallback kernel's place on this machine's boot menu (module kernel, novox/hq ADR 0508 §9 and §10,
|
||||
# issue 1841). Written by the mesh and run as root by mesh-kernel-boot-entries.service at boot and when the
|
||||
# module is applied, and by the pacman hook 90-mesh-kernel-boot-entries.hook after a kernel is installed,
|
||||
# upgraded or removed. Replaced on every push; change the module, never this file.
|
||||
#
|
||||
# boot-entries run [<root> [<this module's state> [<room needed in /boot, KiB>]]]
|
||||
#
|
||||
# The three are arguments only so the module's tests can run this against a stand-in tree.
|
||||
#
|
||||
# It reads from the machine which loader booted it, never from a setting:
|
||||
# systemd-boot the EFI variable LoaderEntrySelected is present: the entry mesh-linux-lts.conf is written
|
||||
# from the command line the machine booted with.
|
||||
# GRUB /boot/grub/grub.cfg exists and that variable does not: the menu is generated again, only
|
||||
# when the set of kernels in /boot differs from the menu's.
|
||||
# neither it fails loudly. The default entry is never touched.
|
||||
# And a pacman hook of someone else's that copies kernel images to one fixed name in /boot is moved into this
|
||||
# module's state first (ADR 0508 §10): it would put the fallback kernel in the default kernel's place.
|
||||
set -eu
|
||||
|
||||
root=/
|
||||
state='${dir:state}'
|
||||
[ $# -ge 2 ] && root=$(printf '%s' "$2" | sed 's|/$||')
|
||||
[ $# -ge 3 ] && state=$3
|
||||
boot=$root/boot
|
||||
hooks=$root/etc/pacman.d/hooks
|
||||
efivar=$root/sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f
|
||||
ours=90-mesh-kernel-boot-entries.hook
|
||||
# The fallback kernel and its initramfs are about 72 MB in /boot (issue 1836): less room fails the unit
|
||||
# before a package writes half of them.
|
||||
need_kib=98304
|
||||
[ $# -ge 4 ] && need_kib=$4
|
||||
|
||||
[ $# -ge 1 ] && [ "$1" = run ] || {
|
||||
echo "usage: boot-entries run [<root> [<state directory> [<KiB needed>]]]" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
# A hook that is not ours, triggered by kernel images, whose action copies to one fixed name in /boot.
|
||||
copies_kernels() {
|
||||
grep -Eqi '^[[:space:]]*Target[[:space:]]*=.*(vmlinuz|usr/lib/modules)' "$1" &&
|
||||
grep -Eq '^[[:space:]]*Exec[[:space:]]*=.*(cp|install|ln|rsync)[[:space:]].*/boot/[A-Za-z0-9._+-]+([^A-Za-z0-9._+/%$*-]|$)' "$1"
|
||||
}
|
||||
|
||||
if [ -d "$hooks" ]; then
|
||||
for hook in "$hooks"/*.hook; do
|
||||
[ -f "$hook" ] || continue
|
||||
[ "$(basename "$hook")" = "$ours" ] && continue
|
||||
copies_kernels "$hook" || continue
|
||||
mkdir -p "$state/taken-over"
|
||||
mv -f "$hook" "$state/taken-over/$(basename "$hook")"
|
||||
echo "taken over: the pacman hook $(basename "$hook") copied kernel images to one fixed name in /boot; moved to $state/taken-over/ so each kernel keeps its own package's name"
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -e "$efivar" ]; then
|
||||
loader=systemd-boot
|
||||
elif [ -f "$boot/grub/grub.cfg" ]; then
|
||||
loader=grub
|
||||
else
|
||||
echo "no boot loader found: the machine was not booted by systemd-boot (no LoaderEntrySelected variable) and there is no $boot/grub/grub.cfg; the fallback kernel has no menu to be placed on" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Room first, only while the fallback kernel is not in /boot yet.
|
||||
if [ ! -e "$boot/vmlinuz-linux-lts" ]; then
|
||||
free=$(df -Pk "$boot" | awk 'NR==2 {print $4}')
|
||||
[ -n "$free" ] || free=0
|
||||
if [ "$free" -lt "$need_kib" ]; then
|
||||
echo "$boot has $free KiB free and the fallback kernel needs $need_kib KiB: free room there before the kernel is installed" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
case "$loader" in
|
||||
systemd-boot)
|
||||
entries=$boot/loader/entries
|
||||
cmdline=$(sed -e 's/BOOT_IMAGE=[^ ]*//g' -e 's/initrd=[^ ]*//g' "$root/proc/cmdline" | tr -s ' \n' ' ' | sed -e 's/^ //' -e 's/ $//')
|
||||
[ -n "$cmdline" ] || {
|
||||
echo "$root/proc/cmdline holds no command line to boot the fallback kernel with" >&2
|
||||
exit 1
|
||||
}
|
||||
want=$(
|
||||
echo "title Arch Linux (linux-lts, fallback)"
|
||||
echo "linux /vmlinuz-linux-lts"
|
||||
for ucode in intel-ucode.img amd-ucode.img; do
|
||||
[ -e "$boot/$ucode" ] && echo "initrd /$ucode"
|
||||
done
|
||||
echo "initrd /initramfs-linux-lts.img"
|
||||
echo "options $cmdline"
|
||||
)
|
||||
mkdir -p "$entries"
|
||||
if [ "$(cat "$entries/mesh-linux-lts.conf" 2>/dev/null || true)" = "$want" ]; then
|
||||
echo "systemd-boot: mesh-linux-lts.conf, as it was"
|
||||
else
|
||||
printf '%s\n' "$want" >"$entries/mesh-linux-lts.conf"
|
||||
echo "systemd-boot: wrote $entries/mesh-linux-lts.conf from the command line the machine booted with; the default entry is as it was"
|
||||
fi
|
||||
;;
|
||||
grub)
|
||||
cfg=$boot/grub/grub.cfg
|
||||
menu=$(sed -n 's|^[[:space:]]*linux[[:space:]]*\(/boot\)\{0,1\}/\(vmlinuz-[^[:space:]]*\).*|\2|p' "$cfg" | sort -u)
|
||||
have=$(cd "$boot" && ls -d vmlinuz-* 2>/dev/null | sort -u || true)
|
||||
if [ "$menu" = "$have" ]; then
|
||||
echo "grub: the menu lists the kernels in /boot, left as it is"
|
||||
else
|
||||
# GRUB lists kernels in reverse version order, so the fallback kernel would come first and become the
|
||||
# default entry: the kernel the machine booted is put first (GRUB_TOP_LEVEL; /etc/default/grub still
|
||||
# overrides it when it sets one), the fallback kernel never.
|
||||
top=$(tr ' ' '\n' <"$root/proc/cmdline" | sed -n 's|^BOOT_IMAGE=\(.*\)$|\1|p' | head -n 1)
|
||||
top=$(printf '%s' "$top" | sed 's|.*/||')
|
||||
case "$top" in
|
||||
vmlinuz-linux-lts | "") top= ;;
|
||||
esac
|
||||
[ -n "$top" ] && [ -e "$boot/$top" ] || top=
|
||||
if [ -z "$top" ]; then
|
||||
if [ -e "$boot/vmlinuz-linux" ]; then
|
||||
top=vmlinuz-linux
|
||||
else
|
||||
top=$(printf '%s\n' $have | grep -v '^vmlinuz-linux-lts$' | head -n 1 || true)
|
||||
fi
|
||||
fi
|
||||
if [ -n "$top" ]; then
|
||||
GRUB_TOP_LEVEL=/boot/$top grub-mkconfig -o "$cfg"
|
||||
else
|
||||
grub-mkconfig -o "$cfg"
|
||||
fi
|
||||
first=$top
|
||||
[ -n "$first" ] || first="the menu's own order"
|
||||
echo "grub: the kernels in /boot changed ($(echo $have | tr '\n' ' ')); the menu is generated again with $first first; the default entry is as it was"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,22 @@
|
||||
[Unit]
|
||||
Description=The fallback kernel's place on this machine's boot menu
|
||||
# Module kernel, novox/hq ADR 0508 §9, issue 1841. Starting this unit reads which loader booted the machine and
|
||||
# places the fallback kernel's boot entry (systemd-boot) or generates the menu again (GRUB); it fails, and says
|
||||
# why, when there is no loader or the boot partition has no room. The kernel packages are declared after this
|
||||
# unit, so the entry is in place before they are installed.
|
||||
After=local-fs.target
|
||||
Wants=local-fs.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=${dir:state}/boot-entries run
|
||||
# It writes the boot partition, the pacman hook directory and its own state, and nothing else.
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/boot ${dir:state} /etc/pacman.d/hooks
|
||||
ProtectHome=yes
|
||||
PrivateNetwork=yes
|
||||
NoNewPrivileges=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -13,7 +13,8 @@
|
||||
}
|
||||
},
|
||||
"tools": [
|
||||
"kernel_huge_pages"
|
||||
"kernel_huge_pages",
|
||||
"kernel_boot"
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
@@ -53,6 +54,52 @@
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "boot-entries",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/boot-entries",
|
||||
"mode": "0755",
|
||||
"trusted": true,
|
||||
"template": "files/boot-entries.sh"
|
||||
},
|
||||
{
|
||||
"id": "boot-unit",
|
||||
"type": "file",
|
||||
"path": "/etc/systemd/system/mesh-kernel-boot-entries.service",
|
||||
"mode": "0644",
|
||||
"template": "files/mesh-kernel-boot-entries.service"
|
||||
},
|
||||
{
|
||||
"id": "boot-hook",
|
||||
"type": "file",
|
||||
"path": "/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook",
|
||||
"mode": "0644",
|
||||
"template": "files/90-mesh-kernel-boot-entries.hook"
|
||||
},
|
||||
{
|
||||
"id": "boot-apply",
|
||||
"type": "service",
|
||||
"unit": "mesh-kernel-boot-entries.service",
|
||||
"state": "running",
|
||||
"boot": "enabled",
|
||||
"restart-on": [
|
||||
"boot-unit",
|
||||
"boot-entries"
|
||||
],
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "linux-lts",
|
||||
"type": "package",
|
||||
"package": "linux-lts"
|
||||
},
|
||||
{
|
||||
"id": "linux-lts-headers",
|
||||
"type": "package",
|
||||
"package": "linux-lts-headers"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
|
||||
@@ -39,12 +39,8 @@ func TestTheBaseParses(t *testing.T) {
|
||||
t.Skip("logrotate is not installed here; the base configuration is not dry-run")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
content := strings.ReplaceAll(templateText(t, manifest(t).resource(t, "config")), "include "+RulesDir, "include "+filepath.Join(dir, "d"))
|
||||
if err := os.Mkdir(filepath.Join(dir, "d"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conf := filepath.Join(dir, "logrotate.conf")
|
||||
if err := os.WriteFile(conf, []byte(content), 0o644); err != nil {
|
||||
if err := os.WriteFile(conf, []byte(dryRunBase(t, dir)), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := exec.Command(logrotate, "-d", "-s", filepath.Join(dir, "state"), conf).CombinedOutput()
|
||||
@@ -52,3 +48,40 @@ func TestTheBaseParses(t *testing.T) {
|
||||
t.Fatalf("logrotate -d: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
// The dry run proves the base parses, not what the machine's logs are: the unit level runs shut off in
|
||||
// bubblewrap (mesh-sdk buildstep.Isolated), where /var/log is not there, and a base block without
|
||||
// missingok then fails its stat (issue 1891).
|
||||
func TestTheDryRunReadsNothingOfTheMachine(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
content := dryRunBase(t, dir)
|
||||
for _, machine := range []string{"include " + RulesDir, "/var/log/"} {
|
||||
if strings.Contains(content, machine) {
|
||||
t.Fatalf("the dry run's base still names %s of the machine:\n%s", machine, content)
|
||||
}
|
||||
}
|
||||
for _, log := range []string{"wtmp", "btmp"} {
|
||||
if _, err := os.Stat(filepath.Join(dir, "log", log)); err != nil {
|
||||
t.Fatalf("the base's own log %s is not in the dry run's directory: %v", log, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dryRunBase is the base configuration pointed into dir: its include at dir/d, and its own logs at dir/log,
|
||||
// both made there.
|
||||
func dryRunBase(t *testing.T, dir string) string {
|
||||
t.Helper()
|
||||
content := strings.ReplaceAll(templateText(t, manifest(t).resource(t, "config")), "include "+RulesDir, "include "+filepath.Join(dir, "d"))
|
||||
content = strings.ReplaceAll(content, "/var/log/", filepath.Join(dir, "log")+"/")
|
||||
for _, d := range []string{"d", "log"} {
|
||||
if err := os.Mkdir(filepath.Join(dir, d), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, log := range []string{"wtmp", "btmp"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, "log", log), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
@@ -37,8 +37,17 @@ Three read tools answer what the module stored in its own states, without the gu
|
||||
forge-account provision's ssh name, so the account's own ssh files and git rewrite are used) into a workspace of the
|
||||
work directory, then runs `go mod download` and `go mod verify` in every directory with a `go.mod` that does not vendor,
|
||||
with `GOTOOLCHAIN=local`, `GOFLAGS=-mod=mod` and the module's cache directory. Every command runs with an
|
||||
environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. Workspaces older than a day
|
||||
are removed.
|
||||
environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes.
|
||||
|
||||
## Room to work and the workspace's end
|
||||
|
||||
Every step is refused, ending `refused` with `refused: <node> has N GiB free under the floor of 20 GiB (on <dir>)`, while
|
||||
the work or the cache directory's filesystem has less than 20 GiB free (build-agent's floor, issue 1839); the health is
|
||||
then unhealthy with the same text, so the builder chooses another node and the controller raises the condition. A
|
||||
workspace is held while a step runs on it. `build_go_release {workspace}` removes it at once (no guard, no floor; one
|
||||
already gone is no error, one a running step holds is refused); its caller calls it once its last step answered. A
|
||||
failed fetch removes its own workspace. A workspace no caller released is removed two hours after its last use (ADR
|
||||
0509), and the removal is said in the log as the caller's fault.
|
||||
|
||||
A pin on a pull request's head (issue 1148): when `go mod download` fails on a `git.novox.be/novox/*` pseudo-version, the
|
||||
fetch looks for that commit among the forge's `refs/pull/*/head`, places the repository at it under the workspace's
|
||||
|
||||
@@ -92,7 +92,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no module in %q of workspace %s: %w", module, workspace, err)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifest(raw)
|
||||
manifest, err := catalogue.ParseManifestForInputs(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the manifest in %q: %w", module, err)
|
||||
}
|
||||
|
||||
@@ -237,3 +237,21 @@ func TestABuildOutputDirectoryLinkedOutsideTheModuleIsRefused(t *testing.T) {
|
||||
t.Errorf("a file outside the module was touched: %q, %v", body, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1751 (ADR 0514 §2, the review's K2): a manifest whose config value falls back to a predefined key builds. The
|
||||
// builder holds no published list of the keys, so it reads the manifest for its inputs (ParseManifestForInputs), not
|
||||
// as a judge of config values; claude-code's real manifest (capturedClaudeCodeManifest) is the fixture.
|
||||
func TestAManifestFallingBackToAPredefinedKeyBuilds(t *testing.T) {
|
||||
raw := []byte(capturedClaudeCodeManifest)
|
||||
if !strings.Contains(string(raw), `"predefined": "agent-managed-settings"`) {
|
||||
t.Fatal("the captured manifest no longer falls back to a predefined key")
|
||||
}
|
||||
m, states, _ := buildModule(t, string(raw), "binary")
|
||||
res, st := runBuild(t, m, states, "tools")
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if res.Artifact != "tools" || res.Binary != "claude-code" || !strings.HasPrefix(res.Digest, "sha256:") {
|
||||
t.Errorf("result = %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
package main
|
||||
|
||||
// capturedClaudeCodeManifest is modules/claude-code/module.json of this catalogue at 56c68be7, copied verbatim on
|
||||
// 2026-10-22 for issue 1751 (the R4e review's finding K2): a real manifest whose config value managed_settings falls
|
||||
// back to the predefined key agent-managed-settings (ADR 0348 §7), which the builder holds no published list for. It
|
||||
// lives in a test file, not under testdata, because the catalogue's paths check (ADR 0260) reads every other file of a
|
||||
// module and the manifest names claude-code's own paths.
|
||||
const capturedClaudeCodeManifest = `{
|
||||
"module": "claude-code",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"slug": "agent",
|
||||
"capabilities": [
|
||||
"package-manager"
|
||||
],
|
||||
"requires": [
|
||||
"mcp-endpoint",
|
||||
{
|
||||
"name": "gitea-read",
|
||||
"optional": true
|
||||
}
|
||||
],
|
||||
"binds": {
|
||||
"mcp-endpoint": "${dir:state}/mcp-endpoint.json"
|
||||
},
|
||||
"secrets": {
|
||||
"gitea-read": "${dir:state}/gitea-read.secret"
|
||||
},
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
],
|
||||
"state": [
|
||||
"servers",
|
||||
"holdings",
|
||||
"config",
|
||||
"proposals",
|
||||
"questions",
|
||||
"memory",
|
||||
"waits"
|
||||
],
|
||||
"reads": [
|
||||
"claude-licence-manager.bindings"
|
||||
],
|
||||
"tools": [
|
||||
"claude_code_status",
|
||||
"claude_code_render",
|
||||
"claude_code_guard",
|
||||
"claude_code_pull",
|
||||
"claude_code_grant",
|
||||
"claude_code_add_api_key",
|
||||
"claude_code_registrations",
|
||||
"claude_code_memory",
|
||||
"claude_code_mcp_list",
|
||||
"claude_code_mcp_register",
|
||||
"claude_code_mcp_unregister",
|
||||
"claude_code_skill_list",
|
||||
"claude_code_skill_register",
|
||||
"claude_code_skill_unregister",
|
||||
"claude_code_agent_list",
|
||||
"claude_code_agent_register",
|
||||
"claude_code_agent_unregister",
|
||||
"claude_code_command_list",
|
||||
"claude_code_command_register",
|
||||
"claude_code_command_unregister",
|
||||
"claude_code_hook_list",
|
||||
"claude_code_hook_register",
|
||||
"claude_code_hook_unregister",
|
||||
"claude_code_output_style_list",
|
||||
"claude_code_output_style_register",
|
||||
"claude_code_output_style_unregister",
|
||||
"claude_code_instructions_list",
|
||||
"claude_code_instructions_register",
|
||||
"claude_code_instructions_unregister",
|
||||
"claude_code_instructions_propose",
|
||||
"claude_code_proposals",
|
||||
"claude_code_ask",
|
||||
"claude_code_answer",
|
||||
"claude_code_wait",
|
||||
"claude_code_settings_get",
|
||||
"claude_code_settings_set",
|
||||
"claude_code_settings_clear",
|
||||
"claude_code_permission_add",
|
||||
"claude_code_permission_remove",
|
||||
"claude_code_config_list",
|
||||
"claude_code_config_show",
|
||||
"claude_code_config_status",
|
||||
"claude_code_config_import",
|
||||
"claude_code_home_show",
|
||||
"claude_code_home_remove",
|
||||
"claude_code_home_removed",
|
||||
"claude_code_home_restore",
|
||||
"claude_code_judge",
|
||||
"claude_code_judged"
|
||||
],
|
||||
"data": {
|
||||
"own": [
|
||||
{
|
||||
"id": "agent-home",
|
||||
"path": "${dir:agent-home}",
|
||||
"class": "valuable",
|
||||
"why": "the operator's agent's own files: its memory, history and projects"
|
||||
}
|
||||
]
|
||||
},
|
||||
"config": {
|
||||
"role": {
|
||||
"type": "string",
|
||||
"description": "What this machine is for, in a sentence, written into the instructions every Claude Code session on it reads; empty says nothing.",
|
||||
"trusted": true,
|
||||
"default": ""
|
||||
},
|
||||
"mcp_servers": {
|
||||
"type": "object",
|
||||
"description": "The tool servers every session on this machine is given beside the mesh's own, by name, each as Claude Code's managed MCP configuration takes it; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"managed_settings": {
|
||||
"type": "object",
|
||||
"description": "Keys of Claude Code's managed settings every session on this machine obeys, such as spinnerTipsEnabled, laid over the mesh's own; empty adds none. Falls back to agent-managed-settings, the mesh's or this node's.",
|
||||
"trusted": true,
|
||||
"predefined": "agent-managed-settings",
|
||||
"default": {}
|
||||
},
|
||||
"instructions": {
|
||||
"type": "object",
|
||||
"description": "The instructions every session on this machine reads, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"output_styles": {
|
||||
"type": "object",
|
||||
"description": "The output styles offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"skills": {
|
||||
"type": "object",
|
||||
"description": "The skills offered to every session on this machine, by name, each its files by path, a text as one line or a list of its lines; a skill without its SKILL.md is left out; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"commands": {
|
||||
"type": "object",
|
||||
"description": "The slash commands offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"agents": {
|
||||
"type": "object",
|
||||
"description": "The agents offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "claude-code"
|
||||
},
|
||||
{
|
||||
"id": "managed",
|
||||
"type": "directory",
|
||||
"path": "/etc/claude-code",
|
||||
"mode": "0755",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "start",
|
||||
"type": "file",
|
||||
"path": "/usr/local/bin/claude-agent",
|
||||
"mode": "0755",
|
||||
"template": "files/claude-agent"
|
||||
},
|
||||
{
|
||||
"id": "agent-account-name",
|
||||
"type": "file",
|
||||
"path": "/etc/claude-code/agent-account",
|
||||
"mode": "0644",
|
||||
"template": "files/agent-account"
|
||||
},
|
||||
{
|
||||
"id": "agent-account",
|
||||
"type": "user",
|
||||
"name": "${machine:agent-account}",
|
||||
"home": "${machine:agent-home}",
|
||||
"root": "${machine:agent-root}"
|
||||
},
|
||||
{
|
||||
"id": "agent-home",
|
||||
"type": "directory",
|
||||
"path": "${machine:agent-home}/.claude",
|
||||
"mode": "0700",
|
||||
"owner": "${machine:agent-account}",
|
||||
"unknown": "module"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "${machine:account}",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "facts",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/facts.json",
|
||||
"mode": "0600",
|
||||
"owner": "${machine:account}",
|
||||
"template": "files/facts.json"
|
||||
},
|
||||
{
|
||||
"id": "settings",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/settings.json",
|
||||
"mode": "0600",
|
||||
"owner": "${machine:account}",
|
||||
"takes-config": ["role", "mcp_servers", "managed_settings", "instructions", "output_styles", "skills", "commands", "agents"],
|
||||
"merge": "json",
|
||||
"template": "files/settings.json"
|
||||
}
|
||||
],
|
||||
"items": [
|
||||
{
|
||||
"collection": "shell:zsh:normal",
|
||||
"template": "files/zsh-normal"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/claude-code",
|
||||
"binary": "claude-code",
|
||||
"loads": [
|
||||
"claude-code"
|
||||
],
|
||||
"env": {
|
||||
"MESH_CLAUDE_CODE_STATE": "${dir:state}",
|
||||
"MESH_CLAUDE_CODE_FACTS": "${dir:state}/facts.json",
|
||||
"MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
`
|
||||
@@ -23,11 +23,6 @@ var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
|
||||
const ForgeAlias = "mesh-forge"
|
||||
|
||||
// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it
|
||||
// fetches and every round it checks its toolchain. A workspace lies in the module's work directory, never in its
|
||||
// home (ADR 0466 §1).
|
||||
const WorkspaceAge = 24 * time.Hour
|
||||
|
||||
var (
|
||||
repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
workspaceOK = regexp.MustCompile(`^[a-z0-9]+$`)
|
||||
@@ -82,20 +77,37 @@ func (m *Module) fetchTool() StepTool {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
// fetch makes a workspace and fills it. A workspace lies in the module's work directory, never in its home (ADR 0466
|
||||
// §1); it is held while the fetch runs, and removed when the fetch fails, since nobody learns its id to release it
|
||||
// (issue 1839).
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
|
||||
// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
|
||||
// named paths, the workspace is narrowed to them (buildstep.Narrow).
|
||||
req, err := buildstep.ParseFetchRequest(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
||||
siblings, err := siblingsArg(args["beside"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
id := buildstep.NewID()
|
||||
done := m.spaces().Hold(id)
|
||||
defer func() {
|
||||
done()
|
||||
if err != nil {
|
||||
if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
|
||||
m.Say("the failed fetch's workspace %s: %v", id, rerr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return m.fetchInto(ctx, req, id, siblings)
|
||||
}
|
||||
|
||||
func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id string, siblings []siblingArg) (any, error) {
|
||||
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
||||
var err error
|
||||
root := filepath.Join(m.Work, id)
|
||||
tree := filepath.Join(root, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
@@ -236,21 +248,10 @@ func (m *Module) goEnvFor(dir string) []string {
|
||||
return env
|
||||
}
|
||||
|
||||
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
|
||||
// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
|
||||
// net for a caller that never released one (issue 1839, ADR 0509).
|
||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||
entries, err := os.ReadDir(m.Work)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
|
||||
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
|
||||
}
|
||||
}
|
||||
m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
|
||||
}
|
||||
|
||||
// clock is the time now: Now when a test set it, else the wall clock.
|
||||
|
||||
@@ -265,10 +265,11 @@ func TestARepositoryThatIsNotOwnerSlashNameIsRefusedBeforeAnythingRuns(t *testin
|
||||
}
|
||||
}
|
||||
|
||||
// Workspaces older than a day are removed; younger ones, and anything that is not a workspace, stay.
|
||||
func TestWorkspacesOlderThanADayAreRemoved(t *testing.T) {
|
||||
// Unheld workspaces last used more than two hours ago are removed (issue 1839, ADR 0509); younger ones, and anything
|
||||
// that is not a workspace, stay.
|
||||
func TestUnheldWorkspacesPastTwoHoursAreRemoved(t *testing.T) {
|
||||
m := &Module{Work: t.TempDir()}
|
||||
for name, age := range map[string]time.Duration{"old1": 25 * time.Hour, "young": time.Hour, "Keep-This": 48 * time.Hour} {
|
||||
for name, age := range map[string]time.Duration{"old1": 3 * time.Hour, "young": time.Hour, "Keep-This": 48 * time.Hour} {
|
||||
dir := filepath.Join(m.Work, name)
|
||||
os.MkdirAll(filepath.Join(dir, "tree"), 0o755)
|
||||
when := time.Now().Add(-age)
|
||||
|
||||
@@ -68,6 +68,7 @@ func (m *Module) integrationShutOff(ctx context.Context, workspace string, packa
|
||||
}
|
||||
iso := m.isolation(workspace, false)
|
||||
iso.ReadOnly = append(iso.ReadOnly, exe) // the program alone, not the directory it lies in
|
||||
iso.ReadOnly = append(iso.ReadOnly, m.nodeToolchains()...)
|
||||
r := buildstep.Isolated(m.Run, iso)(ctx, nil, exe, levelArg, string(spec))
|
||||
var answer levelAnswer
|
||||
line := lastLine(r.Stdout)
|
||||
@@ -80,6 +81,25 @@ func (m *Module) integrationShutOff(ctx context.Context, workspace string, packa
|
||||
return answer.Result, nil
|
||||
}
|
||||
|
||||
// NodeToolchains is the Node build module's toolchains directory on this node: its cache directory (module.json
|
||||
// resource `cache`, mode 0711) joined with toolchain, where it installs typescript, esbuild, @novox/mesh-sdk and nats
|
||||
// from its own lock (ADR 0533 §1, §2; issue 1891).
|
||||
const NodeToolchains = "/var/lib/mesh-build-node-cache/toolchain"
|
||||
|
||||
// nodeToolchains is what the integration level is given of the Node build module's toolchains: their directory,
|
||||
// read-only, when it is there and this account may enter it, and nothing on a node where the Node build module has
|
||||
// installed none (ADR 0533 §1). The unit level is never given it.
|
||||
func (m *Module) nodeToolchains() []string {
|
||||
dir := m.NodeToolchains
|
||||
if dir == "" {
|
||||
dir = NodeToolchains
|
||||
}
|
||||
if _, err := os.ReadDir(dir); err != nil {
|
||||
return nil
|
||||
}
|
||||
return []string{dir}
|
||||
}
|
||||
|
||||
func lastLine(s string) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
return strings.TrimSpace(lines[len(lines)-1])
|
||||
|
||||
@@ -46,6 +46,7 @@ func main() {
|
||||
Work: WorkDir,
|
||||
Cache: CacheDir,
|
||||
}
|
||||
m.Floor = floorOf(m.Node)
|
||||
if u, err := user.Current(); err == nil {
|
||||
m.Home = u.HomeDir
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
@@ -62,9 +63,18 @@ type Module struct {
|
||||
Push func(ctx context.Context, registry, repository, digest string, body []byte) error
|
||||
// Executable is this program's path, which the integration level starts again shut off; os.Executable when empty.
|
||||
Executable string
|
||||
// NodeToolchains is where the Node build module keeps its installed toolchains, bound read-only into the
|
||||
// integration level (ADR 0533 §1); NodeToolchains when empty. A test points it elsewhere.
|
||||
NodeToolchains string
|
||||
// StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own
|
||||
// program, StartBus in bus.go, when nil).
|
||||
StartBus func(ctx context.Context, dir string) (url string, stop func(), err error)
|
||||
// Floor refuses every step while the work or cache filesystem has less free than it (issue 1839); main sets
|
||||
// floorOf(node), and a zero Floor checks nothing.
|
||||
Floor buildstep.Floor
|
||||
|
||||
spacesOnce sync.Once
|
||||
workspaces *buildstep.Workspaces
|
||||
}
|
||||
|
||||
// guard is the SDK guard for the version the probe just read: the module names no Go version (ADR 0477 §2), so the
|
||||
@@ -113,6 +123,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
|
||||
if err := m.checkBubblewrap(ctx); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("%s installed, the version the module builds with (recipe %s)", installed, m.Recipe.Version()))
|
||||
}
|
||||
|
||||
@@ -125,6 +138,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string
|
||||
if err := m.Check(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer m.holdArg(args)()
|
||||
return run(ctx, args)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -141,6 +158,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any,
|
||||
if err := m.Check(m.Ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer m.holdArg(args)()
|
||||
return run(m.Ctx, args)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,7 +115,7 @@ func TestAGoThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {
|
||||
{Name: "short_probe", Run: run},
|
||||
}, m.StepTools()...)
|
||||
for _, tool := range m.Tools(steps) {
|
||||
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool {
|
||||
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool {
|
||||
continue
|
||||
}
|
||||
var reason string
|
||||
|
||||
@@ -13,7 +13,8 @@ import (
|
||||
|
||||
// ADR 0499 §4 (issue 1758): a fetch asked with the module's directory and its named paths leaves the workspace holding
|
||||
// only those, the manifest and the lock files; the rest of the commit is gone before any run step; a request naming
|
||||
// paths without a directory is refused before anything is fetched.
|
||||
// paths without a directory is refused before anything is fetched. go.mod is kept only when named among the paths
|
||||
// (mesh-sdk's narrowKept is the manifest and the lock files, review finding S1), as a Go build that names its paths does.
|
||||
func TestAFetchWithNamedPathsNarrowsTheWorkspace(t *testing.T) {
|
||||
f := newForge(t)
|
||||
_, head := f.repo("novox/narrow", map[string]string{
|
||||
@@ -26,7 +27,7 @@ func TestAFetchWithNamedPathsNarrowsTheWorkspace(t *testing.T) {
|
||||
"other/o.go": "package other\n",
|
||||
})
|
||||
m, states := f.module(t)
|
||||
res, st := fetched(t, m, states, map[string]any{"repository": "novox/narrow", "commit": head, "dir": "lib", "paths": []any{"cmd"}})
|
||||
res, st := fetched(t, m, states, map[string]any{"repository": "novox/narrow", "commit": head, "dir": "lib", "paths": []any{"cmd", "go.mod"}})
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the fetch ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// integrationShutOffWith runs the integration level through the fake bubblewrap and answers what its one shut-off
|
||||
// process was given.
|
||||
func integrationShutOffWith(t *testing.T, toolchains string) shutOff {
|
||||
t.Helper()
|
||||
m, states, ws, _ := workspaceFor(t, "failing")
|
||||
m.Executable = "/opt/bundle/mesh-build-go"
|
||||
m.NodeToolchains = toolchains
|
||||
want := TestResult{Level: LevelIntegration, Passed: true, Tests: 1, Packages: []PackageResult{}, Store: "postgres 18"}
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch {
|
||||
case name == "postgres" && len(args) == 1 && args[0] == "--version":
|
||||
return forgekey.Ran{Stdout: "postgres (PostgreSQL) 18.6\n"}
|
||||
case name == m.Executable && len(args) == 2 && args[0] == levelArg:
|
||||
raw, _ := json.Marshal(levelAnswer{Result: &want})
|
||||
return forgekey.Ran{Stdout: string(raw) + "\n"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1, Err: "not expected here"}
|
||||
}, &seen)
|
||||
run := stepTool(t, m, "build_go_test")
|
||||
if _, err := run(map[string]any{"workspace": ws, "level": LevelIntegration}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if len(seen) != 1 {
|
||||
t.Fatalf("shut off %d processes, want one: %v", len(seen), seen)
|
||||
}
|
||||
checkShutOff(t, m, seen[0], false)
|
||||
return seen[0]
|
||||
}
|
||||
|
||||
// 1891 (ADR 0533 §1): the integration level is given the Node build module's installed toolchains, read-only, so the
|
||||
// Node build module's own integration tests build with the real tsc and esbuild and reach no registry.
|
||||
func TestTheIntegrationLevelIsGivenTheNodeToolchainsReadOnly(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
s := integrationShutOffWith(t, dir)
|
||||
if !slices.Contains(s.roBinds, dir) {
|
||||
t.Errorf("the node toolchains %s are not bound read-only: %v", dir, s.roBinds)
|
||||
}
|
||||
if slices.Contains(s.binds, dir) || slices.Contains(s.overlays, dir) {
|
||||
t.Errorf("the node toolchains %s may be written: binds %v, overlays %v", dir, s.binds, s.overlays)
|
||||
}
|
||||
if slices.Contains(s.roBinds, filepath.Dir(dir)) {
|
||||
t.Errorf("the whole cache %s is bound, not its toolchains alone: %v", filepath.Dir(dir), s.roBinds)
|
||||
}
|
||||
}
|
||||
|
||||
// 1891 (ADR 0533 §1): a node where the Node build module has installed no toolchain runs the level as before, with
|
||||
// nothing more bound (bubblewrap binds only what is there).
|
||||
func TestANodeWithoutNodeToolchainsRunsTheLevelWithNothingMoreBound(t *testing.T) {
|
||||
missing := filepath.Join(t.TempDir(), "none")
|
||||
s := integrationShutOffWith(t, missing)
|
||||
if slices.Contains(s.roBinds, missing) {
|
||||
t.Errorf("a missing %s is bound: %v", missing, s.roBinds)
|
||||
}
|
||||
}
|
||||
|
||||
// 1891 (ADR 0533 §1): the unit level is not given them.
|
||||
func TestTheUnitLevelIsNotGivenTheNodeToolchains(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "failing")
|
||||
dir := t.TempDir()
|
||||
m.NodeToolchains = dir
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, m.Run, &seen)
|
||||
if _, st := runUnit(t, m, states, ws); st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
for _, s := range seen {
|
||||
if slices.Contains(s.roBinds, dir) || slices.Contains(s.binds, dir) {
|
||||
t.Errorf("%s was given the node toolchains", s.command)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1891 (ADR 0533 §1, §2): the place bound is the Node build module's own: its manifest's cache directory joined with
|
||||
// toolchain, and that cache directory lets other accounts enter it without listing it (0711).
|
||||
func TestTheNodeToolchainsAreTheNodeBuildModulesCache(t *testing.T) {
|
||||
if _, err := catalogue.ReadTreeManifest(catalogueRoot, filepath.Join(catalogueRoot, "modules", "mesh-build-node")); err != nil {
|
||||
t.Fatalf("mesh-sdk's validator refuses mesh-build-node's manifest: %v", err)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(catalogueRoot, "modules", "mesh-build-node", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var man struct {
|
||||
Resources []struct {
|
||||
ID, Path, Mode string
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &man); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, r := range man.Resources {
|
||||
if r.ID != "cache" {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if want := filepath.Join(r.Path, "toolchain"); NodeToolchains != want {
|
||||
t.Errorf("NodeToolchains = %s, want %s (mesh-build-node's cache)", NodeToolchains, want)
|
||||
}
|
||||
mode, err := strconv.ParseUint(r.Mode, 8, 32)
|
||||
if err != nil || mode&0o001 == 0 || mode&0o004 != 0 {
|
||||
t.Errorf("mesh-build-node's cache has mode %q; want others to enter it and not list it (0711)", r.Mode)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("mesh-build-node's manifest declares no cache directory")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every step is refused while the work
|
||||
// or cache filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases
|
||||
// it, and swept unheld past buildstep.UnheldBound.
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// ReleaseTool removes a workspace whose caller's steps have ended.
|
||||
const ReleaseTool = "build_go_release"
|
||||
|
||||
// floorOf is the floor the module keeps on node: build-agent's.
|
||||
func floorOf(node string) buildstep.Floor {
|
||||
return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor}
|
||||
}
|
||||
|
||||
// room refuses while the work or cache filesystem is under the floor.
|
||||
func (m *Module) room() error { return m.Floor.Check(m.Work, m.Cache) }
|
||||
|
||||
// spaces are the module's workspaces under Work.
|
||||
func (m *Module) spaces() *buildstep.Workspaces {
|
||||
m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} })
|
||||
return m.workspaces
|
||||
}
|
||||
|
||||
// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none.
|
||||
func (m *Module) holdArg(args map[string]any) func() {
|
||||
ws, _ := args["workspace"].(string)
|
||||
if ws == "" {
|
||||
return func() {}
|
||||
}
|
||||
return m.spaces().Hold(ws)
|
||||
}
|
||||
|
||||
// releaseTool is build_go_release: no guard and no floor, since it only frees room.
|
||||
func (m *Module) releaseTool() stdio.Tool {
|
||||
return stdio.Tool{Name: ReleaseTool,
|
||||
Description: "Remove a workspace whose steps have ended (fetch, test, build): its caller says so once its last step " +
|
||||
"answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " +
|
||||
"is removed two hours after its last use.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"},
|
||||
}, "required": []string{"workspace"}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ws, _ := args["workspace"].(string)
|
||||
if err := m.spaces().Release(ws); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"released": ws}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509).
|
||||
func (m *Module) sweepSaid(removed []string) {
|
||||
if len(removed) > 0 && m.Say != nil {
|
||||
m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
|
||||
|
||||
func underFloor(node string) buildstep.Floor {
|
||||
f := floorOf(node)
|
||||
f.Free = func(string) (uint64, error) { return 4 << 30, nil }
|
||||
return f
|
||||
}
|
||||
|
||||
// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
|
||||
func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
|
||||
if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
|
||||
t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC1: under the floor every step tool is refused, naming the node, what is free and the floor, no step runs, and
|
||||
// the module is unhealthy saying so, so the builder chooses another node.
|
||||
func TestUnderTheFloorEveryStepIsRefusedAndTheModuleIsUnhealthy(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.27.1")
|
||||
m.Work, m.Cache, m.Floor = t.TempDir(), t.TempDir(), underFloor("ace")
|
||||
ran := false
|
||||
run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil }
|
||||
steps := []StepTool{{Name: "long_probe", Long: true, Run: run}, {Name: "short_probe", Run: run}}
|
||||
for _, tool := range m.Tools(steps) {
|
||||
switch tool.Name {
|
||||
case "long_probe":
|
||||
if _, err := tool.Run(map[string]any{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
|
||||
t.Errorf("the long step ended %s: %q, want refused: %q", st.State, st.Reason, underTheFloor)
|
||||
}
|
||||
case "short_probe":
|
||||
if _, err := tool.Run(map[string]any{}); err == nil || !strings.HasPrefix(err.Error(), underTheFloor) {
|
||||
t.Errorf("the short step answered %v, want %q", err, underTheFloor)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ran {
|
||||
t.Errorf("a step ran under the floor")
|
||||
}
|
||||
if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
|
||||
t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
|
||||
}
|
||||
}
|
||||
|
||||
func releaseTool(t *testing.T, m *Module) func(map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == ReleaseTool {
|
||||
return tool.Run
|
||||
}
|
||||
}
|
||||
t.Fatalf("no tool %s", ReleaseTool)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1839/AC2: build_go_release removes the workspace at once, even under the floor; a second release is no error.
|
||||
func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
|
||||
m, _ := fetchModule(t, &fakeStep{})
|
||||
m.Floor = underFloor("ace")
|
||||
ws := filepath.Join(m.Work, "abc1")
|
||||
os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, err := releaseTool(t, m)(map[string]any{"workspace": "abc1"}); err != nil {
|
||||
t.Fatalf("release %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(ws); !os.IsNotExist(err) {
|
||||
t.Errorf("the workspace is still there after its release")
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
|
||||
func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n", goRule: func(string) forgekey.Ran { return forgekey.Ran{Status: 1, Stderr: "boom"} }}
|
||||
m, states := fetchModule(t, f)
|
||||
if _, err := fetchTool(t, m)(fetchArgs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepFailed {
|
||||
t.Fatalf("the step ended %s, want failed", st.State)
|
||||
}
|
||||
if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
|
||||
t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is.
|
||||
func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
|
||||
m := &Module{Work: t.TempDir()}
|
||||
for _, name := range []string{"held1", "lost1"} {
|
||||
dir := filepath.Join(m.Work, name)
|
||||
os.MkdirAll(filepath.Join(dir, "tree"), 0o755)
|
||||
}
|
||||
done := m.spaces().Hold("held1")
|
||||
defer done()
|
||||
when := time.Now().Add(-3 * time.Hour)
|
||||
for _, name := range []string{"held1", "lost1"} {
|
||||
os.Chtimes(filepath.Join(m.Work, name), when, when)
|
||||
}
|
||||
m.CleanWorkspaces(time.Now())
|
||||
if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
|
||||
t.Errorf("the held workspace was swept")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
|
||||
t.Errorf("the unheld workspace three hours old was kept")
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
||||
tools = append(tools, t)
|
||||
}
|
||||
tools = append(tools, m.ReadTools()...)
|
||||
tools = append(tools, m.releaseTool())
|
||||
return append(tools,
|
||||
stdio.Tool{Name: StatusTool,
|
||||
Description: "What the Go build module is on this machine: the Go version its recipe builds with and the one installed, " +
|
||||
|
||||
@@ -3,7 +3,7 @@ module meshbuildgo
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467
|
||||
github.com/distribution/distribution/v3 v3.1.2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52 h1:5k04XjpfYCq6j7rCM6Dy6FfrsBk5iUJlbwGf6WhUOUc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
"build_go_test",
|
||||
"build_go_digest",
|
||||
"build_go_build",
|
||||
"build_go_release",
|
||||
"build_go_step",
|
||||
"build_go_steps",
|
||||
"build_go_toolchains",
|
||||
|
||||
@@ -30,9 +30,19 @@ The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can re
|
||||
`<module>/<artifact>:<tag of its own>` (the first 12 hex digits of the digest unless the call gives one), waiting out
|
||||
a registry held still, and the key carries the digest the registry serves and the reference pinned by it. Without
|
||||
`publish` (a pull request) nothing is pushed.
|
||||
- `build_image_copy` copies an upstream artifact (`module`, `artifact`, `from`) registry to registry into the artifact store
|
||||
over the registry API, the whole index and every blob, by the digest the manifest pins (ADR 0498 §4), and verifies the
|
||||
store serves that digest. Answers the digest and the reference `<registry>/<module>/<artifact>@sha256:<digest>`. A
|
||||
`from` without a digest fails `not pinned:`. It needs no workspace and no buildah.
|
||||
- It refuses, naming why, an image whose build context is a whole repository (`context`) or whose `build.on` names
|
||||
another module's artifact (`unsupportedImage` in `build.go`): those forms belong to the images build-agent builds
|
||||
until it is removed at R4g.
|
||||
- `build_image_release {workspace}` removes a workspace at once (no guard, no floor; one already gone is no error, one a
|
||||
running step holds is refused); its caller calls it once its last step answered. A failed fetch removes its own
|
||||
workspace, and one no caller released is removed two hours after its last use (ADR 0509).
|
||||
- A fetch and a build are refused, `refused: <node> has N GiB free under the floor of 20 GiB (on <dir>)`, and the health
|
||||
is unhealthy with that text, while the work directory's or the image store's filesystem has less than 20 GiB free
|
||||
(issue 1839). A copy writes nothing to the disk and is not refused.
|
||||
- `build_image_step`, `build_image_steps`, `build_image_status` and `build_image_health` read.
|
||||
|
||||
The registry's address is placed into `<home>/registry` as `${seat:mesh-artifact-store:reach}` (host:port): the provisioner's client fills a moved module's placement, and the artifact store has not moved, so a `${bound:artifact-store:...}` is refused at composition (issue 1703).
|
||||
|
||||
@@ -74,6 +74,23 @@ func unsupportedImage(manifest catalogue.Manifest, a catalogue.Artifact) string
|
||||
return ""
|
||||
}
|
||||
|
||||
// qualifyImage names a base image in full, as the controller's parseReference reads it: a name without a registry host
|
||||
// is the public hub's (docker.io), a single name under `library/`. buildah, with this machine's registry configuration,
|
||||
// has no unqualified-search registry and refuses a short name (issue 1824); a name that has a host is kept.
|
||||
func qualifyImage(ref string) string {
|
||||
name := ref
|
||||
if at := strings.Index(name, "@"); at >= 0 {
|
||||
name = name[:at]
|
||||
}
|
||||
if slash := strings.Index(name, "/"); slash >= 0 && (strings.ContainsAny(name[:slash], ".:") || name[:slash] == "localhost") {
|
||||
return ref
|
||||
}
|
||||
if !strings.Contains(name, "/") {
|
||||
return "docker.io/library/" + ref
|
||||
}
|
||||
return "docker.io/" + ref
|
||||
}
|
||||
|
||||
// imageArgs are the --build-arg words of the images the manifest pins in build.on, in its order.
|
||||
func imageArgs(manifest catalogue.Manifest) []string {
|
||||
var out []string
|
||||
@@ -82,7 +99,7 @@ func imageArgs(manifest catalogue.Manifest) []string {
|
||||
}
|
||||
for _, on := range manifest.Build.On {
|
||||
if on.Image != "" && on.Arg != "" {
|
||||
out = append(out, "--build-arg", on.Arg+"="+on.Image)
|
||||
out = append(out, "--build-arg", on.Arg+"="+qualifyImage(on.Image))
|
||||
}
|
||||
}
|
||||
return out
|
||||
@@ -109,7 +126,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no module in %q of workspace %s: %w", module, workspace, err)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifest(raw)
|
||||
manifest, err := catalogue.ParseManifestForInputs(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the manifest in %q: %w", module, err)
|
||||
}
|
||||
|
||||
@@ -198,3 +198,53 @@ func TestBuildFailsForWhatIsNotAnImageOrNotThere(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1751 (ADR 0514 §2, the review's K2): a manifest whose config value falls back to a predefined key builds. The
|
||||
// builder holds no published list of the keys, so it reads the manifest for its inputs (ParseManifestForInputs), not
|
||||
// as a judge of config values. The manifest is claude-code's real one (capturedClaudeCodeManifest) with the image fixture's
|
||||
// base and artifact added to its build, since no image module of the catalogue falls back to a predefined key today.
|
||||
func TestAManifestFallingBackToAPredefinedKeyBuilds(t *testing.T) {
|
||||
raw := []byte(capturedClaudeCodeManifest)
|
||||
if !strings.Contains(string(raw), `"predefined": "agent-managed-settings"`) {
|
||||
t.Fatal("the captured manifest no longer falls back to a predefined key")
|
||||
}
|
||||
m, _, states := setup(t, func(fixture string) string {
|
||||
var real, image map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &real); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(fixture), &image); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The image fixture's base and artifact join claude-code's own artifacts (its archive of files/ stays).
|
||||
var realBuild, imageBuild struct {
|
||||
On json.RawMessage `json:"on,omitempty"`
|
||||
Artifacts []json.RawMessage `json:"artifacts"`
|
||||
}
|
||||
if err := json.Unmarshal(real["build"], &realBuild); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(image["build"], &imageBuild); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
realBuild.On = imageBuild.On
|
||||
realBuild.Artifacts = append(realBuild.Artifacts, imageBuild.Artifacts...)
|
||||
b, err := json.Marshal(realBuild)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
real["build"] = b
|
||||
out, err := json.Marshal(real)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(out)
|
||||
})
|
||||
res, st := runBuild(t, m, states, nil)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
if res.Digest != imageDigest || res.Artifact != "server" {
|
||||
t.Errorf("result = %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
package main
|
||||
|
||||
// capturedClaudeCodeManifest is modules/claude-code/module.json of this catalogue at 56c68be7, copied verbatim on
|
||||
// 2026-10-22 for issue 1751 (the R4e review's finding K2): a real manifest whose config value managed_settings falls
|
||||
// back to the predefined key agent-managed-settings (ADR 0348 §7), which the builder holds no published list for. It
|
||||
// lives in a test file, not under testdata, because the catalogue's paths check (ADR 0260) reads every other file of a
|
||||
// module and the manifest names claude-code's own paths.
|
||||
const capturedClaudeCodeManifest = `{
|
||||
"module": "claude-code",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"slug": "agent",
|
||||
"capabilities": [
|
||||
"package-manager"
|
||||
],
|
||||
"requires": [
|
||||
"mcp-endpoint",
|
||||
{
|
||||
"name": "gitea-read",
|
||||
"optional": true
|
||||
}
|
||||
],
|
||||
"binds": {
|
||||
"mcp-endpoint": "${dir:state}/mcp-endpoint.json"
|
||||
},
|
||||
"secrets": {
|
||||
"gitea-read": "${dir:state}/gitea-read.secret"
|
||||
},
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
],
|
||||
"state": [
|
||||
"servers",
|
||||
"holdings",
|
||||
"config",
|
||||
"proposals",
|
||||
"questions",
|
||||
"memory",
|
||||
"waits"
|
||||
],
|
||||
"reads": [
|
||||
"claude-licence-manager.bindings"
|
||||
],
|
||||
"tools": [
|
||||
"claude_code_status",
|
||||
"claude_code_render",
|
||||
"claude_code_guard",
|
||||
"claude_code_pull",
|
||||
"claude_code_grant",
|
||||
"claude_code_add_api_key",
|
||||
"claude_code_registrations",
|
||||
"claude_code_memory",
|
||||
"claude_code_mcp_list",
|
||||
"claude_code_mcp_register",
|
||||
"claude_code_mcp_unregister",
|
||||
"claude_code_skill_list",
|
||||
"claude_code_skill_register",
|
||||
"claude_code_skill_unregister",
|
||||
"claude_code_agent_list",
|
||||
"claude_code_agent_register",
|
||||
"claude_code_agent_unregister",
|
||||
"claude_code_command_list",
|
||||
"claude_code_command_register",
|
||||
"claude_code_command_unregister",
|
||||
"claude_code_hook_list",
|
||||
"claude_code_hook_register",
|
||||
"claude_code_hook_unregister",
|
||||
"claude_code_output_style_list",
|
||||
"claude_code_output_style_register",
|
||||
"claude_code_output_style_unregister",
|
||||
"claude_code_instructions_list",
|
||||
"claude_code_instructions_register",
|
||||
"claude_code_instructions_unregister",
|
||||
"claude_code_instructions_propose",
|
||||
"claude_code_proposals",
|
||||
"claude_code_ask",
|
||||
"claude_code_answer",
|
||||
"claude_code_wait",
|
||||
"claude_code_settings_get",
|
||||
"claude_code_settings_set",
|
||||
"claude_code_settings_clear",
|
||||
"claude_code_permission_add",
|
||||
"claude_code_permission_remove",
|
||||
"claude_code_config_list",
|
||||
"claude_code_config_show",
|
||||
"claude_code_config_status",
|
||||
"claude_code_config_import",
|
||||
"claude_code_home_show",
|
||||
"claude_code_home_remove",
|
||||
"claude_code_home_removed",
|
||||
"claude_code_home_restore",
|
||||
"claude_code_judge",
|
||||
"claude_code_judged"
|
||||
],
|
||||
"data": {
|
||||
"own": [
|
||||
{
|
||||
"id": "agent-home",
|
||||
"path": "${dir:agent-home}",
|
||||
"class": "valuable",
|
||||
"why": "the operator's agent's own files: its memory, history and projects"
|
||||
}
|
||||
]
|
||||
},
|
||||
"config": {
|
||||
"role": {
|
||||
"type": "string",
|
||||
"description": "What this machine is for, in a sentence, written into the instructions every Claude Code session on it reads; empty says nothing.",
|
||||
"trusted": true,
|
||||
"default": ""
|
||||
},
|
||||
"mcp_servers": {
|
||||
"type": "object",
|
||||
"description": "The tool servers every session on this machine is given beside the mesh's own, by name, each as Claude Code's managed MCP configuration takes it; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"managed_settings": {
|
||||
"type": "object",
|
||||
"description": "Keys of Claude Code's managed settings every session on this machine obeys, such as spinnerTipsEnabled, laid over the mesh's own; empty adds none. Falls back to agent-managed-settings, the mesh's or this node's.",
|
||||
"trusted": true,
|
||||
"predefined": "agent-managed-settings",
|
||||
"default": {}
|
||||
},
|
||||
"instructions": {
|
||||
"type": "object",
|
||||
"description": "The instructions every session on this machine reads, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"output_styles": {
|
||||
"type": "object",
|
||||
"description": "The output styles offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"skills": {
|
||||
"type": "object",
|
||||
"description": "The skills offered to every session on this machine, by name, each its files by path, a text as one line or a list of its lines; a skill without its SKILL.md is left out; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"commands": {
|
||||
"type": "object",
|
||||
"description": "The slash commands offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"agents": {
|
||||
"type": "object",
|
||||
"description": "The agents offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "claude-code"
|
||||
},
|
||||
{
|
||||
"id": "managed",
|
||||
"type": "directory",
|
||||
"path": "/etc/claude-code",
|
||||
"mode": "0755",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "start",
|
||||
"type": "file",
|
||||
"path": "/usr/local/bin/claude-agent",
|
||||
"mode": "0755",
|
||||
"template": "files/claude-agent"
|
||||
},
|
||||
{
|
||||
"id": "agent-account-name",
|
||||
"type": "file",
|
||||
"path": "/etc/claude-code/agent-account",
|
||||
"mode": "0644",
|
||||
"template": "files/agent-account"
|
||||
},
|
||||
{
|
||||
"id": "agent-account",
|
||||
"type": "user",
|
||||
"name": "${machine:agent-account}",
|
||||
"home": "${machine:agent-home}",
|
||||
"root": "${machine:agent-root}"
|
||||
},
|
||||
{
|
||||
"id": "agent-home",
|
||||
"type": "directory",
|
||||
"path": "${machine:agent-home}/.claude",
|
||||
"mode": "0700",
|
||||
"owner": "${machine:agent-account}",
|
||||
"unknown": "module"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "${machine:account}",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "facts",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/facts.json",
|
||||
"mode": "0600",
|
||||
"owner": "${machine:account}",
|
||||
"template": "files/facts.json"
|
||||
},
|
||||
{
|
||||
"id": "settings",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/settings.json",
|
||||
"mode": "0600",
|
||||
"owner": "${machine:account}",
|
||||
"takes-config": ["role", "mcp_servers", "managed_settings", "instructions", "output_styles", "skills", "commands", "agents"],
|
||||
"merge": "json",
|
||||
"template": "files/settings.json"
|
||||
}
|
||||
],
|
||||
"items": [
|
||||
{
|
||||
"collection": "shell:zsh:normal",
|
||||
"template": "files/zsh-normal"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/claude-code",
|
||||
"binary": "claude-code",
|
||||
"loads": [
|
||||
"claude-code"
|
||||
],
|
||||
"env": {
|
||||
"MESH_CLAUDE_CODE_STATE": "${dir:state}",
|
||||
"MESH_CLAUDE_CODE_FACTS": "${dir:state}/facts.json",
|
||||
"MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
`
|
||||
@@ -0,0 +1,374 @@
|
||||
package main
|
||||
|
||||
// The copy step (novox/hq ADR 0498 §4; issue 1587): an upstream artifact is copied registry to registry into the
|
||||
// artifact store, whole (an index with every manifest it lists, every blob), by the digest its manifest pins. The copy
|
||||
// runs no code of a change, so it needs neither buildah nor an isolation; it is made over the registry API, which
|
||||
// keeps every digest as it is (a pull and a push through buildah may rewrite a manifest). An upstream without a
|
||||
// digest is not copied: `not pinned:`. A copy that does not end with the pinned digest in the store is `not stored:`.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// CopyTool is the copy step's tool.
|
||||
const CopyTool = "build_image_copy"
|
||||
|
||||
// CopyResult is what a passed copy step says: the pinned digest, now in the store, and the reference to it in the
|
||||
// form an image's is (<registry>/<module>/<artifact>@sha256:<digest>).
|
||||
type CopyResult struct {
|
||||
Artifact string `json:"artifact"`
|
||||
Digest string `json:"digest"`
|
||||
Reference string `json:"reference"`
|
||||
Pushed bool `json:"pushed"`
|
||||
}
|
||||
|
||||
var (
|
||||
nameOK = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*(/[a-z0-9][a-z0-9._-]*)*$`)
|
||||
pinnedOK = regexp.MustCompile(`@(sha256:[0-9a-f]{64})$`)
|
||||
manifestKind = strings.Join([]string{
|
||||
"application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
"application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json"}, ", ")
|
||||
)
|
||||
|
||||
func (m *Module) copyTool() StepTool {
|
||||
return StepTool{
|
||||
Name: CopyTool,
|
||||
Description: "Copy an upstream image, registry to registry, into the artifact store by the digest the manifest pins " +
|
||||
"(novox/hq ADR 0498 §4). Answers a step id at once; the step ends on its key with the digest in the store and the " +
|
||||
"reference to it. A reference without a digest fails `not pinned:`. Needs no workspace and runs no code of a change.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module that names the upstream artifact"},
|
||||
"artifact": map[string]any{"type": "string", "description": "the upstream artifact's name in the module's manifest"},
|
||||
"from": map[string]any{"type": "string", "description": "the image reference the manifest gives, pinned by digest, such as nats:2.11.17-alpine@sha256:..."},
|
||||
},
|
||||
"required": []string{"module", "artifact", "from"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.copy,
|
||||
}
|
||||
}
|
||||
|
||||
// upstreamRef is an image reference split the way a container runtime does: no host means the public hub.
|
||||
type upstreamRef struct {
|
||||
base, repository, digest string
|
||||
}
|
||||
|
||||
func parseUpstream(from string) (upstreamRef, error) {
|
||||
pin := pinnedOK.FindStringSubmatch(from)
|
||||
if pin == nil {
|
||||
return upstreamRef{}, fmt.Errorf("%q names no digest", from)
|
||||
}
|
||||
name := strings.TrimSuffix(from, "@"+pin[1])
|
||||
if colon := strings.LastIndex(name, ":"); colon > strings.LastIndex(name, "/") {
|
||||
name = name[:colon] // the tag is only what it was called; the digest names the image
|
||||
}
|
||||
if name == "" {
|
||||
return upstreamRef{}, fmt.Errorf("%q is not an image reference", from)
|
||||
}
|
||||
host, repository := "docker.io", name
|
||||
if slash := strings.Index(name, "/"); slash >= 0 && (strings.ContainsAny(name[:slash], ".:") || name[:slash] == "localhost") {
|
||||
host, repository = name[:slash], name[slash+1:]
|
||||
}
|
||||
if host == "docker.io" {
|
||||
host = "registry-1.docker.io"
|
||||
if !strings.Contains(repository, "/") {
|
||||
repository = "library/" + repository
|
||||
}
|
||||
}
|
||||
scheme := "https://"
|
||||
if strings.HasPrefix(host, "localhost") || strings.HasPrefix(host, "127.") {
|
||||
scheme = "http://"
|
||||
}
|
||||
return upstreamRef{base: scheme + host, repository: repository, digest: pin[1]}, nil
|
||||
}
|
||||
|
||||
func (m *Module) copy(ctx context.Context, args map[string]any) (any, error) {
|
||||
module, _ := args["module"].(string)
|
||||
name, _ := args["artifact"].(string)
|
||||
from, _ := args["from"].(string)
|
||||
if !nameOK.MatchString(module) || !nameOK.MatchString(name) {
|
||||
return nil, fmt.Errorf("%q / %q: a module and an artifact are named in lower case letters, digits, dots, dashes and underscores", module, name)
|
||||
}
|
||||
src, err := parseUpstream(from)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("not pinned: %s: %v; an upstream artifact is copied by the digest its manifest pins", name, err)
|
||||
}
|
||||
addr, err := m.registry()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("not stored: %s: %w", name, err)
|
||||
}
|
||||
repository := module + "/" + name
|
||||
c := &copier{m: m, src: src, dstBase: "http://" + addr, dstRepo: repository, done: map[string]bool{}}
|
||||
if err := m.waitForRegistry(ctx, addr, "copying "+from, func() error { return c.copyManifest(ctx, src.digest) }); err != nil {
|
||||
return nil, fmt.Errorf("not stored: %s: %w", name, err)
|
||||
}
|
||||
if err := c.verify(ctx); err != nil {
|
||||
return nil, fmt.Errorf("not stored: %s: %w", name, err)
|
||||
}
|
||||
return CopyResult{Artifact: name, Digest: src.digest, Reference: addr + "/" + repository + "@" + src.digest, Pushed: true}, nil
|
||||
}
|
||||
|
||||
// copier copies one image from its upstream registry into the store's repository.
|
||||
type copier struct {
|
||||
m *Module
|
||||
src upstreamRef
|
||||
dstBase string
|
||||
dstRepo string
|
||||
token string
|
||||
done map[string]bool
|
||||
}
|
||||
|
||||
func (c *copier) client() *http.Client {
|
||||
if c.m.HTTP != nil {
|
||||
return c.m.HTTP
|
||||
}
|
||||
return &http.Client{Timeout: 30 * time.Minute}
|
||||
}
|
||||
|
||||
// get reads from the upstream registry, answering one bearer challenge with an anonymous token.
|
||||
func (c *copier) get(ctx context.Context, path, accept string) (*http.Response, error) {
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.src.base+"/v2/"+c.src.repository+path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if accept != "" {
|
||||
req.Header.Set("Accept", accept)
|
||||
}
|
||||
if c.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
}
|
||||
resp, err := c.client().Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode == http.StatusUnauthorized && attempt == 0 {
|
||||
challenge := resp.Header.Get("Www-Authenticate")
|
||||
resp.Body.Close()
|
||||
if c.token, err = c.fetchToken(ctx, challenge); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("upstream answered %s for %s%s", resp.Status, c.src.repository, path)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
return nil, fmt.Errorf("upstream refused %s%s after a token", c.src.repository, path)
|
||||
}
|
||||
|
||||
var challengeParam = regexp.MustCompile(`(\w+)="([^"]*)"`)
|
||||
|
||||
func (c *copier) fetchToken(ctx context.Context, challenge string) (string, error) {
|
||||
if !strings.HasPrefix(strings.ToLower(challenge), "bearer ") {
|
||||
return "", fmt.Errorf("upstream asks for credentials (%q) and this copy holds none", challenge)
|
||||
}
|
||||
p := map[string]string{}
|
||||
for _, kv := range challengeParam.FindAllStringSubmatch(challenge, -1) {
|
||||
p[kv[1]] = kv[2]
|
||||
}
|
||||
realm := p["realm"]
|
||||
if realm == "" {
|
||||
return "", fmt.Errorf("upstream's challenge %q names no realm", challenge)
|
||||
}
|
||||
q := url.Values{"service": {p["service"]}, "scope": {"repository:" + c.src.repository + ":pull"}}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, realm+"?"+q.Encode(), nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := c.client().Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var tok struct {
|
||||
Token string `json:"token"`
|
||||
Access string `json:"access_token"`
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK || json.NewDecoder(resp.Body).Decode(&tok) != nil || (tok.Token == "" && tok.Access == "") {
|
||||
return "", fmt.Errorf("upstream's token service answered %s", resp.Status)
|
||||
}
|
||||
if tok.Token != "" {
|
||||
return tok.Token, nil
|
||||
}
|
||||
return tok.Access, nil
|
||||
}
|
||||
|
||||
// copyManifest copies the manifest of digest and everything it names, children before the manifest itself.
|
||||
func (c *copier) copyManifest(ctx context.Context, digest string) error {
|
||||
if c.done[digest] {
|
||||
return nil
|
||||
}
|
||||
resp, err := c.get(ctx, "/manifests/"+digest, manifestKind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
|
||||
return fmt.Errorf("upstream served %s for the manifest pinned as %s", got, digest)
|
||||
}
|
||||
var doc struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Config struct{ Digest string } `json:"config"`
|
||||
Layers []struct{ Digest string } `json:"layers"`
|
||||
Manifests []struct{ Digest string } `json:"manifests"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil {
|
||||
return fmt.Errorf("the manifest %s is not JSON: %w", digest, err)
|
||||
}
|
||||
for _, child := range doc.Manifests {
|
||||
if err := c.copyManifest(ctx, child.Digest); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
blobs := []string{}
|
||||
if doc.Config.Digest != "" {
|
||||
blobs = append(blobs, doc.Config.Digest)
|
||||
}
|
||||
for _, l := range doc.Layers {
|
||||
blobs = append(blobs, l.Digest)
|
||||
}
|
||||
for _, b := range blobs {
|
||||
if err := c.copyBlob(ctx, b); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
mediaType := resp.Header.Get("Content-Type")
|
||||
if mediaType == "" {
|
||||
mediaType = doc.MediaType
|
||||
}
|
||||
if err := c.putManifest(ctx, digest, mediaType, body); err != nil {
|
||||
return err
|
||||
}
|
||||
c.done[digest] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *copier) copyBlob(ctx context.Context, digest string) error {
|
||||
if !digestOK.MatchString(digest) {
|
||||
return fmt.Errorf("%q is not a blob digest", digest)
|
||||
}
|
||||
if c.done[digest] {
|
||||
return nil
|
||||
}
|
||||
head, err := http.NewRequestWithContext(ctx, http.MethodHead, c.dstBase+"/v2/"+c.dstRepo+"/blobs/"+digest, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r, err := c.client().Do(head); err != nil {
|
||||
return err
|
||||
} else {
|
||||
r.Body.Close()
|
||||
if r.StatusCode == http.StatusOK {
|
||||
c.done[digest] = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
post, err := http.NewRequestWithContext(ctx, http.MethodPost, c.dstBase+"/v2/"+c.dstRepo+"/blobs/uploads/", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pr, err := c.client().Do(post)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pr.Body.Close()
|
||||
loc, err := pr.Location()
|
||||
if pr.StatusCode != http.StatusAccepted || err != nil {
|
||||
return fmt.Errorf("the store answered %s to a blob upload of %s", pr.Status, digest)
|
||||
}
|
||||
q := loc.Query()
|
||||
q.Set("digest", digest)
|
||||
loc.RawQuery = q.Encode()
|
||||
blob, err := c.get(ctx, "/blobs/"+digest, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer blob.Body.Close()
|
||||
put, err := http.NewRequestWithContext(ctx, http.MethodPut, loc.String(), blob.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
put.ContentLength = blob.ContentLength
|
||||
put.Header.Set("Content-Type", "application/octet-stream")
|
||||
resp, err := c.client().Do(put)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
return fmt.Errorf("the store answered %s to the blob %s", resp.Status, digest)
|
||||
}
|
||||
c.done[digest] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *copier) putManifest(ctx context.Context, digest, mediaType string, body []byte) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.dstBase+"/v2/"+c.dstRepo+"/manifests/"+digest, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", mediaType)
|
||||
resp, err := c.client().Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("the store answered %s to the manifest %s", resp.Status, digest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verify reads back what the store serves for the pinned digest: only the store can say.
|
||||
func (c *copier) verify(ctx context.Context) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, c.dstBase+"/v2/"+c.dstRepo+"/manifests/"+c.src.digest, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Accept", manifestKind)
|
||||
resp, err := c.client().Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
if got := resp.Header.Get("Docker-Content-Digest"); resp.StatusCode != http.StatusOK || got != c.src.digest {
|
||||
return fmt.Errorf("the store answered %s with digest %q for the pinned %s", resp.Status, got, c.src.digest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Plain is a long step that needs nothing of the machine: no guard. The copy runs no buildah.
|
||||
func (m *Module) Plain(tool string, run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
|
||||
return func(args map[string]any) (any, error) {
|
||||
id, err := buildstep.Start(m.Ctx, m.Steps, m.Node, tool, func(ctx context.Context) (any, error) { return run(ctx, args) })
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, _ := delivery.StepKey(m.Node, id)
|
||||
return map[string]any{"step": id, "key": key, "state": delivery.StepRunning}, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func digestOf(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) }
|
||||
|
||||
// memRegistry is a registry in memory: blobs and manifests by digest, per repository.
|
||||
type memRegistry struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte // repo|digest
|
||||
manifests map[string][]byte // repo|digest
|
||||
types map[string]string
|
||||
puts []string
|
||||
srv *httptest.Server
|
||||
}
|
||||
|
||||
func newMem(t *testing.T) *memRegistry {
|
||||
r := &memRegistry{blobs: map[string][]byte{}, manifests: map[string][]byte{}, types: map[string]string{}}
|
||||
r.srv = httptest.NewServer(http.HandlerFunc(r.serve))
|
||||
t.Cleanup(r.srv.Close)
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *memRegistry) addr() string { return r.srv.Listener.Addr().String() }
|
||||
|
||||
func (r *memRegistry) serve(w http.ResponseWriter, req *http.Request) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
path := strings.TrimPrefix(req.URL.Path, "/v2/")
|
||||
if i := strings.Index(path, "/blobs/uploads/"); i >= 0 {
|
||||
if req.Method == http.MethodPost {
|
||||
w.Header().Set("Location", "/v2/"+path[:i]+"/blobs/uploads/u1")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
return
|
||||
}
|
||||
body, _ := io.ReadAll(req.Body)
|
||||
d := req.URL.Query().Get("digest")
|
||||
if digestOf(body) != d {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
r.blobs[path[:i]+"|"+d] = body
|
||||
r.puts = append(r.puts, "blob "+d)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
return
|
||||
}
|
||||
for _, kind := range []string{"blobs", "manifests"} {
|
||||
if i := strings.Index(path, "/"+kind+"/"); i >= 0 {
|
||||
key := path[:i] + "|" + path[i+len(kind)+2:]
|
||||
store := r.blobs
|
||||
if kind == "manifests" {
|
||||
store = r.manifests
|
||||
}
|
||||
if req.Method == http.MethodPut {
|
||||
body, _ := io.ReadAll(req.Body)
|
||||
if digestOf(body) != path[i+len(kind)+2:] {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
r.manifests[key] = body
|
||||
r.types[key] = req.Header.Get("Content-Type")
|
||||
r.puts = append(r.puts, "manifest "+path[i+len(kind)+2:])
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
return
|
||||
}
|
||||
body, ok := store[key]
|
||||
if !ok {
|
||||
http.NotFound(w, req)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Docker-Content-Digest", path[i+len(kind)+2:])
|
||||
w.Header().Set("Content-Type", r.types[key])
|
||||
if req.Method == http.MethodHead {
|
||||
return
|
||||
}
|
||||
w.Write(body)
|
||||
return
|
||||
}
|
||||
}
|
||||
http.NotFound(w, req)
|
||||
}
|
||||
|
||||
// upstreamImage puts a two-platform index (two manifests, each with a config and a layer) into r and answers the index digest.
|
||||
func upstreamImage(r *memRegistry, repo string) string {
|
||||
var children []map[string]string
|
||||
for _, tag := range []string{"amd64", "arm64"} {
|
||||
cfg, layer := []byte(`{"arch":"`+tag+`"}`), []byte("layer-"+tag)
|
||||
r.blobs[repo+"|"+digestOf(cfg)], r.blobs[repo+"|"+digestOf(layer)] = cfg, layer
|
||||
mf, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"config": map[string]string{"digest": digestOf(cfg)}, "layers": []map[string]string{{"digest": digestOf(layer)}}})
|
||||
r.manifests[repo+"|"+digestOf(mf)] = mf
|
||||
r.types[repo+"|"+digestOf(mf)] = "application/vnd.oci.image.manifest.v1+json"
|
||||
children = append(children, map[string]string{"digest": digestOf(mf)})
|
||||
}
|
||||
idx, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json", "manifests": children})
|
||||
d := digestOf(idx)
|
||||
r.manifests[repo+"|"+d] = idx
|
||||
r.types[repo+"|"+d] = "application/vnd.oci.image.index.v1+json"
|
||||
return d
|
||||
}
|
||||
|
||||
func runCopy(t *testing.T, m *Module, states *keptStates, args map[string]any) (CopyResult, delivery.Step) {
|
||||
t.Helper()
|
||||
if _, err := tool(t, m, CopyTool)(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res CopyResult
|
||||
if st.State == delivery.StepPassed {
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
|
||||
// ADR 0498 §4 / AC: the copy leaves the pinned digest in the store and answers it, with the whole index (every
|
||||
// manifest it lists, every blob), and runs no buildah.
|
||||
func TestTheCopyLeavesThePinnedDigestInTheStoreAndAnswersIt(t *testing.T) {
|
||||
up, store := newMem(t), newMem(t)
|
||||
digest := upstreamImage(up, "library/nats")
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
m.HTTP = nil
|
||||
reg := store.addr()
|
||||
writeFile(t, m.RegistryFile, reg+"\n")
|
||||
from := up.addr() + "/library/nats:2.11.17-alpine@" + digest
|
||||
res, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": from})
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
if res.Digest != digest || res.Reference != reg+"/nats/server@"+digest || !res.Pushed || res.Artifact != "server" {
|
||||
t.Errorf("result = %+v", res)
|
||||
}
|
||||
if _, ok := store.manifests["nats/server|"+digest]; !ok {
|
||||
t.Fatal("the index is not in the store")
|
||||
}
|
||||
if len(store.manifests) != 3 || len(store.blobs) != 4 {
|
||||
t.Errorf("store holds %d manifests and %d blobs, want 3 and 4", len(store.manifests), len(store.blobs))
|
||||
}
|
||||
if len(f.lines()) != 0 {
|
||||
t.Errorf("the copy ran commands: %v", f.lines())
|
||||
}
|
||||
// A second copy writes nothing new but the manifests.
|
||||
store.puts = nil
|
||||
if _, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": from}); st.State != delivery.StepPassed {
|
||||
t.Fatalf("again: %+v", st)
|
||||
}
|
||||
for _, p := range store.puts {
|
||||
if strings.HasPrefix(p, "blob ") {
|
||||
t.Errorf("a blob the store holds was sent again: %s", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0498 §4 / AC: an upstream without a digest fails `not pinned:`, and nothing is read from the upstream.
|
||||
func TestAnUpstreamWithoutADigestFailsNotPinned(t *testing.T) {
|
||||
up, store := newMem(t), newMem(t)
|
||||
upstreamImage(up, "library/nats")
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
writeFile(t, m.RegistryFile, store.addr()+"\n")
|
||||
_, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": up.addr() + "/library/nats:2.11.17-alpine"})
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "not pinned: server") {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
if len(store.puts) != 0 {
|
||||
t.Errorf("the store was written: %v", store.puts)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest the upstream serves under another digest than the pin is not copied; the failure is `not stored:`.
|
||||
func TestAnUpstreamServingAnotherManifestThanPinnedIsNotStored(t *testing.T) {
|
||||
up, store := newMem(t), newMem(t)
|
||||
d := upstreamImage(up, "library/nats")
|
||||
up.manifests["library/nats|"+d] = []byte(`{"tampered":true}`)
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
writeFile(t, m.RegistryFile, store.addr()+"\n")
|
||||
_, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": up.addr() + "/library/nats@" + d})
|
||||
if st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, "not stored: server:") {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
if len(store.manifests) != 0 {
|
||||
t.Errorf("the store holds %v", store.manifests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAShortNameIsQualifiedAsTheHubsAndAHostIsKept(t *testing.T) {
|
||||
d := "sha256:" + strings.Repeat("a", 64)
|
||||
for from, want := range map[string]upstreamRef{
|
||||
"alpine@" + d: {"https://registry-1.docker.io", "library/alpine", d},
|
||||
"nats:2.11.17-alpine@" + d: {"https://registry-1.docker.io", "library/nats", d},
|
||||
"grafana/grafana:11@" + d: {"https://registry-1.docker.io", "grafana/grafana", d},
|
||||
"ghcr.io/org/app:1@" + d: {"https://ghcr.io", "org/app", d},
|
||||
"localhost:5000/x/y@" + d: {"http://localhost:5000", "x/y", d},
|
||||
} {
|
||||
got, err := parseUpstream(from)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("%s = %+v, %v; want %+v", from, got, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -26,11 +26,6 @@ var (
|
||||
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
|
||||
const ForgeAlias = "mesh-forge"
|
||||
|
||||
// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it
|
||||
// fetches and every round it checks itself. A workspace lies in the module's work directory, never in its home
|
||||
// (ADR 0466 §1).
|
||||
const WorkspaceAge = 24 * time.Hour
|
||||
|
||||
// FetchResult is what a passed fetch step says: the workspace it made and the commit in it, and, only when the call
|
||||
// gave a base, the merge preview of ADR 0484 clause 3 (the trunk commit, the merge commit of the head onto it, the
|
||||
// files in conflict, the paths the merge changes and the tree of each module directory). A head that does not
|
||||
@@ -66,7 +61,9 @@ func (m *Module) fetchTool() StepTool {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
// fetch makes a workspace and fills it. The workspace is held while the fetch runs, and removed when the fetch fails,
|
||||
// since nobody learns its id to release it (issue 1839).
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
commit, _ := args["commit"].(string)
|
||||
ref, _ := args["ref"].(string)
|
||||
@@ -82,6 +79,19 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
}
|
||||
m.CleanWorkspaces(m.clock())
|
||||
id := buildstep.NewID()
|
||||
done := m.spaces().Hold(id)
|
||||
defer func() {
|
||||
done()
|
||||
if err != nil {
|
||||
if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
|
||||
m.Say("the failed fetch's workspace %s: %v", id, rerr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return m.fetchInto(ctx, repository, commit, ref, base, id)
|
||||
}
|
||||
|
||||
func (m *Module) fetchInto(ctx context.Context, repository, commit, ref, base, id string) (any, error) {
|
||||
tree := filepath.Join(m.Work, id, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
return nil, err
|
||||
@@ -124,21 +134,10 @@ func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base
|
||||
return err
|
||||
}
|
||||
|
||||
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
|
||||
// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
|
||||
// net for a caller that never released one (issue 1839, ADR 0509).
|
||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||
entries, err := os.ReadDir(m.Work)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
|
||||
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
|
||||
}
|
||||
}
|
||||
m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
|
||||
}
|
||||
|
||||
// clock is the time now: Now when a test set it, else the wall clock.
|
||||
|
||||
@@ -53,6 +53,11 @@ func realModule(t *testing.T) (*Module, *keptStates) {
|
||||
m := &Module{Node: "ace", Account: u.Username, UID: u.Uid, Home: home, Work: t.TempDir(), Store: t.TempDir(),
|
||||
Steps: states.putStep, Ctx: context.Background(), Say: t.Logf, Run: StepRunner(home),
|
||||
FetchOptions: []buildstep.Option{buildstep.AllowFileRemote()}}
|
||||
// buildah being installed is not enough: inside the check's sandbox rootless buildah cannot make its user
|
||||
// namespace, so the module's own guard decides, as it does before every step (issue 1894).
|
||||
if err := m.Check(context.Background()); err != nil {
|
||||
t.Skipf("NOT RUN: this machine cannot build images where the check runs (%v); proven live on a build node", err)
|
||||
}
|
||||
return m, states
|
||||
}
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ func main() {
|
||||
Work: WorkDir,
|
||||
Store: StoreDir,
|
||||
}
|
||||
m.Floor = floorOf(m.Node)
|
||||
if u, err := user.Current(); err == nil {
|
||||
m.Home, m.Account, m.UID = u.HomeDir, u.Username, u.Uid
|
||||
}
|
||||
|
||||
@@ -4,8 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
@@ -47,6 +46,12 @@ type Module struct {
|
||||
RegistryPause time.Duration
|
||||
// Now is the clock; time.Now when nil.
|
||||
Now func() time.Time
|
||||
// Floor refuses every fetch and build while the work or the image store's filesystem has less free than it (issue
|
||||
// 1839); main sets floorOf(node), and a zero Floor checks nothing.
|
||||
Floor buildstep.Floor
|
||||
|
||||
spacesOnce sync.Once
|
||||
workspaces *buildstep.Workspaces
|
||||
}
|
||||
|
||||
// Look is what the module found out about its machine.
|
||||
@@ -60,6 +65,11 @@ type Look struct {
|
||||
// missing is a *buildstep.Refusal naming it, so a step ends refused and the health says the same.
|
||||
func (m *Module) look(ctx context.Context) (Look, error) {
|
||||
l := Look{Account: m.Account}
|
||||
// The runtime directory comes first: buildah refuses before anything else when its XDG_RUNTIME_DIR is not there
|
||||
// (issue 1884), so a look that made it only after `buildah --version` never made it at all.
|
||||
if err := m.ensureRuntimeDir(); err != nil {
|
||||
return l, &buildstep.Refusal{Text: err.Error()}
|
||||
}
|
||||
var err error
|
||||
if l.Buildah, err = m.buildahVersion(ctx); err != nil {
|
||||
return l, &buildstep.Refusal{Text: err.Error()}
|
||||
@@ -67,11 +77,6 @@ func (m *Module) look(ctx context.Context) (Look, error) {
|
||||
if l.UIDs, l.GIDs, err = m.ranges(); err != nil {
|
||||
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
|
||||
}
|
||||
for _, d := range []string{m.Store, filepath.Join(m.Store, "xdg")} {
|
||||
if err := os.MkdirAll(d, 0o700); err != nil {
|
||||
return l, err
|
||||
}
|
||||
}
|
||||
if err := m.userNamespace(ctx); err != nil {
|
||||
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
|
||||
}
|
||||
@@ -106,6 +111,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
|
||||
if err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made", l.Buildah, l.Account, l.UIDs, l.GIDs))
|
||||
}
|
||||
|
||||
@@ -128,6 +136,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string
|
||||
if err := m.Check(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer m.holdArg(args)()
|
||||
return run(ctx, args)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -144,6 +156,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any,
|
||||
if err := m.Check(m.Ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer m.holdArg(args)()
|
||||
return run(m.Ctx, args)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,6 +111,25 @@ func (m *Module) storeFlags() []string {
|
||||
// env is what buildah is given beyond the base environment: a runtime directory of its own inside the store, so
|
||||
// nothing is read from the account's runtime directory, and no registry credential file anywhere (the module has none
|
||||
// and REGISTRY_AUTH_FILE is never set).
|
||||
//
|
||||
// Every call makes the runtime directory first if it is not there (issue 1884): a directory removed after the last look
|
||||
// is made again before buildah runs. A failure to make it is not lost: buildah then names the directory in its own
|
||||
// error, and the next look refuses with it.
|
||||
func (m *Module) env() []string {
|
||||
return []string{"XDG_RUNTIME_DIR=" + filepath.Join(m.Store, "xdg")}
|
||||
_ = m.ensureRuntimeDir()
|
||||
return []string{"XDG_RUNTIME_DIR=" + m.runtimeDir()}
|
||||
}
|
||||
|
||||
// runtimeDir is buildah's runtime directory, inside the store.
|
||||
func (m *Module) runtimeDir() string { return filepath.Join(m.Store, "xdg") }
|
||||
|
||||
// ensureRuntimeDir makes the store and the runtime directory inside it, 0700 and under the module's own account (the
|
||||
// process runs as it), when they are not there; it changes nothing that is.
|
||||
func (m *Module) ensureRuntimeDir() error {
|
||||
for _, d := range []string{m.Store, m.runtimeDir()} {
|
||||
if err := os.MkdirAll(d, 0o700); err != nil {
|
||||
return fmt.Errorf("buildah's runtime directory %s cannot be made: %w", m.runtimeDir(), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
const pin = "@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00"
|
||||
|
||||
// 1824: a short base image name is qualified the way the controller's parseReference does (the public hub, `library/`
|
||||
// for a single name), because buildah has no unqualified-search registry here; a qualified name is left as it is.
|
||||
func TestAShortBaseImageNameIsQualifiedAsTheHubsAndAQualifiedOneIsKept(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"nats" + pin: "docker.io/library/nats" + pin,
|
||||
"nats:2.11.17-alpine" + pin: "docker.io/library/nats:2.11.17-alpine" + pin,
|
||||
"n8nio/n8n" + pin: "docker.io/n8nio/n8n" + pin,
|
||||
"docker.io/library/alpine" + pin: "docker.io/library/alpine" + pin,
|
||||
"ghcr.io/org/app:1" + pin: "ghcr.io/org/app:1" + pin,
|
||||
"localhost:5000/x/y" + pin: "localhost:5000/x/y" + pin,
|
||||
"registry-1.docker.io/library/a": "registry-1.docker.io/library/a",
|
||||
} {
|
||||
if got := qualifyImage(in); got != want {
|
||||
t.Errorf("%s = %s, want %s", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBuildArgsCarryTheQualifiedName(t *testing.T) {
|
||||
m := catalogue.Manifest{Build: &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "NATS_BASE", Image: "nats" + pin}}}}
|
||||
if got, want := imageArgs(m), []string{"--build-arg", "NATS_BASE=docker.io/library/nats" + pin}; !slices.Equal(got, want) {
|
||||
t.Errorf("%v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every fetch and build is refused while
|
||||
// the work or image store's filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases
|
||||
// it, and swept unheld past buildstep.UnheldBound.
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// ReleaseTool removes a workspace whose caller's steps have ended.
|
||||
const ReleaseTool = "build_image_release"
|
||||
|
||||
// floorOf is the floor the module keeps on node: build-agent's.
|
||||
func floorOf(node string) buildstep.Floor {
|
||||
return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor}
|
||||
}
|
||||
|
||||
// room refuses while the work or the image store's filesystem is under the floor.
|
||||
func (m *Module) room() error { return m.Floor.Check(m.Work, m.Store) }
|
||||
|
||||
// spaces are the module's workspaces under Work.
|
||||
func (m *Module) spaces() *buildstep.Workspaces {
|
||||
m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} })
|
||||
return m.workspaces
|
||||
}
|
||||
|
||||
// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none.
|
||||
func (m *Module) holdArg(args map[string]any) func() {
|
||||
ws, _ := args["workspace"].(string)
|
||||
if ws == "" {
|
||||
return func() {}
|
||||
}
|
||||
return m.spaces().Hold(ws)
|
||||
}
|
||||
|
||||
// releaseTool is build_image_release: no guard and no floor, since it only frees room.
|
||||
func (m *Module) releaseTool() stdio.Tool {
|
||||
return stdio.Tool{Name: ReleaseTool,
|
||||
Description: "Remove a workspace whose steps have ended (fetch, build): its caller says so once its last step " +
|
||||
"answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " +
|
||||
"is removed two hours after its last use.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"},
|
||||
}, "required": []string{"workspace"}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ws, _ := args["workspace"].(string)
|
||||
if err := m.spaces().Release(ws); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"released": ws}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509).
|
||||
func (m *Module) sweepSaid(removed []string) {
|
||||
if len(removed) > 0 && m.Say != nil {
|
||||
m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
|
||||
|
||||
func underFloor(node string) buildstep.Floor {
|
||||
f := floorOf(node)
|
||||
f.Free = func(string) (uint64, error) { return 4 << 30, nil }
|
||||
return f
|
||||
}
|
||||
|
||||
// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
|
||||
func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
|
||||
if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
|
||||
t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC1: under the floor a fetch and a build are refused, naming the node, what is free and the floor, nothing runs,
|
||||
// and the module is unhealthy saying so, so the builder chooses another node.
|
||||
func TestUnderTheFloorAFetchAndABuildAreRefusedAndTheModuleIsUnhealthy(t *testing.T) {
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
m.Floor = underFloor("ace")
|
||||
for name, args := range map[string]map[string]any{
|
||||
FetchTool: {"repository": "novox/mesh-catalog", "commit": commitSHA},
|
||||
BuildTool: {"workspace": "abc1", "module": "modules/m", "artifact": "image"},
|
||||
} {
|
||||
if _, err := tool(t, m, name)(args); err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
|
||||
t.Errorf("%s ended %s: %q, want refused: %q", name, st.State, st.Reason, underTheFloor)
|
||||
}
|
||||
}
|
||||
for _, l := range f.lines() {
|
||||
if strings.HasPrefix(l, "git ") || strings.Contains(l, " bud ") {
|
||||
t.Errorf("a refused step ran %q", l)
|
||||
}
|
||||
}
|
||||
if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
|
||||
t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC2: build_image_release removes the workspace at once, even under the floor; a second release is no error.
|
||||
func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
|
||||
m := moduleWith(t, newFake(t), newStates())
|
||||
m.Floor = underFloor("ace")
|
||||
ws := filepath.Join(m.Work, "abc1")
|
||||
os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, err := tool(t, m, ReleaseTool)(map[string]any{"workspace": "abc1"}); err != nil {
|
||||
t.Fatalf("release %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(ws); !os.IsNotExist(err) {
|
||||
t.Errorf("the workspace is still there after its release")
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
|
||||
func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if name == "git" && strings.Contains(strings.Join(args, " "), " fetch ") {
|
||||
return forgekey.Ran{Status: 128, Stderr: "fatal: unreachable"}
|
||||
}
|
||||
return f.run(ctx, env, name, args...)
|
||||
}
|
||||
if _, err := tool(t, m, FetchTool)(map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepFailed {
|
||||
t.Fatalf("the step ended %s (%s), want failed", st.State, st.Reason)
|
||||
}
|
||||
if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
|
||||
t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is.
|
||||
func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
|
||||
m := &Module{Work: t.TempDir()}
|
||||
for _, name := range []string{"held1", "lost1"} {
|
||||
os.MkdirAll(filepath.Join(m.Work, name, "tree"), 0o755)
|
||||
}
|
||||
done := m.spaces().Hold("held1")
|
||||
defer done()
|
||||
when := time.Now().Add(-3 * time.Hour)
|
||||
for _, name := range []string{"held1", "lost1"} {
|
||||
os.Chtimes(filepath.Join(m.Work, name), when, when)
|
||||
}
|
||||
m.CleanWorkspaces(time.Now())
|
||||
if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
|
||||
t.Errorf("the held workspace was swept")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
|
||||
t.Errorf("the unheld workspace three hours old was kept")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// buildahLike refuses as buildah does when the XDG_RUNTIME_DIR it is given is not there (seen on ace, issue 1884):
|
||||
// "lstat <dir>: no such file or directory", before it does anything else.
|
||||
func buildahLike(t *testing.T, run forgekey.Runner, seen *[]string) forgekey.Runner {
|
||||
t.Helper()
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if name == "buildah" {
|
||||
for _, e := range env {
|
||||
dir, ok := strings.CutPrefix(e, "XDG_RUNTIME_DIR=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Lstat(dir); err != nil {
|
||||
*seen = append(*seen, "missing")
|
||||
return forgekey.Ran{Status: 125, Stderr: "Error: lstat " + dir + ": no such file or directory\n"}
|
||||
}
|
||||
*seen = append(*seen, "present")
|
||||
}
|
||||
}
|
||||
return run(ctx, env, name, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// 1884/AC1: given a store directory that exists without its runtime directory, when the health is looked at, then the
|
||||
// module makes the runtime directory (0700) before it runs buildah, and the health is healthy.
|
||||
func TestHealthMakesTheRuntimeDirectoryBeforeBuildah(t *testing.T) {
|
||||
f := newFake(t)
|
||||
m := moduleWith(t, f, newStates())
|
||||
var seen []string
|
||||
m.Run = buildahLike(t, m.Run, &seen)
|
||||
xdg := filepath.Join(m.Store, "xdg")
|
||||
if _, err := os.Stat(xdg); !os.IsNotExist(err) {
|
||||
t.Fatalf("the test needs a store without its runtime directory: %v", err)
|
||||
}
|
||||
a := m.Health(context.Background())
|
||||
if a.State != health.StateHealthy || !a.Healthy {
|
||||
t.Fatalf("health = %+v, want healthy", a)
|
||||
}
|
||||
st, err := os.Stat(xdg)
|
||||
if err != nil {
|
||||
t.Fatalf("the runtime directory was not made: %v", err)
|
||||
}
|
||||
if st.Mode().Perm() != 0o700 || !st.IsDir() {
|
||||
t.Errorf("the runtime directory is %v, want a directory 0700", st.Mode())
|
||||
}
|
||||
for _, s := range seen {
|
||||
if s != "present" {
|
||||
t.Fatalf("buildah ran without its runtime directory: %v", seen)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1884/AC2: given a module whose runtime directory was removed after its last look, when a build step runs buildah, then
|
||||
// the runtime directory is made again before buildah runs.
|
||||
func TestEveryBuildahRunMakesTheRuntimeDirectoryAgain(t *testing.T) {
|
||||
f := newFake(t)
|
||||
m := moduleWith(t, f, newStates())
|
||||
var seen []string
|
||||
m.Run = buildahLike(t, m.Run, &seen)
|
||||
if err := m.Check(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Store, "xdg")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen = nil
|
||||
m.remove("localhost/x:y")
|
||||
if len(seen) != 1 || seen[0] != "present" {
|
||||
t.Fatalf("buildah's runs saw their runtime directory %v, want [present]", seen)
|
||||
}
|
||||
}
|
||||
@@ -25,14 +25,18 @@ type StepTool struct {
|
||||
}
|
||||
|
||||
// StepTools are the module's step tools.
|
||||
func (m *Module) StepTools() []StepTool { return []StepTool{m.fetchTool(), m.buildTool()} }
|
||||
func (m *Module) StepTools() []StepTool {
|
||||
return []StepTool{m.fetchTool(), m.buildTool(), m.copyTool()}
|
||||
}
|
||||
|
||||
// Tools is every tool the module serves: its step tools behind the guard, its read tools, its status and its health.
|
||||
func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
||||
tools := []stdio.Tool{}
|
||||
for _, s := range steps {
|
||||
t := stdio.Tool{Name: s.Name, Description: s.Description, Input: s.Input}
|
||||
if s.Long {
|
||||
if s.Name == CopyTool {
|
||||
t.Run = m.Plain(s.Name, s.Run) // a copy needs no buildah, so the buildah guard does not stand in front of it
|
||||
} else if s.Long {
|
||||
t.Run = m.Long(s.Name, s.Run)
|
||||
} else {
|
||||
t.Run = m.Short(s.Run)
|
||||
@@ -40,6 +44,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
||||
tools = append(tools, t)
|
||||
}
|
||||
tools = append(tools, m.ReadTools()...)
|
||||
tools = append(tools, m.releaseTool())
|
||||
return append(tools,
|
||||
stdio.Tool{Name: StatusTool,
|
||||
Description: "What the image build module is on this machine: the buildah installed, the account's subordinate " +
|
||||
|
||||
@@ -3,7 +3,7 @@ module meshbuildimage
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52 h1:5k04XjpfYCq6j7rCM6Dy6FfrsBk5iUJlbwGf6WhUOUc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022163107-d1af87a83a52/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
"tools": [
|
||||
"build_image_fetch",
|
||||
"build_image_build",
|
||||
"build_image_copy",
|
||||
"build_image_release",
|
||||
"build_image_step",
|
||||
"build_image_steps",
|
||||
"build_image_status",
|
||||
|
||||
@@ -57,7 +57,11 @@ toolchain install: a long step answers at once with its id and ends on `<node>.<
|
||||
removed). The mesh's registry reaches `npm ci` through a private config file (`--userconfig`), so
|
||||
other `@novox/` packages install. A module with packages and no lock does not fail the fetch: its result lists it
|
||||
under `refused` with the reason, and a build of that bundle fails naming it (issue 1459). Every command runs through `forgekey.ExecRunnerClean(0)` with PATH, HOME, the locale and its own
|
||||
words, nothing of this process's environment. Workspaces older than a day are removed at each fetch.
|
||||
words, nothing of this process's environment. A failed fetch removes its own workspace; `build_node_release {workspace}` removes one at once (no guard, no
|
||||
floor; its caller calls it once its last step answered), and one no caller released is removed two hours after its
|
||||
last use (ADR 0509), never while a step holds it. Every step is refused, `refused: <node> has N GiB free under the
|
||||
floor of 20 GiB (on <dir>)`, and the health is unhealthy with that text, while the work or the toolchain cache
|
||||
filesystem has less than 20 GiB free (issue 1839).
|
||||
The arguments are mesh-sdk's `buildstep.FetchRequest` (ADR 0499 §4): with `dir` and `paths` (from the builder, never
|
||||
from a manifest) the workspace is narrowed to those paths, the manifest and the lock files before anything else. A
|
||||
member is built against the SDK version its own lock names (the lead's decision L10 of R4e, ADR 0500 §5-§6): the
|
||||
|
||||
@@ -3,18 +3,25 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// realMachine installs the real toolchain from the module's own lock (npm ci --ignore-scripts, from the mesh's
|
||||
// registry for @novox, which reads without a credential) and makes two workspaces holding one fixture bundle.
|
||||
// realMachine gives the tests the real toolchain of the module's own lock and makes two workspaces holding one fixture
|
||||
// bundle. The toolchain is the one this node's installed Node build module keeps (ADR 0533 §1): the integration level
|
||||
// is given its toolchains read-only and no network, so it is copied into the test's own cache from there. Only where
|
||||
// no toolchain is installed (a desk) is it installed with `npm ci --ignore-scripts` from the mesh's registry, as the
|
||||
// module does, which needs the network.
|
||||
func realMachine(t *testing.T) (*Module, string) {
|
||||
t.Helper()
|
||||
for _, p := range []string{"node", "npm", "bwrap"} { // bwrap: tsc, esbuild and the scripts run shut off (1309)
|
||||
@@ -40,8 +47,11 @@ func realMachine(t *testing.T) (*Module, string) {
|
||||
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil
|
||||
}}}
|
||||
withStore(t, m)
|
||||
if from := installedToolchain(t, filepath.Join(CacheDir, "toolchain"), filepath.Join(home, "toolchain", "package-lock.json")); from != "" {
|
||||
copyToolchain(t, from, m.Toolchain.Paths.Dir(m.Recipe.Version()))
|
||||
}
|
||||
if err := m.Toolchain.Ensure(context.Background(), m.Recipe.Version()); err != nil {
|
||||
t.Fatal(err)
|
||||
t.Fatalf("no toolchain of this lock is installed under %s, and installing it from the registry failed: %v", filepath.Join(CacheDir, "toolchain"), err)
|
||||
}
|
||||
for _, id := range []string{"w1", "w2"} {
|
||||
for p, body := range map[string]string{
|
||||
@@ -115,3 +125,74 @@ func TestAPackageWhoseTestsImportItsBuildPassesTheUnitLevel(t *testing.T) {
|
||||
t.Errorf("result = %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// installedToolchain is the toolchain directory under toolchains that this node's Node build module installed: the
|
||||
// one installed from the lock at lockPath, byte for byte, or, when the change carries a lock no delivered release has
|
||||
// installed yet, the one installed last, said in the test's log (ADR 0533 §3: a new lock is proven by the module that
|
||||
// installs it on its first node). "" when none is there or the directory cannot be read: no Node build module here.
|
||||
func installedToolchain(t *testing.T, toolchains, lockPath string) string {
|
||||
t.Helper()
|
||||
want, err := os.ReadFile(lockPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries, err := os.ReadDir(toolchains)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
last, lastAt := "", time.Time{}
|
||||
for _, e := range entries {
|
||||
dir := filepath.Join(toolchains, e.Name())
|
||||
mark, err := os.Stat(filepath.Join(dir, installedMark))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
got, err := os.ReadFile(filepath.Join(dir, "package-lock.json"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if bytes.Equal(got, want) {
|
||||
t.Logf("the toolchain is this node's installed %s, of this change's lock", dir)
|
||||
return dir
|
||||
}
|
||||
if mark.ModTime().After(lastAt) {
|
||||
last, lastAt = dir, mark.ModTime()
|
||||
}
|
||||
}
|
||||
if last != "" {
|
||||
t.Logf("NOT THIS LOCK: no toolchain of this change's lock is installed here; the tests run with the one installed last, %s", last)
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
// copyToolchain copies the installed toolchain from into to whole, files and folders with their modes, so the test's
|
||||
// steps run behind a toolchain of their own and the installed one is only read.
|
||||
func copyToolchain(t *testing.T, from, to string) {
|
||||
t.Helper()
|
||||
err := filepath.WalkDir(from, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(from, p)
|
||||
dst := filepath.Join(to, rel)
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case d.IsDir():
|
||||
return os.MkdirAll(dst, 0o755)
|
||||
case info.Mode().IsRegular():
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(dst, raw, info.Mode().Perm()|0o600)
|
||||
default:
|
||||
return fmt.Errorf("%s is neither a file nor a folder", p)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("copying the installed toolchain %s: %v", from, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,9 +33,6 @@ const ForgeAlias = "mesh-forge"
|
||||
// sdkPackage is the package the toolchain carries and a module's own dependencies never supply.
|
||||
const sdkPackage = "@novox/mesh-sdk"
|
||||
|
||||
// WorkspaceAge is how long a workspace is kept.
|
||||
const WorkspaceAge = 24 * time.Hour
|
||||
|
||||
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
|
||||
var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
@@ -105,20 +102,36 @@ func (m *Module) fetchTool() StepTool {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
// fetch makes a workspace and fills it. The workspace is held while the fetch runs, and removed when the fetch fails,
|
||||
// since nobody learns its id to release it (issue 1839).
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
|
||||
// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
|
||||
// named paths, the workspace is narrowed to them (buildstep.Narrow).
|
||||
req, err := buildstep.ParseFetchRequest(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
||||
group, err := groupPackageArg(args["package"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
id := buildstep.NewID()
|
||||
done := m.spaces().Hold(id)
|
||||
defer func() {
|
||||
done()
|
||||
if err != nil {
|
||||
if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
|
||||
m.Say("the failed fetch's workspace %s: %v", id, rerr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return m.fetchInto(ctx, req, group, id)
|
||||
}
|
||||
|
||||
func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, group *groupPackage, id string) (any, error) {
|
||||
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
||||
var err error
|
||||
root := filepath.Join(m.Work, id)
|
||||
tree := filepath.Join(root, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
@@ -354,19 +367,8 @@ func removeSDKCopies(nm string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
|
||||
// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
|
||||
// net for a caller that never released one (issue 1839, ADR 0509).
|
||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||
entries, err := os.ReadDir(m.Work)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
|
||||
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
|
||||
}
|
||||
}
|
||||
m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ func main() {
|
||||
Bubblewrap: ProbeBubblewrap,
|
||||
RegistryFile: filepath.Join(home, "registry"),
|
||||
}
|
||||
m.Floor = floorOf(m.Node)
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
if m.Node != "" {
|
||||
go m.Start(ctx)
|
||||
|
||||
@@ -59,8 +59,14 @@ type Module struct {
|
||||
// unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing.
|
||||
Bubblewrap func(ctx context.Context) (string, error)
|
||||
|
||||
mu sync.Mutex // guards Recipe
|
||||
install sync.Mutex // one install at a time
|
||||
// Floor refuses every step while the work or toolchain cache filesystem has less free than it (issue 1839); main
|
||||
// sets floorOf(node), and a zero Floor checks nothing.
|
||||
Floor buildstep.Floor
|
||||
|
||||
spacesOnce sync.Once
|
||||
workspaces *buildstep.Workspaces
|
||||
mu sync.Mutex // guards Recipe
|
||||
install sync.Mutex // one install at a time
|
||||
}
|
||||
|
||||
func (m *Module) recipe() Recipe {
|
||||
@@ -122,6 +128,7 @@ func (m *Module) Start(ctx context.Context) {
|
||||
if err := m.Prepare(ctx); err != nil && m.Say != nil {
|
||||
m.Say("the toolchain: %v", err)
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
@@ -138,6 +145,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
|
||||
if err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("node %s installed, the version the module builds with (recipe %s)", installed, m.recipe().Version()))
|
||||
}
|
||||
|
||||
@@ -150,6 +160,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string
|
||||
if err := m.Prepare(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer m.holdArg(args)()
|
||||
return run(ctx, args)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -166,6 +180,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any,
|
||||
if err := m.Prepare(m.Ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer m.holdArg(args)()
|
||||
return run(m.Ctx, args)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,7 +117,7 @@ func TestANodeThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T)
|
||||
{Name: "short_probe", Run: run},
|
||||
}, (&Module{}).StepTools()...)
|
||||
for _, tool := range m.Tools(steps) {
|
||||
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool {
|
||||
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool {
|
||||
continue
|
||||
}
|
||||
var reason string
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user